From e0366372bc3b90f41788864367d9a90e19b8812e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 14 Nov 2024 10:53:03 +0100 Subject: [PATCH] refactor(iroh-willow): Move `Spaces` RPC & client to `iroh-willow` crate (#2926) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - Move all the spaces-specific RPC stuff from `iroh` to `iroh-willow` - Add `add_node_addr` and `node_addr` RPC calls, remove use of `.net()` sub-rpc - Pass in `iroh_blobs::store::Store` instead of requring `iroh_blobs` RPC - Revert all changes in `iroh` - Update to latest crates & `main` ## Change checklist - [ ] Self-review. - [ ] Documentation updates following the [style guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text), if relevant. - [ ] Tests if relevant. - [ ] All breaking changes documented. --------- Co-authored-by: Friedel Ziegelmayer Co-authored-by: Rüdiger Klaehn Co-authored-by: Floris Bruynooghe Co-authored-by: Asmir Avdicevic Co-authored-by: Diva M Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com> Co-authored-by: Vítor Vasconcellos --- .config/nextest.toml | 3 + .github/workflows/ci.yml | 4 +- .github/workflows/flaky.yaml | 1 + .github/workflows/netsim_runner.yaml | 6 +- .github/workflows/release.yml | 18 + .github/workflows/tests.yaml | 4 +- CHANGELOG.md | 79 +- Cargo.lock | 1605 ++++++---- Cargo.toml | 35 +- Makefile.toml | 4 +- deny.toml | 11 +- iroh-base/Cargo.toml | 6 +- iroh-base/src/lib.rs | 1 - iroh-base/src/rpc.rs | 34 - iroh-blobs/Cargo.toml | 89 - iroh-blobs/README.md | 45 - iroh-blobs/docs/img/get_machine.drawio | 238 -- iroh-blobs/docs/img/get_machine.drawio.svg | 3 - iroh-blobs/examples/connect/mod.rs | 95 - iroh-blobs/examples/fetch-fsm.rs | 162 - iroh-blobs/examples/fetch-stream.rs | 230 -- iroh-blobs/examples/provide-bytes.rs | 124 - .../protocol/range_spec.txt | 8 - iroh-blobs/proptest-regressions/provider.txt | 7 - iroh-blobs/src/downloader.rs | 1514 ---------- iroh-blobs/src/downloader/get.rs | 97 - iroh-blobs/src/downloader/invariants.rs | 163 - iroh-blobs/src/downloader/progress.rs | 194 -- iroh-blobs/src/downloader/test.rs | 531 ---- iroh-blobs/src/downloader/test/dialer.rs | 100 - iroh-blobs/src/downloader/test/getter.rs | 89 - iroh-blobs/src/export.rs | 136 - iroh-blobs/src/format.rs | 16 - iroh-blobs/src/format/collection.rs | 347 --- iroh-blobs/src/get.rs | 945 ------ iroh-blobs/src/get/db.rs | 699 ----- iroh-blobs/src/get/error.rs | 190 -- iroh-blobs/src/get/progress.rs | 185 -- iroh-blobs/src/get/request.rs | 202 -- iroh-blobs/src/hashseq.rs | 156 - iroh-blobs/src/lib.rs | 49 - iroh-blobs/src/metrics.rs | 65 - iroh-blobs/src/protocol.rs | 516 ---- iroh-blobs/src/protocol/range_spec.rs | 560 ---- iroh-blobs/src/provider.rs | 662 ----- iroh-blobs/src/store.rs | 96 - iroh-blobs/src/store/bao_file.rs | 1043 ------- iroh-blobs/src/store/fs.rs | 2612 ----------------- iroh-blobs/src/store/fs/migrate_redb_v1_v2.rs | 323 -- iroh-blobs/src/store/fs/tables.rs | 164 -- iroh-blobs/src/store/fs/test_support.rs | 401 --- iroh-blobs/src/store/fs/tests.rs | 811 ----- iroh-blobs/src/store/fs/util.rs | 79 - iroh-blobs/src/store/fs/validate.rs | 492 ---- iroh-blobs/src/store/mem.rs | 464 --- iroh-blobs/src/store/mutable_mem_storage.rs | 131 - iroh-blobs/src/store/readonly_mem.rs | 345 --- iroh-blobs/src/store/traits.rs | 1063 ------- iroh-blobs/src/util.rs | 365 --- iroh-blobs/src/util/io.rs | 102 - iroh-blobs/src/util/local_pool.rs | 685 ----- iroh-blobs/src/util/mem_or_file.rs | 104 - iroh-blobs/src/util/progress.rs | 687 ----- iroh-blobs/src/util/sparse_mem_file.rs | 122 - iroh-cli/Cargo.toml | 12 +- iroh-cli/src/commands/blobs.rs | 10 +- iroh-cli/src/commands/docs.rs | 57 +- iroh-cli/src/commands/doctor.rs | 5 +- iroh-cli/src/commands/gossip.rs | 6 +- iroh-cli/src/commands/net.rs | 3 +- iroh-cli/src/commands/start.rs | 2 +- iroh-cli/src/config.rs | 2 +- iroh-cli/tests/cli.rs | 19 +- iroh-dns-server/Cargo.toml | 8 +- iroh-dns-server/config.dev.toml | 2 + iroh-dns-server/config.prod.toml | 2 + iroh-dns-server/src/config.rs | 7 +- iroh-dns-server/src/http.rs | 16 +- iroh-dns-server/src/http/rate_limiting.rs | 63 +- iroh-dns-server/src/server.rs | 8 +- iroh-docs/Cargo.toml | 66 - iroh-docs/LICENSE-APACHE | 201 -- iroh-docs/LICENSE-MIT | 25 - iroh-docs/README.md | 48 - iroh-docs/proptest-regressions/ranger.txt | 9 - iroh-docs/src/actor.rs | 1040 ------- iroh-docs/src/engine.rs | 410 --- iroh-docs/src/engine/gossip.rs | 214 -- iroh-docs/src/engine/live.rs | 963 ------ iroh-docs/src/engine/state.rs | 260 -- iroh-docs/src/heads.rs | 157 - iroh-docs/src/keys.rs | 516 ---- iroh-docs/src/lib.rs | 60 - iroh-docs/src/metrics.rs | 85 - iroh-docs/src/net.rs | 368 --- iroh-docs/src/net/codec.rs | 693 ----- iroh-docs/src/ranger.rs | 1637 ----------- iroh-docs/src/store.rs | 426 --- iroh-docs/src/store/fs.rs | 1144 -------- iroh-docs/src/store/fs/bounds.rs | 295 -- iroh-docs/src/store/fs/migrate_v1_v2.rs | 144 - iroh-docs/src/store/fs/migrations.rs | 138 - iroh-docs/src/store/fs/query.rs | 159 - iroh-docs/src/store/fs/ranges.rs | 166 -- iroh-docs/src/store/fs/tables.rs | 186 -- iroh-docs/src/store/pubkeys.rs | 70 - iroh-docs/src/store/util.rs | 88 - iroh-docs/src/sync.rs | 2533 ---------------- iroh-docs/src/ticket.rs | 104 - iroh-gossip/Cargo.toml | 67 - iroh-gossip/README.md | 29 - iroh-gossip/examples/chat.rs | 325 -- iroh-gossip/src/lib.rs | 15 - iroh-gossip/src/metrics.rs | 69 - iroh-gossip/src/net.rs | 1008 ------- iroh-gossip/src/net/handles.rs | 276 -- iroh-gossip/src/net/util.rs | 128 - iroh-gossip/src/proto.rs | 376 --- iroh-gossip/src/proto/hyparview.rs | 718 ----- iroh-gossip/src/proto/plumtree.rs | 878 ------ iroh-gossip/src/proto/state.rs | 353 --- iroh-gossip/src/proto/tests.rs | 468 --- iroh-gossip/src/proto/topic.rs | 346 --- iroh-gossip/src/proto/util.rs | 470 --- iroh-metrics/Cargo.toml | 2 +- iroh-metrics/src/core.rs | 4 +- iroh-metrics/src/lib.rs | 30 + iroh-metrics/src/metrics.rs | 28 +- iroh-metrics/src/service.rs | 52 +- iroh-net/Cargo.toml | 60 +- iroh-net/bench/Cargo.toml | 2 +- iroh-net/bench/src/iroh.rs | 3 +- iroh-net/examples/connect-unreliable.rs | 6 +- iroh-net/examples/connect.rs | 6 +- iroh-net/examples/listen-unreliable.rs | 2 +- iroh-net/examples/listen.rs | 2 +- iroh-net/src/defaults.rs | 37 +- iroh-net/src/disco.rs | 15 +- iroh-net/src/discovery.rs | 16 +- iroh-net/src/discovery/dns.rs | 2 +- .../src/discovery/local_swarm_discovery.rs | 37 +- iroh-net/src/discovery/pkarr.rs | 2 +- iroh-net/src/discovery/pkarr/dht.rs | 34 +- iroh-net/src/discovery/static_provider.rs | 162 + iroh-net/src/endpoint.rs | 94 +- iroh-net/src/endpoint/rtt_actor.rs | 81 +- iroh-net/src/lib.rs | 18 +- iroh-net/src/magicsock.rs | 11 +- iroh-net/src/magicsock/metrics.rs | 17 + iroh-net/src/magicsock/node_map.rs | 10 +- iroh-net/src/magicsock/node_map/node_state.rs | 24 +- iroh-net/src/magicsock/node_map/path_state.rs | 3 +- iroh-net/src/magicsock/relay_actor.rs | 18 +- iroh-net/src/magicsock/udp_conn.rs | 6 +- iroh-net/src/metrics.rs | 13 +- iroh-net/src/netcheck.rs | 154 +- iroh-net/src/netcheck/reportgen.rs | 159 +- iroh-net/src/netcheck/reportgen/hairpin.rs | 4 +- iroh-net/src/netcheck/reportgen/probes.rs | 7 +- iroh-net/src/relay.rs | 40 - iroh-net/src/{relay/map.rs => relay_map.rs} | 2 +- iroh-net/src/stun.rs | 545 ---- iroh-net/src/test_utils.rs | 38 +- iroh-net/src/util.rs | 2 - iroh-relay/Cargo.toml | 103 + iroh-relay/README.md | 43 + .../src/relay => iroh-relay/src}/client.rs | 180 +- .../relay => iroh-relay/src}/client/conn.rs | 18 +- .../src}/client/streams.rs | 26 +- .../chain.rs => iroh-relay/src/client/util.rs | 0 iroh-relay/src/defaults.rs | 42 + iroh-relay/src/dns.rs | 64 + .../src/relay => iroh-relay/src}/http.rs | 15 +- iroh-relay/src/lib.rs | 42 + .../iroh-relay.rs => iroh-relay/src/main.rs | 50 +- iroh-relay/src/protos.rs | 5 + iroh-relay/src/protos/disco.rs | 20 + .../src/protos/relay.rs | 45 +- iroh-relay/src/protos/stun.rs | 388 +++ .../src/relay => iroh-relay/src}/server.rs | 125 +- .../relay => iroh-relay/src}/server/actor.rs | 127 +- .../src}/server/client_conn.rs | 36 +- .../src}/server/clients.rs | 11 +- .../src}/server/http_server.rs | 186 +- .../src}/server/metrics.rs | 0 .../src}/server/streams.rs | 2 +- iroh-relay/src/server/testing.rs | 75 + .../relay => iroh-relay/src}/server/types.rs | 3 +- iroh-router/Cargo.toml | 37 + iroh-router/README.md | 20 + iroh-router/examples/custom-protocol.rs | 232 ++ iroh-router/src/lib.rs | 5 + iroh-router/src/protocol.rs | 77 + iroh-router/src/router.rs | 211 ++ iroh-test/Cargo.toml | 2 +- iroh-willow/Cargo.toml | 16 +- iroh-willow/examples/bench.rs | 11 +- iroh-willow/src/engine.rs | 4 +- iroh-willow/src/engine/peer_manager.rs | 6 +- iroh-willow/src/interest.rs | 2 +- iroh-willow/src/lib.rs | 1 + iroh-willow/src/net.rs | 5 +- iroh-willow/src/proto/data_model.rs | 13 +- iroh-willow/src/proto/grouping.rs | 3 +- iroh-willow/src/proto/keys.rs | 11 +- iroh-willow/src/proto/meadowcap.rs | 7 +- iroh-willow/src/proto/pai.rs | 4 +- iroh-willow/src/proto/wgps/messages.rs | 18 +- iroh-willow/src/rpc.rs | 6 + .../src/rpc/client.rs | 152 +- .../src/rpc/handler.rs | 104 +- .../spaces.rs => iroh-willow/src/rpc/proto.rs | 52 +- iroh-willow/src/session.rs | 7 +- iroh-willow/src/session/channels.rs | 3 +- iroh-willow/src/session/data.rs | 9 +- iroh-willow/src/session/intents.rs | 10 +- iroh-willow/src/session/pai_finder.rs | 5 +- iroh-willow/src/session/payload.rs | 3 +- iroh-willow/src/session/resource.rs | 3 +- iroh-willow/src/session/run.rs | 13 +- iroh-willow/src/store.rs | 13 +- iroh-willow/src/store/memory.rs | 25 +- iroh-willow/src/store/persistent.rs | 20 +- iroh-willow/tests/basic.rs | 12 +- {iroh => iroh-willow}/tests/spaces.rs | 250 +- iroh/Cargo.toml | 28 +- iroh/examples/client.rs | 4 +- iroh/examples/collection-provide.rs | 13 +- iroh/examples/custom-protocol.rs | 4 +- iroh/examples/hello-world-provide.rs | 9 +- iroh/examples/local-swarm-discovery.rs | 2 +- iroh/src/client.rs | 46 +- iroh/src/client/authors.rs | 134 - iroh/src/client/blobs.rs | 1688 ----------- iroh/src/client/blobs/batch.rs | 462 --- iroh/src/client/docs.rs | 879 ------ iroh/src/client/gossip.rs | 111 - iroh/src/client/net.rs | 22 +- iroh/src/client/quic.rs | 11 +- iroh/src/client/tags.rs | 60 - iroh/src/lib.rs | 8 +- iroh/src/node.rs | 242 +- iroh/src/node/builder.rs | 201 +- iroh/src/node/docs.rs | 63 - iroh/src/node/protocol.rs | 359 --- iroh/src/node/rpc.rs | 1264 +------- iroh/src/node/rpc/docs.rs | 310 -- iroh/src/rpc_protocol.rs | 24 +- iroh/src/rpc_protocol/authors.rs | 123 - iroh/src/rpc_protocol/blobs.rs | 345 --- iroh/src/rpc_protocol/docs.rs | 425 --- iroh/src/rpc_protocol/gossip.rs | 41 - iroh/src/rpc_protocol/net.rs | 5 +- iroh/src/rpc_protocol/node.rs | 3 +- iroh/src/rpc_protocol/tags.rs | 110 - iroh/src/util/fs.rs | 111 - iroh/src/util/path.rs | 3 - iroh/tests/client.rs | 2 +- iroh/tests/gc.rs | 43 +- iroh/tests/provide.rs | 18 +- iroh/tests/spaces.proptest-regressions | 30 - iroh/tests/sync.rs | 1375 --------- net-tools/netwatch/Cargo.toml | 48 + net-tools/netwatch/README.md | 24 + .../netwatch/src}/interfaces.rs | 40 +- .../netwatch/src}/interfaces/bsd.rs | 0 .../netwatch/src}/interfaces/bsd/freebsd.rs | 0 .../netwatch/src}/interfaces/bsd/macos.rs | 0 .../netwatch/src}/interfaces/bsd/netbsd.rs | 0 .../netwatch/src}/interfaces/bsd/openbsd.rs | 0 .../netwatch/src}/interfaces/linux.rs | 0 .../netwatch/src}/interfaces/windows.rs | 0 .../src/net => net-tools/netwatch/src}/ip.rs | 0 .../netwatch/src}/ip_family.rs | 0 .../net.rs => net-tools/netwatch/src/lib.rs | 2 +- .../net => net-tools/netwatch/src}/netmon.rs | 21 +- .../netwatch/src}/netmon/actor.rs | 57 +- .../netwatch/src}/netmon/android.rs | 0 .../netwatch/src}/netmon/bsd.rs | 43 +- .../netwatch/src}/netmon/linux.rs | 2 +- .../netwatch/src}/netmon/windows.rs | 0 .../src/net => net-tools/netwatch/src}/udp.rs | 0 net-tools/portmapper/Cargo.toml | 49 + net-tools/portmapper/README.md | 24 + .../portmapper/src}/current_mapping.rs | 0 .../portmapper/src/lib.rs | 16 +- .../portmapper/src}/mapping.rs | 0 .../portmapper/src}/metrics.rs | 0 .../portmapper/src}/nat_pmp.rs | 3 +- .../portmapper/src}/nat_pmp/protocol.rs | 0 .../src}/nat_pmp/protocol/request.rs | 0 .../src}/nat_pmp/protocol/response.rs | 0 .../portmapper/src}/pcp.rs | 3 +- .../portmapper/src}/pcp/protocol.rs | 0 .../src}/pcp/protocol/opcode_data.rs | 0 .../portmapper/src}/pcp/protocol/request.rs | 1 - .../portmapper/src}/pcp/protocol/response.rs | 1 - .../portmapper/src}/upnp.rs | 41 +- net-tools/portmapper/src/util.rs | 30 + 299 files changed, 4922 insertions(+), 47057 deletions(-) delete mode 100644 iroh-base/src/rpc.rs delete mode 100644 iroh-blobs/Cargo.toml delete mode 100644 iroh-blobs/README.md delete mode 100644 iroh-blobs/docs/img/get_machine.drawio delete mode 100644 iroh-blobs/docs/img/get_machine.drawio.svg delete mode 100644 iroh-blobs/examples/connect/mod.rs delete mode 100644 iroh-blobs/examples/fetch-fsm.rs delete mode 100644 iroh-blobs/examples/fetch-stream.rs delete mode 100644 iroh-blobs/examples/provide-bytes.rs delete mode 100644 iroh-blobs/proptest-regressions/protocol/range_spec.txt delete mode 100644 iroh-blobs/proptest-regressions/provider.txt delete mode 100644 iroh-blobs/src/downloader.rs delete mode 100644 iroh-blobs/src/downloader/get.rs delete mode 100644 iroh-blobs/src/downloader/invariants.rs delete mode 100644 iroh-blobs/src/downloader/progress.rs delete mode 100644 iroh-blobs/src/downloader/test.rs delete mode 100644 iroh-blobs/src/downloader/test/dialer.rs delete mode 100644 iroh-blobs/src/downloader/test/getter.rs delete mode 100644 iroh-blobs/src/export.rs delete mode 100644 iroh-blobs/src/format.rs delete mode 100644 iroh-blobs/src/format/collection.rs delete mode 100644 iroh-blobs/src/get.rs delete mode 100644 iroh-blobs/src/get/db.rs delete mode 100644 iroh-blobs/src/get/error.rs delete mode 100644 iroh-blobs/src/get/progress.rs delete mode 100644 iroh-blobs/src/get/request.rs delete mode 100644 iroh-blobs/src/hashseq.rs delete mode 100644 iroh-blobs/src/lib.rs delete mode 100644 iroh-blobs/src/metrics.rs delete mode 100644 iroh-blobs/src/protocol.rs delete mode 100644 iroh-blobs/src/protocol/range_spec.rs delete mode 100644 iroh-blobs/src/provider.rs delete mode 100644 iroh-blobs/src/store.rs delete mode 100644 iroh-blobs/src/store/bao_file.rs delete mode 100644 iroh-blobs/src/store/fs.rs delete mode 100644 iroh-blobs/src/store/fs/migrate_redb_v1_v2.rs delete mode 100644 iroh-blobs/src/store/fs/tables.rs delete mode 100644 iroh-blobs/src/store/fs/test_support.rs delete mode 100644 iroh-blobs/src/store/fs/tests.rs delete mode 100644 iroh-blobs/src/store/fs/util.rs delete mode 100644 iroh-blobs/src/store/fs/validate.rs delete mode 100644 iroh-blobs/src/store/mem.rs delete mode 100644 iroh-blobs/src/store/mutable_mem_storage.rs delete mode 100644 iroh-blobs/src/store/readonly_mem.rs delete mode 100644 iroh-blobs/src/store/traits.rs delete mode 100644 iroh-blobs/src/util.rs delete mode 100644 iroh-blobs/src/util/io.rs delete mode 100644 iroh-blobs/src/util/local_pool.rs delete mode 100644 iroh-blobs/src/util/mem_or_file.rs delete mode 100644 iroh-blobs/src/util/progress.rs delete mode 100644 iroh-blobs/src/util/sparse_mem_file.rs delete mode 100644 iroh-docs/Cargo.toml delete mode 100644 iroh-docs/LICENSE-APACHE delete mode 100644 iroh-docs/LICENSE-MIT delete mode 100644 iroh-docs/README.md delete mode 100644 iroh-docs/proptest-regressions/ranger.txt delete mode 100644 iroh-docs/src/actor.rs delete mode 100644 iroh-docs/src/engine.rs delete mode 100644 iroh-docs/src/engine/gossip.rs delete mode 100644 iroh-docs/src/engine/live.rs delete mode 100644 iroh-docs/src/engine/state.rs delete mode 100644 iroh-docs/src/heads.rs delete mode 100644 iroh-docs/src/keys.rs delete mode 100644 iroh-docs/src/lib.rs delete mode 100644 iroh-docs/src/metrics.rs delete mode 100644 iroh-docs/src/net.rs delete mode 100644 iroh-docs/src/net/codec.rs delete mode 100644 iroh-docs/src/ranger.rs delete mode 100644 iroh-docs/src/store.rs delete mode 100644 iroh-docs/src/store/fs.rs delete mode 100644 iroh-docs/src/store/fs/bounds.rs delete mode 100644 iroh-docs/src/store/fs/migrate_v1_v2.rs delete mode 100644 iroh-docs/src/store/fs/migrations.rs delete mode 100644 iroh-docs/src/store/fs/query.rs delete mode 100644 iroh-docs/src/store/fs/ranges.rs delete mode 100644 iroh-docs/src/store/fs/tables.rs delete mode 100644 iroh-docs/src/store/pubkeys.rs delete mode 100644 iroh-docs/src/store/util.rs delete mode 100644 iroh-docs/src/sync.rs delete mode 100644 iroh-docs/src/ticket.rs delete mode 100644 iroh-gossip/Cargo.toml delete mode 100644 iroh-gossip/README.md delete mode 100644 iroh-gossip/examples/chat.rs delete mode 100644 iroh-gossip/src/lib.rs delete mode 100644 iroh-gossip/src/metrics.rs delete mode 100644 iroh-gossip/src/net.rs delete mode 100644 iroh-gossip/src/net/handles.rs delete mode 100644 iroh-gossip/src/net/util.rs delete mode 100644 iroh-gossip/src/proto.rs delete mode 100644 iroh-gossip/src/proto/hyparview.rs delete mode 100644 iroh-gossip/src/proto/plumtree.rs delete mode 100644 iroh-gossip/src/proto/state.rs delete mode 100644 iroh-gossip/src/proto/tests.rs delete mode 100644 iroh-gossip/src/proto/topic.rs delete mode 100644 iroh-gossip/src/proto/util.rs create mode 100644 iroh-net/src/discovery/static_provider.rs delete mode 100644 iroh-net/src/relay.rs rename iroh-net/src/{relay/map.rs => relay_map.rs} (99%) delete mode 100644 iroh-net/src/stun.rs create mode 100644 iroh-relay/Cargo.toml create mode 100644 iroh-relay/README.md rename {iroh-net/src/relay => iroh-relay/src}/client.rs (87%) rename {iroh-net/src/relay => iroh-relay/src}/client/conn.rs (97%) rename {iroh-net/src/relay => iroh-relay/src}/client/streams.rs (92%) rename iroh-net/src/util/chain.rs => iroh-relay/src/client/util.rs (100%) create mode 100644 iroh-relay/src/defaults.rs create mode 100644 iroh-relay/src/dns.rs rename {iroh-net/src/relay => iroh-relay/src}/http.rs (72%) create mode 100644 iroh-relay/src/lib.rs rename iroh-net/src/bin/iroh-relay.rs => iroh-relay/src/main.rs (91%) create mode 100644 iroh-relay/src/protos.rs create mode 100644 iroh-relay/src/protos/disco.rs rename iroh-net/src/relay/codec.rs => iroh-relay/src/protos/relay.rs (96%) create mode 100644 iroh-relay/src/protos/stun.rs rename {iroh-net/src/relay => iroh-relay/src}/server.rs (92%) rename {iroh-net/src/relay => iroh-relay/src}/server/actor.rs (85%) rename {iroh-net/src/relay => iroh-relay/src}/server/client_conn.rs (97%) rename {iroh-net/src/relay => iroh-relay/src}/server/clients.rs (98%) rename {iroh-net/src/relay => iroh-relay/src}/server/http_server.rs (82%) rename {iroh-net/src/relay => iroh-relay/src}/server/metrics.rs (100%) rename {iroh-net/src/relay => iroh-relay/src}/server/streams.rs (99%) create mode 100644 iroh-relay/src/server/testing.rs rename {iroh-net/src/relay => iroh-relay/src}/server/types.rs (87%) create mode 100644 iroh-router/Cargo.toml create mode 100644 iroh-router/README.md create mode 100644 iroh-router/examples/custom-protocol.rs create mode 100644 iroh-router/src/lib.rs create mode 100644 iroh-router/src/protocol.rs create mode 100644 iroh-router/src/router.rs create mode 100644 iroh-willow/src/rpc.rs rename iroh/src/client/spaces.rs => iroh-willow/src/rpc/client.rs (86%) rename iroh/src/node/rpc/spaces.rs => iroh-willow/src/rpc/handler.rs (71%) rename iroh/src/rpc_protocol/spaces.rs => iroh-willow/src/rpc/proto.rs (87%) rename {iroh => iroh-willow}/tests/spaces.rs (59%) delete mode 100644 iroh/src/client/authors.rs delete mode 100644 iroh/src/client/blobs.rs delete mode 100644 iroh/src/client/blobs/batch.rs delete mode 100644 iroh/src/client/docs.rs delete mode 100644 iroh/src/client/gossip.rs delete mode 100644 iroh/src/client/tags.rs delete mode 100644 iroh/src/node/docs.rs delete mode 100644 iroh/src/node/protocol.rs delete mode 100644 iroh/src/node/rpc/docs.rs delete mode 100644 iroh/src/rpc_protocol/authors.rs delete mode 100644 iroh/src/rpc_protocol/blobs.rs delete mode 100644 iroh/src/rpc_protocol/docs.rs delete mode 100644 iroh/src/rpc_protocol/gossip.rs delete mode 100644 iroh/src/rpc_protocol/tags.rs delete mode 100644 iroh/tests/spaces.proptest-regressions delete mode 100644 iroh/tests/sync.rs create mode 100644 net-tools/netwatch/Cargo.toml create mode 100644 net-tools/netwatch/README.md rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces.rs (90%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/bsd.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/bsd/freebsd.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/bsd/macos.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/bsd/netbsd.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/bsd/openbsd.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/linux.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/interfaces/windows.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/ip.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/ip_family.rs (100%) rename iroh-net/src/net.rs => net-tools/netwatch/src/lib.rs (83%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon.rs (89%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon/actor.rs (81%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon/android.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon/bsd.rs (68%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon/linux.rs (99%) rename {iroh-net/src/net => net-tools/netwatch/src}/netmon/windows.rs (100%) rename {iroh-net/src/net => net-tools/netwatch/src}/udp.rs (100%) create mode 100644 net-tools/portmapper/Cargo.toml create mode 100644 net-tools/portmapper/README.md rename {iroh-net/src/portmapper => net-tools/portmapper/src}/current_mapping.rs (100%) rename iroh-net/src/portmapper.rs => net-tools/portmapper/src/lib.rs (98%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/mapping.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/metrics.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/nat_pmp.rs (98%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/nat_pmp/protocol.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/nat_pmp/protocol/request.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/nat_pmp/protocol/response.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/pcp.rs (98%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/pcp/protocol.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/pcp/protocol/opcode_data.rs (100%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/pcp/protocol/request.rs (99%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/pcp/protocol/response.rs (99%) rename {iroh-net/src/portmapper => net-tools/portmapper/src}/upnp.rs (80%) create mode 100644 net-tools/portmapper/src/util.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 4f0a061fb04..bd35988a09a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -8,3 +8,6 @@ run-in-isolation = { max-threads = 32 } filter = 'test(::run_in_isolation::)' test-group = 'run-in-isolation' threads-required = 32 + +[profile.default] +slow-timeout = { period = "20s", terminate-after = 3 } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b5290d62e4..d4971c5d493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,7 @@ jobs: # uses: obi1kenobi/cargo-semver-checks-action@v2 uses: n0-computer/cargo-semver-checks-action@feat-baseline with: - package: iroh, iroh-base, iroh-blobs, iroh-cli, iroh-dns-server, iroh-gossip, iroh-metrics, iroh-net, iroh-net-bench, iroh-docs + package: iroh, iroh-base, iroh-cli, iroh-dns-server, iroh-metrics, iroh-net, iroh-net-bench, iroh-router, netwatch, portmapper, iroh-relay baseline-rev: ${{ env.HEAD_COMMIT_SHA }} use-cache: false @@ -298,7 +298,7 @@ jobs: netsim_branch: "main" sim_paths: "sims/iroh/iroh.json,sims/integration" pr_number: ${{ github.event.pull_request.number || '' }} - + docker_build_and_test: name: Docker Test if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" diff --git a/.github/workflows/flaky.yaml b/.github/workflows/flaky.yaml index 7b2ecccc6f4..094f6b39a9f 100644 --- a/.github/workflows/flaky.yaml +++ b/.github/workflows/flaky.yaml @@ -93,6 +93,7 @@ jobs: uses: n0-computer/discord-webhook-notify@v1 if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }} with: + text: "Flaky tests in **${{ github.repository }}**:" severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }} details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }} webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} diff --git a/.github/workflows/netsim_runner.yaml b/.github/workflows/netsim_runner.yaml index 38e885b35d8..ae7b20d08c1 100644 --- a/.github/workflows/netsim_runner.yaml +++ b/.github/workflows/netsim_runner.yaml @@ -198,8 +198,10 @@ jobs: overwrite: true - name: Fail Job if Tests Failed - if: ${{ failure() && steps.run_tests.outcome == 'failure' }} - run: exit 1 + if: ${{ steps.run_tests.outcome == 'failure' }} + run: | + echo "Tests failed logs are available at: ${{steps.upload-report.outputs.artifact-url}}" + exit 1 - name: Find Docs Comment if: ${{ inputs.pr_number != '' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15aa17c2790..547decfecea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -280,6 +280,24 @@ jobs: upload_url: ${{ github.event.inputs.upload_url || needs.create-release.outputs.upload_url }} asset_path: ${{ env.ASSET }} + - name: attach artifacts + uses: actions/upload-artifact@v4 + if : matrix.os != 'windows-latest' + with: + name: iroh-${{ matrix.release-os }}-${{ matrix.release-arch }}-bundle + path: iroh-*.tar.gz + compression-level: 0 + retention-days: 1 + + - name: attach artifacts + uses: actions/upload-artifact@v4 + if : matrix.os == 'windows-latest' + with: + name: iroh-${{ matrix.release-os }}-${{ matrix.release-arch }}-bundle + path: iroh-*.zip + compression-level: 0 + retention-days: 1 + docker: needs: build_release uses: './.github/workflows/docker.yaml' diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7d9700bfc24..5640af42c9d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ env: RUSTFLAGS: -Dwarnings RUSTDOCFLAGS: -Dwarnings SCCACHE_CACHE_SIZE: "50G" - CRATES_LIST: "iroh,iroh-blobs,iroh-gossip,iroh-metrics,iroh-net,iroh-net-bench,iroh-docs,iroh-test,iroh-cli,iroh-dns-server" + CRATES_LIST: "iroh,iroh-metrics,iroh-net,iroh-net-bench,iroh-test,iroh-cli,iroh-dns-server,iroh-router,netwatch,portmapper,iroh-relay" IROH_FORCE_STAGING_RELAYS: "1" jobs: @@ -218,7 +218,7 @@ jobs: env: RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} NEXTEST_EXPERIMENTAL_LIBTEST_JSON: 1 - + - name: upload results if: ${{ failure() && inputs.flaky }} uses: actions/upload-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index da4f5ada637..39451f196e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,81 @@ All notable changes to iroh will be documented in this file. -## [0.27.0](https://github.com/n0-computer/iroh/compare/v0.26.0..0.27.0) - 2024-10-21 +## [0.28.1](https://github.com/n0-computer/iroh/compare/v0.28.0..v0.28.1) - 2024-11-04 + +### 🐛 Bug Fixes + +- Switch to correctly patched quic-rpc and iroh-quinn - ([d925da4](https://github.com/n0-computer/iroh/commit/d925da442993fb79d55b905d4c17a324e9549bd2)) + +### 📚 Documentation + +- Fixup changelog - ([5066102](https://github.com/n0-computer/iroh/commit/50661022258e607775af6e6b83c4c25fc57ed088)) + +### ⚙️ Miscellaneous Tasks + +- Release - ([134a93b](https://github.com/n0-computer/iroh/commit/134a93b5a60103b3ce8fa4aacb52cdbcb291d00b)) + + +## [0.28.0](https://github.com/n0-computer/iroh/compare/v0.27.0..v0.28.0) - 2024-11-04 + +### ⛰️ Features + +- *(iroh-dns-server)* [**breaking**] Make http rate limit configurable ([#2772](https://github.com/n0-computer/iroh/issues/2772)) - ([fe684c2](https://github.com/n0-computer/iroh/commit/fe684c23e60fc8c35d3ec02bf088f36e1e248c50)) +- *(iroh-net)* Add StaticDiscovery to provide static info to endpoints ([#2825](https://github.com/n0-computer/iroh/issues/2825)) - ([c9d1ba7](https://github.com/n0-computer/iroh/commit/c9d1ba7c09948fd24fee6f1aff4d772049d5a86a)) +- *(iroh-net)* More Quinn re-exports ([#2838](https://github.com/n0-computer/iroh/issues/2838)) - ([9495c21](https://github.com/n0-computer/iroh/commit/9495c21a4f5ee93e1f5f7312b37c0752327100bd)) +- *(iroh-net)* Send HTTP/1.1 `HOST` header on requests to relay ([#2881](https://github.com/n0-computer/iroh/issues/2881)) - ([4bfa58e](https://github.com/n0-computer/iroh/commit/4bfa58eaed49595602c64ac07155bf36c1469ffd)) +- [**breaking**] Introduce iroh-router crate ([#2832](https://github.com/n0-computer/iroh/issues/2832)) - ([8f75005](https://github.com/n0-computer/iroh/commit/8f7500545ac71a151b0a8ac4389e558b64e58a4c)) +- Collect metrics for direct connections & add opt-in push metrics ([#2805](https://github.com/n0-computer/iroh/issues/2805)) - ([86b494a](https://github.com/n0-computer/iroh/commit/86b494a9088e1558150d70481051227845c827e1)) + +### 🐛 Bug Fixes + +- *(ci)* Better error reporting on netsim fails ([#2886](https://github.com/n0-computer/iroh/issues/2886)) - ([e1aab51](https://github.com/n0-computer/iroh/commit/e1aab5188c55c571fd2024b98e34b144143bd2be)) +- *(iroh-net)* When switching to a direct path reset the mtu ([#2835](https://github.com/n0-computer/iroh/issues/2835)) - ([93f7900](https://github.com/n0-computer/iroh/commit/93f79009bdc1b9cb8e8db41e143a1df14fdb7f25)) +- *(iroh-relay)* Respect `enable_stun` setting in `iroh-relay::Config` ([#2879](https://github.com/n0-computer/iroh/issues/2879)) - ([2507e62](https://github.com/n0-computer/iroh/commit/2507e625c0fd05924a72af1e21696ba4ff7e4dc7)) +- *(metrics)* Allow external crates to encode their metrics ([#2885](https://github.com/n0-computer/iroh/issues/2885)) - ([362076e](https://github.com/n0-computer/iroh/commit/362076ee742c810ea8ccb28f415fd90e0b8171c3)) +- *(portmapper)* Enforce timeouts for upnp ([#2877](https://github.com/n0-computer/iroh/issues/2877)) - ([00a3f88](https://github.com/n0-computer/iroh/commit/00a3f88cbb2a93dc15144da91674af9cb95bb06f)) + +### 🚜 Refactor + +- *(iroh)* Move protocol relevant impls into node/protocols ([#2831](https://github.com/n0-computer/iroh/issues/2831)) - ([67df1c1](https://github.com/n0-computer/iroh/commit/67df1c148f9eee8008e0288dbc2c8829be05c891)) +- *(iroh)* Move ProtocolHandler impl to iroh-gossip ([#2849](https://github.com/n0-computer/iroh/issues/2849)) - ([6c6827d](https://github.com/n0-computer/iroh/commit/6c6827d63ec12d9c9583b73b5530a7641060535c)) +- *(iroh)* Move blobs protocol to iroh-blobs ([#2853](https://github.com/n0-computer/iroh/issues/2853)) - ([30f3e03](https://github.com/n0-computer/iroh/commit/30f3e03cde8a58af3b84a5f11134fb970ec3efa1)) +- *(iroh)* [**breaking**] Remove gossip rpc types ([#2834](https://github.com/n0-computer/iroh/issues/2834)) - ([a55529b](https://github.com/n0-computer/iroh/commit/a55529b52e198527590493e67e7706290c9656f0)) +- *(iroh-net)* Portmapper and network monitor are crates ([#2855](https://github.com/n0-computer/iroh/issues/2855)) - ([fad3e24](https://github.com/n0-computer/iroh/commit/fad3e24b3f2698ce6c1fa3fdad54201bec668298)) +- Move iroh-gossip to external repo ([#2826](https://github.com/n0-computer/iroh/issues/2826)) - ([e659405](https://github.com/n0-computer/iroh/commit/e659405241692feb94030a8145e0b66a1a248641)) +- Move iroh-docs to external repo ([#2830](https://github.com/n0-computer/iroh/issues/2830)) - ([3e17210](https://github.com/n0-computer/iroh/commit/3e17210c1e2ff7b8788feb3534e57e8c3af3cd4b)) +- Remove iroh-blobs and use crates.io dependency ([#2829](https://github.com/n0-computer/iroh/issues/2829)) - ([d29537d](https://github.com/n0-computer/iroh/commit/d29537da6fc07ff82b8d56ff5fae9fdef9445858)) +- [**breaking**] Remove iroh_base::rpc ([#2840](https://github.com/n0-computer/iroh/issues/2840)) - ([bfba7a4](https://github.com/n0-computer/iroh/commit/bfba7a42284a5b1b9065186c558bc614dba351f2)) +- Move ProtocolHandler docs to iroh-docs ([#2859](https://github.com/n0-computer/iroh/issues/2859)) - ([61acd96](https://github.com/n0-computer/iroh/commit/61acd9688af60e55c62e357eb69272ea24097ffc)) + +### 📚 Documentation + +- *(iroh-net)* Link to Endpoint in the first few paragraphs ([#2875](https://github.com/n0-computer/iroh/issues/2875)) - ([f0590be](https://github.com/n0-computer/iroh/commit/f0590be408c8dbe412897525a97a170e694dd650)) + +### 🧪 Testing + +- *(iroh-net)* Give this a longer timeout ([#2857](https://github.com/n0-computer/iroh/issues/2857)) - ([ed13453](https://github.com/n0-computer/iroh/commit/ed13453697fbe600fdb50afb374543b69125bbc9)) +- *(iroh-net)* Make dht_discovery_smoke test less flaky ([#2884](https://github.com/n0-computer/iroh/issues/2884)) - ([ce8d94d](https://github.com/n0-computer/iroh/commit/ce8d94de083d7aa997cd79936cf1606131420e6e)) +- *(netwatch)* Simplify dev-deps - ([029830f](https://github.com/n0-computer/iroh/commit/029830fd75be4690a840185973ed3210692a167c)) + +### ⚙️ Miscellaneous Tasks + +- *(ci)* Identify which repository the flakes are reported for ([#2824](https://github.com/n0-computer/iroh/issues/2824)) - ([b2e587d](https://github.com/n0-computer/iroh/commit/b2e587d7c96e84b2c2df75e33fea80ef3bd97450)) +- *(iroh-net)* Fixup portmapper version - ([37f620d](https://github.com/n0-computer/iroh/commit/37f620dffa929a427374e1508737f68ee1e8f543)) +- Add iroh-router to crates list ([#2850](https://github.com/n0-computer/iroh/issues/2850)) - ([2d17636](https://github.com/n0-computer/iroh/commit/2d17636d32840f09bb6f86ab91a49d9ecea07bd9)) +- Release - ([860b90f](https://github.com/n0-computer/iroh/commit/860b90f1bad660a470d30f1e81ee0d6984de6106)) +- Release - ([8bae5c3](https://github.com/n0-computer/iroh/commit/8bae5c3ec4465e7d6369440d1c55f7de7ca0e770)) +- Release - ([d6c39c9](https://github.com/n0-computer/iroh/commit/d6c39c974f1383603e06b742c9394969b644c0f7)) +- Release - ([2073bf4](https://github.com/n0-computer/iroh/commit/2073bf40176789c36a607d10a6bbaef38b16846b)) +- Upgrade 0.28 iroh-net - ([13da047](https://github.com/n0-computer/iroh/commit/13da0478b89202904a8a67c9e0a1ff4ad15882b7)) +- Release - ([5751521](https://github.com/n0-computer/iroh/commit/5751521cf50434c588e387ce483daf407919571b)) +- Release - ([5437dbb](https://github.com/n0-computer/iroh/commit/5437dbb4e409200f66c3d97d9b277be14a2b6b33)) +- Upgrade 0.28 iroh-router - ([297b874](https://github.com/n0-computer/iroh/commit/297b8743296d6103d8b0457b431597f4d6168c7d)) +- Update 0.28 iroh-docs, iroh-gossip, iroh-blobs - ([7e80a92](https://github.com/n0-computer/iroh/commit/7e80a9221eb61d04a02830ed1c1794503f8113ff)) +- Release - ([fa926be](https://github.com/n0-computer/iroh/commit/fa926beef29260b143f941f07dc00f1f77b4ffc5)) +- Release - ([4c58bd8](https://github.com/n0-computer/iroh/commit/4c58bd8db5c90567ec3ffae9f19474887d037445)) + +## [0.27.0](https://github.com/n0-computer/iroh/compare/v0.26.0..v0.27.0) - 2024-10-21 ### ⛰️ Features @@ -57,6 +131,7 @@ All notable changes to iroh will be documented in this file. - *(iroh-net)* Upgrade igd-next, remove hyper 0.14 ([#2804](https://github.com/n0-computer/iroh/issues/2804)) - ([5e40fe1](https://github.com/n0-computer/iroh/commit/5e40fe138f9581a195d47c251992e3de8b1ec8c1)) - Format imports using rustfmt ([#2812](https://github.com/n0-computer/iroh/issues/2812)) - ([8808a36](https://github.com/n0-computer/iroh/commit/8808a360c9f8299984a7e5a739fa9377eeffe73a)) - Increase version numbers and update ([#2821](https://github.com/n0-computer/iroh/issues/2821)) - ([71b5903](https://github.com/n0-computer/iroh/commit/71b5903e2840daafcfb972df3e481b152bbbe990)) +- Release - ([3f5b778](https://github.com/n0-computer/iroh/commit/3f5b778b379529f9f11deeafaf1f612b533b5c94)) ### Deps @@ -1786,5 +1861,3 @@ All notable changes to iroh will be documented in this file. - On_collection doesn't need to be FnMut ([#136](https://github.com/n0-computer/iroh/issues/136)) - ([eac7b65](https://github.com/n0-computer/iroh/commit/eac7b65a6760c0cf55d455ca5a7e9e523698c7a1)) - Allow older rust version ([#142](https://github.com/n0-computer/iroh/issues/142)) - ([f3086a9](https://github.com/n0-computer/iroh/commit/f3086a9576fdc0cdfbd6b0646745bec9e91f7d60)) - Use our own bao crate - ([659d2d2](https://github.com/n0-computer/iroh/commit/659d2d22254ea1d3f185ec0d4c8be4e7bf4374df)) - - diff --git a/Cargo.lock b/Cargo.lock index 72fba41a2bc..ecf5dd415d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,18 +18,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -92,9 +92,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -107,43 +107,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "arc-swap" @@ -153,21 +153,21 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -181,13 +181,13 @@ dependencies = [ [[package]] name = "asn1-rs-derive" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "synstructure", ] @@ -199,7 +199,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -222,18 +222,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -264,15 +264,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -280,7 +280,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-util", @@ -297,7 +297,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -305,20 +305,20 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -326,14 +326,13 @@ dependencies = [ [[package]] name = "axum-macros" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ - "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -346,7 +345,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-util", @@ -356,7 +355,7 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tower", + "tower 0.4.13", "tower-service", ] @@ -373,17 +372,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -393,7 +392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f7a89a8ee5889d2593ae422ce6e1bb03e48a0e8a16e4fa0882dfcbe7e182ef" dependencies = [ "bytes", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "genawaiter", "iroh-blake3", "iroh-io", @@ -478,23 +477,11 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "blake3" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", @@ -532,18 +519,18 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" dependencies = [ "serde", ] [[package]] name = "camino" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -593,9 +580,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +dependencies = [ + "shlex", +] [[package]] name = "cesu8" @@ -609,6 +599,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -675,9 +671,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.9" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -685,9 +681,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -697,21 +693,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.8" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "clipboard-win" @@ -732,9 +728,9 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" @@ -811,9 +807,19 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "cordyceps" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec10f0a762d93c4498d2e97a333805cb6250d60bead623f71d8034f9a4152ba3" +dependencies = [ + "loom", + "tracing", +] [[package]] name = "core-foundation" @@ -827,15 +833,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -940,7 +946,7 @@ dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "libc", - "mio", + "mio 0.8.11", "parking_lot", "signal-hook", "signal-hook-mio", @@ -1043,7 +1049,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1067,7 +1073,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1078,7 +1084,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1133,7 +1139,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1163,7 +1169,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "unicode-xid", ] @@ -1180,9 +1186,9 @@ dependencies = [ [[package]] name = "diatomic-waker" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af873b6853650fb206431c52fa7bbf6917146b70a8a9979d6d141f5d5394086b" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" [[package]] name = "diff" @@ -1252,7 +1258,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1402,14 +1408,14 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1422,7 +1428,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1442,7 +1448,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1535,9 +1541,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fd-lock" @@ -1566,11 +1572,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -1584,6 +1596,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1603,17 +1621,11 @@ dependencies = [ "thiserror", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1626,10 +1638,11 @@ dependencies = [ [[package]] name = "futures-buffered" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fa130f3777d0d4b0993653c20bc433026d3290627693c4ed1b18dd237357ab" +checksum = "34acda8ae8b63fbe0b2195c998b180cff89a8212fb2622a78b572a9f1c6f7684" dependencies = [ + "cordyceps", "diatomic-waker", "futures-core", "pin-project-lite", @@ -1647,11 +1660,11 @@ dependencies = [ [[package]] name = "futures-concurrency" -version = "7.6.1" +version = "7.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b14ac911e85d57c5ea6eef76d7b4d4a3177ecd15f4bea2e61927e9e3823e19f" +checksum = "d9b724496da7c26fcce66458526ce68fc2ecf4aaaa994281cf322ded5755520c" dependencies = [ - "bitvec", + "fixedbitset", "futures-buffered", "futures-core", "futures-lite 1.13.0", @@ -1668,9 +1681,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1700,11 +1713,11 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.1.1", "futures-core", "futures-io", "parking", @@ -1719,7 +1732,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1789,6 +1802,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows 0.48.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1815,9 +1841,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -1858,9 +1884,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1868,7 +1894,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1907,7 +1933,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +dependencies = [ "allocator-api2", + "equivalent", + "foldhash", ] [[package]] @@ -1961,6 +1997,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2168,17 +2210,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -2198,15 +2229,15 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2235,16 +2266,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", "futures-util", "h2", "http 1.1.0", - "http-body 1.0.1", + "http-body", "httparse", "httpdate", "itoa", @@ -2256,9 +2287,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", @@ -2274,29 +2305,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "hyper", "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2315,6 +2345,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2341,6 +2489,27 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "igd-next" version = "0.15.1" @@ -2375,12 +2544,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.1", "serde", ] @@ -2434,31 +2603,32 @@ dependencies = [ "socket2", "widestring", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" dependencies = [ "serde", ] [[package]] name = "iroh" -version = "0.27.0" +version = "0.28.1" dependencies = [ "anyhow", "async-channel", "bao-tree", "bytes", + "cc", "clap", "console", "derive_more", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "genawaiter", "hex", @@ -2472,8 +2642,9 @@ dependencies = [ "iroh-metrics", "iroh-net", "iroh-quinn", + "iroh-relay", + "iroh-router", "iroh-test", - "iroh-willow", "nested_enum_utils", "num_cpus", "parking_lot", @@ -2487,6 +2658,7 @@ dependencies = [ "ref-cast", "regex", "serde", + "serde-error", "serde_json", "strum 0.25.0", "tempfile", @@ -2500,15 +2672,15 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "walkdir", ] [[package]] name = "iroh-base" -version = "0.27.0" +version = "0.28.0" dependencies = [ "aead", "anyhow", + "cc", "crypto_box", "data-encoding", "derive_more", @@ -2522,9 +2694,8 @@ dependencies = [ "proptest", "rand", "rand_core", - "redb 2.1.1", + "redb 2.2.0", "serde", - "serde-error", "serde_json", "serde_test", "ssh-key", @@ -2549,7 +2720,8 @@ dependencies = [ [[package]] name = "iroh-blobs" -version = "0.27.0" +version = "0.28.1" +source = "git+https://github.com/n0-computer/iroh-blobs?branch=matheus23/verified-streams#51f5707402f2a31a78a68fde40b480777e7d545a" dependencies = [ "anyhow", "async-channel", @@ -2558,50 +2730,49 @@ dependencies = [ "chrono", "derive_more", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "genawaiter", "hashlink", "hex", - "http-body 0.4.6", "iroh-base", - "iroh-blobs", "iroh-io", "iroh-metrics", "iroh-net", "iroh-quinn", - "iroh-test", + "iroh-router", + "nested_enum_utils", "num_cpus", "oneshot", "parking_lot", "pin-project", + "portable-atomic", "postcard", - "proptest", + "quic-rpc", + "quic-rpc-derive", "rand", "range-collections", - "rcgen", "redb 1.5.1", - "redb 2.1.1", + "redb 2.2.0", + "ref-cast", "reflink-copy", - "rustls", "self_cell", "serde", - "serde_json", - "serde_test", + "serde-error", "smallvec", + "strum 0.26.3", "tempfile", - "testresult", "thiserror", "tokio", "tokio-util", "tracing", "tracing-futures", - "tracing-subscriber", + "walkdir", ] [[package]] name = "iroh-cli" -version = "0.27.0" +version = "0.28.1" dependencies = [ "anyhow", "async-channel", @@ -2617,18 +2788,20 @@ dependencies = [ "dirs-next", "duct", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "hex", "human-time", "indicatif", "iroh", + "iroh-docs", "iroh-gossip", "iroh-metrics", "nix 0.27.1", "parking_lot", "pkarr", "portable-atomic", + "portmapper", "postcard", "quic-rpc", "rand", @@ -2658,7 +2831,7 @@ dependencies = [ [[package]] name = "iroh-dns-server" -version = "0.27.0" +version = "0.28.0" dependencies = [ "anyhow", "async-trait", @@ -2669,7 +2842,7 @@ dependencies = [ "clap", "derive_more", "dirs-next", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "governor", "hickory-proto 0.25.0-alpha.2", "hickory-resolver", @@ -2683,7 +2856,7 @@ dependencies = [ "parking_lot", "pkarr", "rcgen", - "redb 2.1.1", + "redb 2.2.0", "regex", "rustls", "rustls-pemfile", @@ -2707,7 +2880,8 @@ dependencies = [ [[package]] name = "iroh-docs" -version = "0.27.0" +version = "0.28.0" +source = "git+https://github.com/n0-computer/iroh-docs?branch=main#9166b226fedc889218986eb6ef2a3d3cdcdee367" dependencies = [ "anyhow", "async-channel", @@ -2715,7 +2889,7 @@ dependencies = [ "derive_more", "ed25519-dalek", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "hex", "iroh-base", @@ -2724,21 +2898,23 @@ dependencies = [ "iroh-gossip", "iroh-metrics", "iroh-net", - "iroh-test", + "iroh-router", "lru", + "nested_enum_utils", "num_enum", + "portable-atomic", "postcard", - "proptest", + "quic-rpc", + "quic-rpc-derive", "rand", - "rand_chacha", "rand_core", "redb 1.5.1", - "redb 2.1.1", + "redb 2.2.0", "self_cell", "serde", - "strum 0.25.0", + "serde-error", + "strum 0.26.3", "tempfile", - "test-strategy 0.3.1", "thiserror", "tokio", "tokio-stream", @@ -2748,43 +2924,45 @@ dependencies = [ [[package]] name = "iroh-gossip" -version = "0.27.0" +version = "0.28.1" +source = "git+https://github.com/n0-computer/iroh-gossip?branch=main#3d6659daf326f57cbafec189be996b89c7f441bd" dependencies = [ "anyhow", "async-channel", "bytes", - "clap", "derive_more", "ed25519-dalek", "futures-concurrency", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", - "indexmap 2.2.6", + "indexmap 2.6.0", "iroh-base", "iroh-blake3", "iroh-metrics", "iroh-net", - "iroh-test", + "iroh-router", + "nested_enum_utils", "postcard", + "quic-rpc", + "quic-rpc-derive", "rand", - "rand_chacha", "rand_core", "serde", + "serde-error", + "strum 0.26.3", "tokio", "tokio-util", "tracing", - "tracing-subscriber", - "url", ] [[package]] name = "iroh-io" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d1047ad5ca29ab4ff316b6830d86e7ea52cea54325e4d4a849692e1274b498" +checksum = "17e302c5ad649c6a7aa9ae8468e1c4dc2469321af0c6de7341c1be1bdaab434b" dependencies = [ "bytes", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "pin-project", "smallvec", "tokio", @@ -2792,7 +2970,7 @@ dependencies = [ [[package]] name = "iroh-metrics" -version = "0.27.0" +version = "0.28.0" dependencies = [ "anyhow", "erased_set", @@ -2811,13 +2989,14 @@ dependencies = [ [[package]] name = "iroh-net" -version = "0.27.0" +version = "0.28.1" dependencies = [ "anyhow", "axum", "backoff", "base64 0.22.1", "bytes", + "cc", "clap", "criterion", "crypto_box", @@ -2826,7 +3005,7 @@ dependencies = [ "duct", "futures-buffered", "futures-concurrency", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-sink", "futures-util", "genawaiter", @@ -2846,6 +3025,7 @@ dependencies = [ "iroh-quinn", "iroh-quinn-proto", "iroh-quinn-udp", + "iroh-relay", "iroh-test", "libc", "mainline", @@ -2853,15 +3033,15 @@ dependencies = [ "netlink-packet-core", "netlink-packet-route", "netlink-sys", - "ntest", + "netwatch", "num_enum", "once_cell", "parking_lot", "pin-project", "pkarr", + "portmapper", "postcard", "pretty_assertions", - "proptest", "rand", "rand_chacha", "rcgen", @@ -2870,29 +3050,24 @@ dependencies = [ "ring", "rtnetlink", "rustls", - "rustls-pemfile", "rustls-webpki", "serde", "serde_json", - "serde_with", "smallvec", "socket2", "strum 0.26.3", "stun-rs", "surge-ping", "swarm-discovery", - "testdir", "testresult", "thiserror", "time", "tokio", "tokio-rustls", - "tokio-rustls-acme", "tokio-stream", "tokio-tungstenite", "tokio-tungstenite-wasm", "tokio-util", - "toml", "tracing", "tracing-subscriber", "tungstenite", @@ -2912,7 +3087,7 @@ dependencies = [ "anyhow", "bytes", "clap", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "hdrhistogram", "iroh-metrics", "iroh-net", @@ -2927,15 +3102,15 @@ dependencies = [ [[package]] name = "iroh-quinn" -version = "0.11.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fd590a39a14cfc168efa4d894de5039d65641e62d8da4a80733018ababe3c33" +checksum = "35ba75a5c57cff299d2d7ca1ddee053f66339d1756bd79ec637bcad5aa61100e" dependencies = [ "bytes", "iroh-quinn-proto", "iroh-quinn-udp", "pin-project-lite", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "socket2", "thiserror", @@ -2945,14 +3120,14 @@ dependencies = [ [[package]] name = "iroh-quinn-proto" -version = "0.11.6" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd0538ff12efe3d61ea1deda2d7913f4270873a519d43e6995c6e87a1558538" +checksum = "e2c869ba52683d3d067c83ab4c00a2fda18eaf13b1434d4c1352f428674d4a5d" dependencies = [ "bytes", "rand", "ring", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "rustls-platform-verifier", "slab", @@ -2963,9 +3138,9 @@ dependencies = [ [[package]] name = "iroh-quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0619b59471fdd393ac8a6c047f640171119c1c8b41f7d2927db91776dcdbc5f" +checksum = "bfcfc0abc2fdf8cf18a6c72893b7cbebeac2274a3b1306c1760c48c0e10ac5e0" dependencies = [ "libc", "once_cell", @@ -2974,9 +3149,88 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "iroh-relay" +version = "0.28.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "clap", + "crypto_box", + "derive_more", + "duct", + "futures-lite 2.4.0", + "futures-sink", + "futures-util", + "governor", + "hex", + "hickory-proto 0.25.0-alpha.2", + "hickory-resolver", + "hostname", + "http 1.1.0", + "http-body-util", + "hyper", + "hyper-util", + "iroh-base", + "iroh-metrics", + "iroh-test", + "libc", + "num_enum", + "once_cell", + "parking_lot", + "pin-project", + "postcard", + "proptest", + "rand", + "rand_chacha", + "rcgen", + "regex", + "reqwest", + "ring", + "rustls", + "rustls-pemfile", + "rustls-webpki", + "serde", + "serde_json", + "smallvec", + "socket2", + "stun-rs", + "thiserror", + "time", + "tokio", + "tokio-rustls", + "tokio-rustls-acme", + "tokio-tungstenite", + "tokio-tungstenite-wasm", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", + "tungstenite", + "url", + "webpki-roots", +] + +[[package]] +name = "iroh-router" +version = "0.28.0" +dependencies = [ + "anyhow", + "clap", + "futures-buffered", + "futures-lite 2.4.0", + "futures-util", + "iroh-net", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "iroh-test" -version = "0.27.0" +version = "0.28.0" dependencies = [ "anyhow", "tokio", @@ -2996,7 +3250,7 @@ dependencies = [ "either", "futures-buffered", "futures-concurrency", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "genawaiter", "hex", @@ -3008,19 +3262,25 @@ dependencies = [ "iroh-net", "iroh-test", "meadowcap", + "nested_enum_utils", "postcard", "proptest", + "quic-rpc", + "quic-rpc-derive", "rand", "rand_chacha", "rand_core", - "redb 2.1.1", + "redb 2.2.0", + "ref-cast", "self_cell", "serde", + "serde-error", "sha2", "strum 0.26.3", "syncify", "tempfile", "test-strategy 0.3.1", + "testresult", "thiserror", "tokio", "tokio-stream", @@ -3037,20 +3297,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -3107,9 +3367,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -3125,15 +3385,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libredox" @@ -3157,6 +3417,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "litrs" version = "0.4.1" @@ -3179,13 +3445,26 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.1", ] [[package]] @@ -3289,11 +3568,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -3308,6 +3587,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -3411,6 +3702,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "netwatch" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "derive_more", + "futures-lite 2.4.0", + "futures-sink", + "futures-util", + "libc", + "netdev", + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", + "once_cell", + "rtnetlink", + "serde", + "socket2", + "thiserror", + "time", + "tokio", + "tokio-util", + "tracing", + "windows 0.51.1", + "wmi", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3597,29 +3916,29 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "num_enum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3639,27 +3958,27 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.1" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "oid-registry" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" dependencies = [ "asn1-rs", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oneshot" @@ -3693,12 +4012,12 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "os_pipe" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d73ba8daf8fac13b0501d1abeddcfe21ba7401ada61a819144b6c2a4f32209" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3747,9 +4066,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -3769,7 +4088,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -3807,9 +4126,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -3818,9 +4137,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" dependencies = [ "pest", "pest_generator", @@ -3828,22 +4147,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" dependencies = [ "once_cell", "pest", @@ -3852,29 +4171,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -3932,9 +4251,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -3945,15 +4264,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -3976,7 +4295,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4013,9 +4332,38 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "portmapper" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "derive_more", + "futures-lite 2.4.0", + "futures-util", + "igd-next", + "iroh-metrics", + "libc", + "netwatch", + "ntest", + "num_enum", + "rand", + "rand_chacha", + "serde", + "smallvec", + "socket2", + "thiserror", + "time", + "tokio", + "tokio-util", + "tracing", + "url", +] [[package]] name = "positioned-io" @@ -4060,15 +4408,18 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "precis-core" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d73e9dd26361c32e7cd13d1032bb01c4e26a23287274e8a4e2f228cf2c9ff77b" +checksum = "25a414cabc93f5f45d53463e73b3d89d3c5c0dc4a34dbf6901f0c6358f017203" dependencies = [ "precis-tools", "ucd-parse", @@ -4077,9 +4428,9 @@ dependencies = [ [[package]] name = "precis-profiles" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde4bd6624c60cb0abe2bea1dbdbb9085f629a853861e64df4abb099f8076ad4" +checksum = "f58e2841ef58164e2626464d4fde67fa301d5e2c78a10300c1756312a03b169f" dependencies = [ "lazy_static", "precis-core", @@ -4089,9 +4440,9 @@ dependencies = [ [[package]] name = "precis-tools" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07ecadec70b0f560f09abf815ae0ee1a940d38d2354c938ba7229ac7c9f5f52" +checksum = "016da884bc4c2c4670211641abef402d15fa2b06c6e9088ff270dac93675aee2" dependencies = [ "lazy_static", "regex", @@ -4110,9 +4461,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -4129,11 +4480,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -4170,9 +4521,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -4197,7 +4548,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4214,7 +4565,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -4237,16 +4588,16 @@ dependencies = [ [[package]] name = "quic-rpc" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cb85690ab1688eade9a5de4d94545a9ceef60639b3370f5e1a28f525eb5589" +checksum = "61e131f594054d27d077162815db3b5e9ddd76a28fbb9091b68095971e75c286" dependencies = [ "anyhow", "bincode", "derive_more", "educe", "flume", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-sink", "futures-util", "hex", @@ -4262,9 +4613,9 @@ dependencies = [ [[package]] name = "quic-rpc-derive" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6150a9fd3cf6c34d25730fe55a247b99d1c6e4fad6e7b7843f729a431a57e919" +checksum = "cbef4c942978f74ef296ae40d43d4375c9d730b65a582688a358108cfd5c0cf7" dependencies = [ "proc-macro2", "quic-rpc", @@ -4280,16 +4631,17 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 1.1.0", + "rustc-hash", "rustls", + "socket2", "thiserror", "tokio", "tracing", @@ -4297,14 +4649,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", "ring", - "rustc-hash 1.1.0", + "rustc-hash", "rustls", "slab", "thiserror", @@ -4314,21 +4666,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a78e6f726d84fcf960409f509ae354a32648f090c8d32a2ea8b1a1bc3bab14" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", - "windows-sys 0.52.0", + "tracing", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -4343,12 +4697,6 @@ dependencies = [ "pest_derive", ] -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "radix_trie" version = "0.2.1" @@ -4432,9 +4780,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -4482,36 +4830,27 @@ dependencies = [ [[package]] name = "redb" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd20d3cdeb9c7d2366a0b16b93b35b75aec15309fbeb7ce477138c9f68c8c0" +checksum = "84b1de48a7cf7ba193e81e078d17ee2b786236eed1d3f7c60f8a09545efc4925" dependencies = [ "libc", ] [[package]] name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -4535,7 +4874,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4551,14 +4890,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -4572,13 +4911,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -4595,22 +4934,22 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", "http 1.1.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-rustls", @@ -4638,7 +4977,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg 0.52.0", + "windows-registry", ] [[package]] @@ -4721,12 +5060,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.0.0" @@ -4735,9 +5068,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -4753,9 +5086,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -4766,9 +5099,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "log", "once_cell", @@ -4794,19 +5127,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-platform-verifier" @@ -4837,9 +5169,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -4848,9 +5180,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "rusty-fork" @@ -4913,13 +5245,19 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4956,9 +5294,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -4981,18 +5319,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde-error" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e988182713aeed6a619a88bca186f6d6407483485ffe44c869ee264f8eabd13f" +checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47" dependencies = [ "serde", ] @@ -5018,22 +5356,23 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5050,18 +5389,18 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] [[package]] name = "serde_test" -version = "1.0.176" +version = "1.0.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" dependencies = [ "serde", ] @@ -5080,15 +5419,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -5098,14 +5437,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5157,12 +5496,12 @@ dependencies = [ [[package]] name = "shared_child" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" dependencies = [ "libc", - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -5180,6 +5519,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.17" @@ -5192,12 +5537,12 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", - "mio", + "mio 0.8.11", "signal-hook", ] @@ -5314,9 +5659,9 @@ dependencies = [ [[package]] name = "ssh-key" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca9b366a80cf18bb6406f4cf4d10aebfb46140a8c0c33f666a144c5c76ecbafc" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" dependencies = [ "ed25519-dalek", "p256", @@ -5340,7 +5685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d904e7009df136af5297832a3ace3370cd14ff1546a232f4f185036c2736fcac" dependencies = [ "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5387,7 +5732,7 @@ dependencies = [ "proc-macro2", "quote", "struct_iterable_internal", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5405,7 +5750,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.2.0", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5417,7 +5762,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.3.0", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5428,7 +5773,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5439,7 +5784,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5470,7 +5815,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5483,14 +5828,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "stun-rs" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0adebf9fb8fba5c39ee34092b0383f247e4d1255b98fcffec94b4b797b85b677" +checksum = "b79cc624c9a747353810310af44f1f03f71eb4561284a894acc0396e6d0de76e" dependencies = [ "base64 0.22.1", "bounded-integer", @@ -5560,9 +5905,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -5591,6 +5936,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "syncify" @@ -5600,7 +5948,7 @@ checksum = "7e2f83220c0c5abf77ec9f4910c6590f75f1bf1405c7f2762bf35fb1bd11c5e7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5611,7 +5959,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5630,9 +5978,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -5649,22 +5997,17 @@ dependencies = [ "libc", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.1.1", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5676,7 +6019,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta 0.2.0", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5688,7 +6031,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta 0.3.0", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5713,22 +6056,22 @@ checksum = "614b328ff036a4ef882c61570f72918f7e9c5bee1da33f8e7f91e01daee7e56c" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5774,6 +6117,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -5801,32 +6154,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.1" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", "libc", - "mio", - "num_cpus", + "mio 1.0.2", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5887,9 +6239,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -5945,48 +6297,37 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.16", + "toml_edit", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.16" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.14", + "winnow", ] [[package]] @@ -5999,6 +6340,21 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", "tokio", "tower-layer", "tower-service", @@ -6014,7 +6370,7 @@ dependencies = [ "bitflags 2.6.0", "bytes", "http 1.1.0", - "http-body 1.0.1", + "http-body", "http-body-util", "pin-project-lite", "tower-layer", @@ -6024,15 +6380,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower_governor" @@ -6046,7 +6402,7 @@ dependencies = [ "http 1.1.0", "pin-project", "thiserror", - "tower", + "tower 0.4.13", "tracing", ] @@ -6082,7 +6438,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6186,15 +6542,15 @@ dependencies = [ [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "ufotofu" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c773ca845a07603d8ebe7292a3cefa20bf41305ce91246332c7fc1e3029eecaa" +checksum = "530b5dd047cf0fbc98b5138a17419d9f914de9b98971b6ac8a1b9e2050b711ef" dependencies = [ "either", "ufotofu_queues", @@ -6215,30 +6571,30 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-truncate" @@ -6253,15 +6609,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "universal-hash" @@ -6281,9 +6637,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ "base64 0.22.1", "log", @@ -6296,12 +6652,12 @@ dependencies = [ [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", "serde", ] @@ -6312,6 +6668,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -6326,9 +6694,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" @@ -6378,34 +6746,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -6415,9 +6784,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6425,22 +6794,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "watchable" @@ -6456,9 +6825,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -6466,20 +6835,20 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] [[package]] name = "whoami" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.4.1", + "redox_syscall", "wasite", "web-sys", ] @@ -6522,7 +6891,7 @@ dependencies = [ "genawaiter", "hex", "itertools 0.13.0", - "redb 2.1.1", + "redb 2.2.0", "ref-cast", "self_cell", "smallvec", @@ -6548,11 +6917,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6563,24 +6932,21 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.51.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-core 0.51.1", "windows-targets 0.48.5", ] [[package]] name = "windows" -version = "0.52.0" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" dependencies = [ - "windows-core 0.52.0", - "windows-implement 0.52.0", - "windows-interface 0.52.0", - "windows-targets 0.52.6", + "windows-core 0.51.1", + "windows-targets 0.48.5", ] [[package]] @@ -6617,24 +6983,13 @@ version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" dependencies = [ - "windows-implement 0.58.0", - "windows-interface 0.58.0", + "windows-implement", + "windows-interface", "windows-result", "windows-strings", "windows-targets 0.52.6", ] -[[package]] -name = "windows-implement" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12168c33176773b86799be25e2a2ba07c7aab9968b37541f1094dbd7a60c8946" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "windows-implement" version = "0.58.0" @@ -6643,29 +6998,29 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "windows-interface" -version = "0.52.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d8dc32e0095a7eeccebd0e3f09e9509365ecb3fc6ac4d6f5f14a3f6392942d1" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] -name = "windows-interface" -version = "0.58.0" +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", ] [[package]] @@ -6837,18 +7192,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.6.14" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -6863,28 +7209,19 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wmi" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f0a4062ca522aad4705a2948fd4061b3857537990202a8ddd5af21607f79a" +checksum = "ff00ac1309d4c462be86f03a55e409509e8bf4323ec296aeb4b381dd9aabe6ec" dependencies = [ "chrono", "futures", "log", "serde", "thiserror", - "windows 0.52.0", + "windows 0.58.0", + "windows-core 0.58.0", ] [[package]] @@ -6897,13 +7234,16 @@ dependencies = [ ] [[package]] -name = "wyz" -version = "0.5.1" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "x509-parser" @@ -6924,9 +7264,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.20" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" +checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" [[package]] name = "xmltree" @@ -6939,9 +7279,9 @@ dependencies = [ [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yasna" @@ -6952,6 +7292,30 @@ dependencies = [ "time", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "z32" version = "1.1.1" @@ -6976,7 +7340,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -6984,3 +7369,25 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] diff --git a/Cargo.toml b/Cargo.toml index 5fbf4f36ce1..3f73396f5e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,18 @@ [workspace] members = [ - "iroh", - "iroh-blobs", - "iroh-base", - "iroh-dns-server", - "iroh-gossip", - "iroh-metrics", - "iroh-net", - "iroh-docs", - "iroh-test", - "iroh-net/bench", - "iroh-cli", - "iroh-willow", + "iroh", + "iroh-base", + "iroh-dns-server", + "iroh-metrics", + "iroh-net", + "iroh-test", + "iroh-net/bench", + "iroh-cli", + "iroh-relay", + "iroh-router", + "iroh-willow", + "net-tools/netwatch", + "net-tools/portmapper", ] resolver = "2" @@ -63,3 +64,13 @@ unused-async = "warn" willow-data-model = { git = "https://github.com/n0-computer/willow-rs.git", branch = "main" } willow-encoding = { git = "https://github.com/n0-computer/willow-rs.git", branch = "main" } meadowcap = { git = "https://github.com/n0-computer/willow-rs.git", branch = "main" } + +iroh-base = { path = "./iroh-base" } +iroh-net = { path = "./iroh-net" } +iroh-metrics = { path = "./iroh-metrics" } +iroh-test = { path = "./iroh-test" } +iroh-router = { path = "./iroh-router" } + +iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "main" } +iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } +iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "matheus23/verified-streams" } diff --git a/Makefile.toml b/Makefile.toml index afee382a3c2..5dde2e70dd8 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -10,7 +10,7 @@ args = [ "--config", "unstable_features=true", "--config", - "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true", + "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true,format_code_in_doc_comments=true", ] [tasks.format-check] @@ -24,5 +24,5 @@ args = [ "--config", "unstable_features=true", "--config", - "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true", + "imports_granularity=Crate,group_imports=StdExternalCrate,reorder_imports=true,format_code_in_doc_comments=true", ] diff --git a/deny.toml b/deny.toml index 6fea480cfd0..784d4b49818 100644 --- a/deny.toml +++ b/deny.toml @@ -21,7 +21,8 @@ allow = [ "Unicode-DFS-2016", "Zlib", "MPL-2.0", # https://fossa.com/blog/open-source-software-licenses-101-mozilla-public-license-2-0/ - "CC-PDDC" # https://spdx.org/licenses/CC-PDDC.html + "CC-PDDC", # https://spdx.org/licenses/CC-PDDC.html + "Unicode-3.0", ] [[licenses.clarify]] @@ -34,6 +35,14 @@ license-files = [ [advisories] ignore = [ "RUSTSEC-2024-0370", # unmaintained, no upgrade available + "RUSTSEC-2024-0384", # unmaintained, no upgrade available +] + +[sources] +allow-git = [ + "https://github.com/n0-computer/iroh-blobs.git", + "https://github.com/n0-computer/iroh-gossip.git", + "https://github.com/n0-computer/iroh-docs.git", ] # TODO(Frando): added for iroh-willow development, maybe remove again before release? diff --git a/iroh-base/Cargo.toml b/iroh-base/Cargo.toml index 577e8387bbd..22945ad860e 100644 --- a/iroh-base/Cargo.toml +++ b/iroh-base/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-base" -version = "0.27.0" +version = "0.28.0" edition = "2021" readme = "README.md" description = "base type and utilities for Iroh" @@ -15,6 +15,7 @@ rust-version = "1.76" workspace = true [dependencies] +cc = "=1.1.31" # enforce cc version, because of https://github.com/rust-lang/cc-rs/issues/1278 anyhow = { version = "1" } blake3 = { version = "1.4.5", package = "iroh-blake3", optional = true } data-encoding = { version = "2.3.3", optional = true } @@ -22,7 +23,6 @@ hex = "0.4.3" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"], optional = true } redb = { version = "2.0.0", optional = true } serde = { version = "1", features = ["derive"] } -serde-error = "0.1.2" thiserror = "1" # key module @@ -41,7 +41,7 @@ url = { version = "2.5.0", features = ["serde"], optional = true } getrandom = { version = "0.2", default-features = false, optional = true } [dev-dependencies] -iroh-test = { path = "../iroh-test" } +iroh-test = "0.28.0" proptest = "1.0.0" serde_json = "1.0.107" serde_test = "1.0.176" diff --git a/iroh-base/src/lib.rs b/iroh-base/src/lib.rs index cd726a60066..2c8ec03fdc0 100644 --- a/iroh-base/src/lib.rs +++ b/iroh-base/src/lib.rs @@ -13,7 +13,6 @@ pub mod key; #[cfg(feature = "key")] #[cfg_attr(iroh_docsrs, doc(cfg(feature = "key")))] pub mod node_addr; -pub mod rpc; #[cfg(feature = "base32")] #[cfg_attr(iroh_docsrs, doc(cfg(feature = "base32")))] pub mod ticket; diff --git a/iroh-base/src/rpc.rs b/iroh-base/src/rpc.rs deleted file mode 100644 index 845ee7438d4..00000000000 --- a/iroh-base/src/rpc.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::fmt; - -use serde::{Deserialize, Serialize}; - -/// A serializable error type for use in RPC responses. -#[derive(Serialize, Deserialize, Debug, thiserror::Error)] -pub struct RpcError(serde_error::Error); - -impl fmt::Display for RpcError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -impl From for RpcError { - fn from(e: anyhow::Error) -> Self { - RpcError(serde_error::Error::new(&*e)) - } -} - -impl From for RpcError { - fn from(e: std::io::Error) -> Self { - RpcError(serde_error::Error::new(&e)) - } -} - -impl std::clone::Clone for RpcError { - fn clone(&self) -> Self { - RpcError(serde_error::Error::new(self)) - } -} - -/// A serializable result type for use in RPC responses. -pub type RpcResult = std::result::Result; diff --git a/iroh-blobs/Cargo.toml b/iroh-blobs/Cargo.toml deleted file mode 100644 index af83e98ea08..00000000000 --- a/iroh-blobs/Cargo.toml +++ /dev/null @@ -1,89 +0,0 @@ -[package] -name = "iroh-blobs" -version = "0.27.0" -edition = "2021" -readme = "README.md" -description = "blob and collection transfer support for iroh" -license = "MIT OR Apache-2.0" -authors = ["dignifiedquire ", "n0 team"] -repository = "https://github.com/n0-computer/iroh" -keywords = ["hashing", "quic", "blake3"] - -# Sadly this also needs to be updated in .github/workflows/ci.yml -rust-version = "1.76" - -[lints] -workspace = true - -[dependencies] -anyhow = { version = "1" } -async-channel = "2.3.1" -bao-tree = { version = "0.13", features = ["tokio_fsm", "validate"], default-features = false } -bytes = { version = "1.7", features = ["serde"] } -chrono = "0.4.31" -derive_more = { version = "1.0.0", features = ["debug", "display", "deref", "deref_mut", "from", "try_into", "into"] } -futures-buffered = "0.2.4" -futures-lite = "2.3" -genawaiter = { version = "0.99.1", features = ["futures03"] } -hashlink = { version = "0.9.0", optional = true } -hex = "0.4.3" -iroh-base = { version = "0.27.0", features = ["redb"], path = "../iroh-base" } -iroh-io = { version = "0.6.0", features = ["stats"] } -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics", default-features = false } -iroh-net = { version = "0.27.0", path = "../iroh-net" } -num_cpus = "1.15.0" -oneshot = "0.1.8" -parking_lot = { version = "0.12.1", optional = true } -pin-project = "1.1.5" -postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -quinn = { package = "iroh-quinn", version = "0.11", features = ["ring"] } -rand = "0.8" -range-collections = "0.4.0" -redb = { version = "2.0.0", optional = true } -redb_v1 = { package = "redb", version = "1.5.1", optional = true } -reflink-copy = { version = "0.1.8", optional = true } -self_cell = "1.0.1" -serde = { version = "1", features = ["derive"] } -smallvec = { version = "1.10.0", features = ["serde", "const_new"] } -tempfile = { version = "3.10.0", optional = true } -thiserror = "1" -tokio = { version = "1", features = ["fs"] } -tokio-util = { version = "0.7", features = ["io-util", "io"] } -tracing = "0.1" -tracing-futures = "0.2.5" - -[dev-dependencies] -http-body = "0.4.5" -iroh-blobs = { path = ".", features = ["downloader"] } -iroh-test = { path = "../iroh-test" } -futures-buffered = "0.2.4" -proptest = "1.0.0" -serde_json = "1.0.107" -serde_test = "1.0.176" -testresult = "0.4.0" -tokio = { version = "1", features = ["macros", "test-util"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -rcgen = "0.12.0" -rustls = { version = "0.23", default-features = false, features = ["ring"] } -tempfile = "3.10.0" -futures-util = "0.3.30" - -[features] -default = ["fs-store"] -downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"] -fs-store = ["dep:reflink-copy", "redb", "dep:redb_v1", "dep:tempfile"] -metrics = ["iroh-metrics/metrics"] -redb = ["dep:redb"] - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "iroh_docsrs"] - -[[example]] -name = "provide-bytes" - -[[example]] -name = "fetch-fsm" - -[[example]] -name = "fetch-stream" diff --git a/iroh-blobs/README.md b/iroh-blobs/README.md deleted file mode 100644 index 0958f5ffd1e..00000000000 --- a/iroh-blobs/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# iroh-blobs - -This crate provides blob and collection transfer support for iroh. It implements a simple request-response protocol based on blake3 verified streaming. - -A request describes data in terms of blake3 hashes and byte ranges. It is possible to -request blobs or ranges of blobs, as well as collections. - -The requester opens a quic stream to the provider and sends the request. The provider answers with the requested data, encoded as [blake3](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf) verified streams, on the same quic stream. - -This crate is usually used together with [iroh-net](https://crates.io/crates/iroh-net), but can also be used with normal [quinn](https://crates.io/crates/quinn) connections. Connection establishment is left up to the user or a higher level APIs such as the iroh CLI. - -## Concepts - -- **Blob:** a sequence of bytes of arbitrary size, without any metadata. - -- **Link:** a 32 byte blake3 hash of a blob. - -- **Collection:** any blob that contains links. The simplest collection is just an array of 32 byte blake3 hashes. - -- **Provider:** The side that provides data and answers requests. Providers wait for incoming requests from Requests. - -- **Requester:** The side that asks for data. It is initiating requests to one or many providers. - -## Examples - -Examples that use `iroh-blobs` can be found in the `iroh` crate. the iroh crate publishes `iroh_blobs` as `iroh::bytes`. - - -# License - -This project is licensed under either of - - * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or - http://www.apache.org/licenses/LICENSE-2.0) - * MIT license ([LICENSE-MIT](LICENSE-MIT) or - http://opensource.org/licenses/MIT) - -at your option. - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in this project by you, as defined in the Apache-2.0 license, -shall be dual licensed as above, without any additional terms or conditions. - diff --git a/iroh-blobs/docs/img/get_machine.drawio b/iroh-blobs/docs/img/get_machine.drawio deleted file mode 100644 index 363a2ec60fa..00000000000 --- a/iroh-blobs/docs/img/get_machine.drawio +++ /dev/null @@ -1,238 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/iroh-blobs/docs/img/get_machine.drawio.svg b/iroh-blobs/docs/img/get_machine.drawio.svg deleted file mode 100644 index e512582a9f0..00000000000 --- a/iroh-blobs/docs/img/get_machine.drawio.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -
Initial
Initial
Connected
Connected
StartChild
StartChild
StartRoot
StartRoot
BlobHeader
BlobHeader
BlobContent
BlobContent
BlobEnd
BlobEnd
Closing
Closing
Stats
Stats
Error
Error
(hash)
(hash)Text is not SVG - cannot display \ No newline at end of file diff --git a/iroh-blobs/examples/connect/mod.rs b/iroh-blobs/examples/connect/mod.rs deleted file mode 100644 index 61958589a32..00000000000 --- a/iroh-blobs/examples/connect/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Common code used to created quinn connections in the examples -use std::{path::PathBuf, sync::Arc}; - -use anyhow::{bail, Context, Result}; -use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; -use tokio::fs; - -pub const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/bytes/0"; - -// Path where the tls certificates are saved. This example expects that you have run the `provide-bytes` example first, which generates the certificates. -pub const CERT_PATH: &str = "./certs"; - -// derived from `quinn/examples/client.rs` -// load the certificates from CERT_PATH -// Assumes that you have already run the `provide-bytes` example, that generates the certificates -#[allow(unused)] -pub async fn load_certs() -> Result { - let mut roots = rustls::RootCertStore::empty(); - let path = PathBuf::from(CERT_PATH).join("cert.der"); - match fs::read(path).await { - Ok(cert) => { - roots.add(rustls::pki_types::CertificateDer::from(cert))?; - } - Err(e) => { - bail!("failed to open local server certificate: {}\nYou must run the `provide-bytes` example to create the certificate.\n\tcargo run --example provide-bytes", e); - } - } - Ok(roots) -} - -// derived from `quinn/examples/server.rs` -// creates a self signed certificate and saves it to "./certs" -#[allow(unused)] -pub async fn make_and_write_certs() -> Result<( - rustls::pki_types::PrivateKeyDer<'static>, - rustls::pki_types::CertificateDer<'static>, -)> { - let path = std::path::PathBuf::from(CERT_PATH); - let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); - let key_path = path.join("key.der"); - let cert_path = path.join("cert.der"); - - let key = cert.serialize_private_key_der(); - let cert = cert.serialize_der().unwrap(); - tokio::fs::create_dir_all(path) - .await - .context("failed to create certificate directory")?; - tokio::fs::write(cert_path, &cert) - .await - .context("failed to write certificate")?; - tokio::fs::write(key_path, &key) - .await - .context("failed to write private key")?; - - Ok(( - rustls::pki_types::PrivateKeyDer::try_from(key).unwrap(), - rustls::pki_types::CertificateDer::from(cert), - )) -} - -// derived from `quinn/examples/client.rs` -// Creates a client quinn::Endpoint -#[allow(unused)] -pub fn make_client_endpoint(roots: rustls::RootCertStore) -> Result { - let mut client_crypto = rustls::ClientConfig::builder() - .with_root_certificates(roots) - .with_no_client_auth(); - - client_crypto.alpn_protocols = vec![EXAMPLE_ALPN.to_vec()]; - let client_config: QuicClientConfig = client_crypto.try_into()?; - let client_config = quinn::ClientConfig::new(Arc::new(client_config)); - let mut endpoint = quinn::Endpoint::client("[::]:0".parse().unwrap())?; - endpoint.set_default_client_config(client_config); - Ok(endpoint) -} - -// derived from `quinn/examples/server.rs` -// makes a quinn server endpoint -#[allow(unused)] -pub fn make_server_endpoint( - key: rustls::pki_types::PrivateKeyDer<'static>, - cert: rustls::pki_types::CertificateDer<'static>, -) -> Result { - let mut server_crypto = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(vec![cert], key)?; - server_crypto.alpn_protocols = vec![EXAMPLE_ALPN.to_vec()]; - let server_config: QuicServerConfig = server_crypto.try_into()?; - let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(server_config)); - let transport_config = Arc::get_mut(&mut server_config.transport).unwrap(); - transport_config.max_concurrent_uni_streams(0_u8.into()); - - let endpoint = quinn::Endpoint::server(server_config, "[::1]:4433".parse()?)?; - Ok(endpoint) -} diff --git a/iroh-blobs/examples/fetch-fsm.rs b/iroh-blobs/examples/fetch-fsm.rs deleted file mode 100644 index 6c7379b3a14..00000000000 --- a/iroh-blobs/examples/fetch-fsm.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! An example how to download a single blob or collection from a node and write it to stdout using the `get` finite state machine directly. -//! -//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. -//! -//! Run the provide-bytes example first. It will give instructions on how to run this example properly. -use std::net::SocketAddr; - -use anyhow::{Context, Result}; -use iroh_blobs::{ - get::fsm::{AtInitial, ConnectedNext, EndBlobNext}, - hashseq::HashSeq, - protocol::GetRequest, - Hash, -}; -use iroh_io::ConcatenateSliceWriter; -use tracing_subscriber::{prelude::*, EnvFilter}; - -mod connect; -use connect::{load_certs, make_client_endpoint}; - -// set the RUST_LOG env var to one of {debug,info,warn} to see logging info -pub fn setup_logging() { - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) - .with(EnvFilter::from_default_env()) - .try_init() - .ok(); -} - -#[tokio::main] -async fn main() -> Result<()> { - println!("\nfetch bytes example!"); - setup_logging(); - let args: Vec<_> = std::env::args().collect(); - if args.len() != 4 { - anyhow::bail!("usage: fetch-bytes [HASH] [SOCKET_ADDR] [FORMAT]"); - } - let hash: Hash = args[1].parse().context("unable to parse [HASH]")?; - let addr: SocketAddr = args[2].parse().context("unable to parse [SOCKET_ADDR]")?; - let format = { - if args[3] != "blob" && args[3] != "collection" { - anyhow::bail!( - "expected either 'blob' or 'collection' for FORMAT argument, got {}", - args[3] - ); - } - args[3].clone() - }; - - // load tls certificates - // This will error if you have not run the `provide-bytes` example - let roots = load_certs().await?; - - // create an endpoint to listen for incoming connections - let endpoint = make_client_endpoint(roots)?; - println!("\nlistening on {}", endpoint.local_addr()?); - println!("fetching hash {hash} from {addr}"); - - // connect - let connection = endpoint.connect(addr, "localhost")?.await?; - - if format == "collection" { - // create a request for a collection - let request = GetRequest::all(hash); - // create the initial state of the finite state machine - let initial = iroh_blobs::get::fsm::start(connection, request); - - write_collection(initial).await - } else { - // create a request for a single blob - let request = GetRequest::single(hash); - // create the initial state of the finite state machine - let initial = iroh_blobs::get::fsm::start(connection, request); - - write_blob(initial).await - } -} - -async fn write_blob(initial: AtInitial) -> Result<()> { - // connect (create a stream pair) - let connected = initial.next().await?; - - // we expect a start root message, since we requested a single blob - let ConnectedNext::StartRoot(start_root) = connected.next().await? else { - panic!("expected start root") - }; - // we can just call next to proceed to the header, since we know the root hash - let header = start_root.next(); - - // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not - // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes. - let writer = ConcatenateSliceWriter::new(tokio::io::stdout()); - - // make the spacing nicer in the terminal - println!(); - // use the utility function write_all to write the entire blob - let end = header.write_all(writer).await?; - - // we requested a single blob, so we expect to enter the closing state - let EndBlobNext::Closing(closing) = end.next() else { - panic!("expected closing") - }; - - // close the connection and get the stats - let _stats = closing.next().await?; - Ok(()) -} - -async fn write_collection(initial: AtInitial) -> Result<()> { - // connect - let connected = initial.next().await?; - // read the first bytes - let ConnectedNext::StartRoot(start_root) = connected.next().await? else { - anyhow::bail!("failed to parse collection"); - }; - // check that we requested the whole collection - if !start_root.ranges().is_all() { - anyhow::bail!("collection was not requested completely"); - } - - // move to the header - let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next(); - let (root_end, hashes_bytes) = header.concatenate_into_vec().await?; - let next = root_end.next(); - let EndBlobNext::MoreChildren(at_meta) = next else { - anyhow::bail!("missing meta blob, got {next:?}"); - }; - // parse the hashes from the hash sequence bytes - let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes)) - .context("failed to parse hashes")? - .into_iter() - .collect::>(); - let meta_hash = hashes.first().context("missing meta hash")?; - - let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?; - let mut curr = meta_end.next(); - let closing = loop { - match curr { - EndBlobNext::MoreChildren(more) => { - let Some(hash) = hashes.get(more.child_offset() as usize) else { - break more.finish(); - }; - let header = more.next(*hash); - - // we need to wrap stdout in a struct that implements AsyncSliceWriter. Since we can not - // seek in stdout we use ConcatenateSliceWriter which just concatenates all the writes. - let writer = ConcatenateSliceWriter::new(tokio::io::stdout()); - - // use the utility function write_all to write the entire blob - let end = header.write_all(writer).await?; - println!(); - curr = end.next(); - } - EndBlobNext::Closing(closing) => { - break closing; - } - } - }; - // close the connection - let _stats = closing.next().await?; - Ok(()) -} diff --git a/iroh-blobs/examples/fetch-stream.rs b/iroh-blobs/examples/fetch-stream.rs deleted file mode 100644 index 6b50d55f92f..00000000000 --- a/iroh-blobs/examples/fetch-stream.rs +++ /dev/null @@ -1,230 +0,0 @@ -//! An example how to download a single blob or collection from a node and write it to stdout, using a helper method to turn the `get` finite state machine into a stream. -//! -//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. -//! -//! Run the provide-bytes example first. It will give instructions on how to run this example properly. -use std::{io, net::SocketAddr}; - -use anyhow::{Context, Result}; -use bao_tree::io::fsm::BaoContentItem; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use genawaiter::sync::{Co, Gen}; -use iroh_blobs::{ - get::fsm::{AtInitial, BlobContentNext, ConnectedNext, EndBlobNext}, - hashseq::HashSeq, - protocol::GetRequest, - Hash, -}; -use tokio::io::AsyncWriteExt; -use tracing_subscriber::{prelude::*, EnvFilter}; - -mod connect; -use connect::{load_certs, make_client_endpoint}; - -// set the RUST_LOG env var to one of {debug,info,warn} to see logging info -pub fn setup_logging() { - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) - .with(EnvFilter::from_default_env()) - .try_init() - .ok(); -} - -#[tokio::main] -async fn main() -> Result<()> { - println!("\nfetch stream example!"); - setup_logging(); - let args: Vec<_> = std::env::args().collect(); - if args.len() != 4 { - anyhow::bail!("usage: fetch-bytes [HASH] [SOCKET_ADDR] [FORMAT]"); - } - let hash: Hash = args[1].parse().context("unable to parse [HASH]")?; - let addr: SocketAddr = args[2].parse().context("unable to parse [SOCKET_ADDR]")?; - let format = { - if args[3] != "blob" && args[3] != "collection" { - anyhow::bail!( - "expected either 'blob' or 'collection' for FORMAT argument, got {}", - args[3] - ); - } - args[3].clone() - }; - - // load tls certificates - // This will error if you have not run the `provide-bytes` example - let roots = load_certs().await?; - - // create an endpoint to listen for incoming connections - let endpoint = make_client_endpoint(roots)?; - println!("\nlistening on {}", endpoint.local_addr()?); - println!("fetching hash {hash} from {addr}"); - - // connect - let connection = endpoint.connect(addr, "localhost")?.await?; - - let mut stream = if format == "collection" { - // create a request for a collection - let request = GetRequest::all(hash); - - // create the initial state of the finite state machine - let initial = iroh_blobs::get::fsm::start(connection, request); - - // create a stream that yields all the data of the blob - stream_children(initial).boxed_local() - } else { - // create a request for a single blob - let request = GetRequest::single(hash); - - // create the initial state of the finite state machine - let initial = iroh_blobs::get::fsm::start(connection, request); - - // create a stream that yields all the data of the blob - stream_blob(initial).boxed_local() - }; - while let Some(item) = stream.next().await { - let item = item?; - tokio::io::stdout().write_all(&item).await?; - println!(); - } - Ok(()) -} - -/// Stream the response for a request for a single blob. -/// -/// If the request was for a part of the blob, this will stream just the requested -/// blocks. -/// -/// This will stream the root blob and close the connection. -fn stream_blob(initial: AtInitial) -> impl Stream> + 'static { - async fn inner(initial: AtInitial, co: &Co>) -> io::Result<()> { - // connect - let connected = initial.next().await?; - // read the first bytes - let ConnectedNext::StartRoot(start_root) = connected.next().await? else { - return Err(io::Error::new(io::ErrorKind::Other, "expected start root")); - }; - // move to the header - let header = start_root.next(); - // get the size of the content - let (mut content, _size) = header.next().await?; - // manually loop over the content and yield all data - let done = loop { - match content.next().await { - BlobContentNext::More((next, data)) => { - if let BaoContentItem::Leaf(leaf) = data? { - // yield the data - co.yield_(Ok(leaf.data)).await; - } - content = next; - } - BlobContentNext::Done(done) => { - // we are done with the root blob - break done; - } - } - }; - // close the connection even if there is more data - let closing = match done.next() { - EndBlobNext::Closing(closing) => closing, - EndBlobNext::MoreChildren(more) => more.finish(), - }; - // close the connection - let _stats = closing.next().await?; - Ok(()) - } - - Gen::new(|co| async move { - if let Err(e) = inner(initial, &co).await { - co.yield_(Err(e)).await; - } - }) -} - -/// Stream the response for a request for an iroh collection and its children. -/// -/// If the request was for a part of the children, this will stream just the requested -/// blocks. -/// -/// The root blob is not streamed. It must be fully included in the response. -fn stream_children(initial: AtInitial) -> impl Stream> + 'static { - async fn inner(initial: AtInitial, co: &Co>) -> io::Result<()> { - // connect - let connected = initial.next().await?; - // read the first bytes - let ConnectedNext::StartRoot(start_root) = connected.next().await? else { - return Err(io::Error::new( - io::ErrorKind::Other, - "failed to parse collection", - )); - }; - // check that we requested the whole collection - if !start_root.ranges().is_all() { - return Err(io::Error::new( - io::ErrorKind::Other, - "collection was not requested completely", - )); - } - // move to the header - let header: iroh_blobs::get::fsm::AtBlobHeader = start_root.next(); - let (root_end, hashes_bytes) = header.concatenate_into_vec().await?; - - // parse the hashes from the hash sequence bytes - let hashes = HashSeq::try_from(bytes::Bytes::from(hashes_bytes)) - .map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("failed to parse hashes: {e}")) - })? - .into_iter() - .collect::>(); - - let next = root_end.next(); - let EndBlobNext::MoreChildren(at_meta) = next else { - return Err(io::Error::new(io::ErrorKind::Other, "missing meta blob")); - }; - let meta_hash = hashes - .first() - .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "missing meta link"))?; - let (meta_end, _meta_bytes) = at_meta.next(*meta_hash).concatenate_into_vec().await?; - let mut curr = meta_end.next(); - let closing = loop { - match curr { - EndBlobNext::MoreChildren(more) => { - let Some(hash) = hashes.get(more.child_offset() as usize) else { - break more.finish(); - }; - let header = more.next(*hash); - let (mut content, _size) = header.next().await?; - // manually loop over the content and yield all data - let done = loop { - match content.next().await { - BlobContentNext::More((next, data)) => { - if let BaoContentItem::Leaf(leaf) = data? { - // yield the data - co.yield_(Ok(leaf.data)).await; - } - content = next; - } - BlobContentNext::Done(done) => { - // we are done with the root blob - break done; - } - } - }; - curr = done.next(); - } - EndBlobNext::Closing(closing) => { - break closing; - } - } - }; - // close the connection - let _stats = closing.next().await?; - Ok(()) - } - - Gen::new(|co| async move { - if let Err(e) = inner(initial, &co).await { - co.yield_(Err(e)).await; - } - }) -} diff --git a/iroh-blobs/examples/provide-bytes.rs b/iroh-blobs/examples/provide-bytes.rs deleted file mode 100644 index 98146f28c7a..00000000000 --- a/iroh-blobs/examples/provide-bytes.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! An example that provides a blob or a collection over a Quinn connection. -//! -//! Since this example does not use [`iroh-net::Endpoint`], it does not do any holepunching, and so will only work locally or between two processes that have public IP addresses. -//! -//! Run this example with -//! cargo run --example provide-bytes blob -//! To provide a blob (single file) -//! -//! Run this example with -//! cargo run --example provide-bytes collection -//! To provide a collection (multiple blobs) -use anyhow::Result; -use iroh_blobs::{format::collection::Collection, util::local_pool::LocalPool, Hash}; -use tracing::warn; -use tracing_subscriber::{prelude::*, EnvFilter}; - -mod connect; -use connect::{make_and_write_certs, make_server_endpoint, CERT_PATH}; - -// set the RUST_LOG env var to one of {debug,info,warn} to see logging info -pub fn setup_logging() { - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) - .with(EnvFilter::from_default_env()) - .try_init() - .ok(); -} - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - if args.len() != 2 { - anyhow::bail!( - "usage: provide-bytes [FORMAT], where [FORMAT] is either 'blob' or 'collection'\n\nThe 'blob' example demonstrates sending a single blob of bytes. The 'collection' example demonstrates sending multiple blobs of bytes, grouped together in a 'collection'." - ); - } - let format = { - if args[1] != "blob" && args[1] != "collection" { - anyhow::bail!( - "expected either 'blob' or 'collection' for FORMAT argument, got {}", - args[1] - ); - } - args[1].clone() - }; - println!("\nprovide bytes {format} example!"); - - let (db, hash) = if format == "collection" { - let (mut db, names) = iroh_blobs::store::readonly_mem::Store::new([ - ("blob1", b"the first blob of bytes".to_vec()), - ("blob2", b"the second blob of bytes".to_vec()), - ]); // create a collection - let collection: Collection = names - .into_iter() - .map(|(name, hash)| (name, Hash::from(hash))) - .collect(); - // add it to the db - let hash = db.insert_many(collection.to_blobs()).unwrap(); - (db, hash) - } else { - // create a new database and add a blob - let (db, names) = - iroh_blobs::store::readonly_mem::Store::new([("hello", b"Hello World!".to_vec())]); - - // get the hash of the content - let hash = names.get("hello").unwrap(); - (db, Hash::from(hash.as_bytes())) - }; - - // create tls certs and save to CERT_PATH - let (key, cert) = make_and_write_certs().await?; - - // create an endpoint to listen for incoming connections - let endpoint = make_server_endpoint(key, cert)?; - let addr = endpoint.local_addr()?; - println!("\nlistening on {addr}"); - println!("providing hash {hash}"); - - println!("\nfetch the content using a finite state machine by running the following example:\n\ncargo run --example fetch-fsm {hash} \"{addr}\" {format}"); - println!("\nfetch the content using a stream by running the following example:\n\ncargo run --example fetch-stream {hash} \"{addr}\" {format}\n"); - - // create a new local pool handle with 1 worker thread - let lp = LocalPool::single(); - - let accept_task = tokio::spawn(async move { - while let Some(incoming) = endpoint.accept().await { - println!("connection incoming"); - - let conn = match incoming.accept() { - Ok(conn) => conn, - Err(err) => { - warn!("incoming connection failed: {err:#}"); - // we can carry on in these cases: - // this can be caused by retransmitted datagrams - continue; - } - }; - let db = db.clone(); - let lp = lp.clone(); - - // spawn a task to handle the connection - tokio::spawn(async move { - let remote_addr = conn.remote_address(); - let conn = match conn.await { - Ok(conn) => conn, - Err(err) => { - warn!(%remote_addr, "Error connecting: {err:#}"); - return; - } - }; - iroh_blobs::provider::handle_connection(conn, db, Default::default(), lp).await - }); - } - }); - - match tokio::signal::ctrl_c().await { - Ok(()) => { - tokio::fs::remove_dir_all(std::path::PathBuf::from(CERT_PATH)).await?; - accept_task.abort(); - Ok(()) - } - Err(e) => Err(anyhow::anyhow!("unable to listen for ctrl-c: {e}")), - } -} diff --git a/iroh-blobs/proptest-regressions/protocol/range_spec.txt b/iroh-blobs/proptest-regressions/protocol/range_spec.txt deleted file mode 100644 index 8558e5a406a..00000000000 --- a/iroh-blobs/proptest-regressions/protocol/range_spec.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 7375b003a63bfe725eb4bcb2f266fae6afd9b3c921f9c2018f97daf6ef05a364 # shrinks to ranges = [RangeSet{ChunkNum(0)..ChunkNum(1)}, RangeSet{}] -cc 23322efa46881646f1468137a688e66aee7ec2a3d01895ccad851d442a7828af # shrinks to ranges = [RangeSet{}, RangeSet{ChunkNum(0)..ChunkNum(1)}] diff --git a/iroh-blobs/proptest-regressions/provider.txt b/iroh-blobs/proptest-regressions/provider.txt deleted file mode 100644 index 9471db84699..00000000000 --- a/iroh-blobs/proptest-regressions/provider.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 25ec044e2b84054195984d7e04b93d9b39e2cc25eaee4037dc1be9398f9fd4b4 # shrinks to db = Database(RwLock { data: {}, poisoned: false, .. }) diff --git a/iroh-blobs/src/downloader.rs b/iroh-blobs/src/downloader.rs deleted file mode 100644 index ee8952550df..00000000000 --- a/iroh-blobs/src/downloader.rs +++ /dev/null @@ -1,1514 +0,0 @@ -//! Handle downloading blobs and collections concurrently and from nodes. -//! -//! The [`Downloader`] interacts with four main components to this end. -//! - [`Dialer`]: Used to queue opening connections to nodes we need to perform downloads. -//! - `ProviderMap`: Where the downloader obtains information about nodes that could be -//! used to perform a download. -//! - [`Store`]: Where data is stored. -//! -//! Once a download request is received, the logic is as follows: -//! 1. The `ProviderMap` is queried for nodes. From these nodes some are selected -//! prioritizing connected nodes with lower number of active requests. If no useful node is -//! connected, or useful connected nodes have no capacity to perform the request, a connection -//! attempt is started using the [`Dialer`]. -//! 2. The download is queued for processing at a later time. Downloads are not performed right -//! away. Instead, they are initially delayed to allow the node to obtain the data itself, and -//! to wait for the new connection to be established if necessary. -//! 3. Once a request is ready to be sent after a delay (initial or for a retry), the preferred -//! node is used if available. The request is now considered active. -//! -//! Concurrency is limited in different ways: -//! - *Total number of active request:* This is a way to prevent a self DoS by overwhelming our own -//! bandwidth capacity. This is a best effort heuristic since it doesn't take into account how -//! much data we are actually requesting or receiving. -//! - *Total number of connected nodes:* Peer connections are kept for a longer time than they are -//! strictly needed since it's likely they will be useful soon again. -//! - *Requests per node*: to avoid overwhelming nodes with requests, the number of concurrent -//! requests to a single node is also limited. - -use std::{ - collections::{ - hash_map::{self, Entry}, - HashMap, HashSet, - }, - fmt, - future::Future, - num::NonZeroUsize, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - time::Duration, -}; - -use futures_lite::{future::BoxedLocal, Stream, StreamExt}; -use hashlink::LinkedHashSet; -use iroh_base::hash::{BlobFormat, Hash, HashAndFormat}; -use iroh_metrics::inc; -use iroh_net::{endpoint, Endpoint, NodeAddr, NodeId}; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinSet, -}; -use tokio_util::{either::Either, sync::CancellationToken, time::delay_queue}; -use tracing::{debug, error_span, trace, warn, Instrument}; - -use crate::{ - get::{db::DownloadProgress, Stats}, - metrics::Metrics, - store::Store, - util::{local_pool::LocalPoolHandle, progress::ProgressSender}, -}; - -mod get; -mod invariants; -mod progress; -mod test; - -use self::progress::{BroadcastProgressSender, ProgressSubscriber, ProgressTracker}; - -/// Duration for which we keep nodes connected after they were last useful to us. -const IDLE_PEER_TIMEOUT: Duration = Duration::from_secs(10); -/// Capacity of the channel used to communicate between the [`Downloader`] and the [`Service`]. -const SERVICE_CHANNEL_CAPACITY: usize = 128; - -/// Identifier for a download intent. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, derive_more::Display)] -pub struct IntentId(pub u64); - -/// Trait modeling a dialer. This allows for IO-less testing. -pub trait Dialer: Stream)> + Unpin { - /// Type of connections returned by the Dialer. - type Connection: Clone + 'static; - /// Dial a node. - fn queue_dial(&mut self, node_id: NodeId); - /// Get the number of dialing nodes. - fn pending_count(&self) -> usize; - /// Check if a node is being dialed. - fn is_pending(&self, node: NodeId) -> bool; - /// Get the node id of our node. - fn node_id(&self) -> NodeId; -} - -/// Signals what should be done with the request when it fails. -#[derive(Debug)] -pub enum FailureAction { - /// The request was cancelled by us. - AllIntentsDropped, - /// An error occurred that prevents the request from being retried at all. - AbortRequest(anyhow::Error), - /// An error occurred that suggests the node should not be used in general. - DropPeer(anyhow::Error), - /// An error occurred in which neither the node nor the request are at fault. - RetryLater(anyhow::Error), -} - -/// Future of a get request, for the checking stage. -type GetStartFut = BoxedLocal, FailureAction>>; -/// Future of a get request, for the downloading stage. -type GetProceedFut = BoxedLocal; - -/// Trait modelling performing a single request over a connection. This allows for IO-less testing. -pub trait Getter { - /// Type of connections the Getter requires to perform a download. - type Connection: 'static; - /// Type of the intermediary state returned from [`Self::get`] if a connection is needed. - type NeedsConn: NeedsConn; - /// Returns a future that checks the local store if the request is already complete, returning - /// a struct implementing [`NeedsConn`] if we need a network connection to proceed. - fn get( - &mut self, - kind: DownloadKind, - progress_sender: BroadcastProgressSender, - ) -> GetStartFut; -} - -/// Trait modelling the intermediary state when a connection is needed to proceed. -pub trait NeedsConn: std::fmt::Debug + 'static { - /// Proceeds the download with the given connection. - fn proceed(self, conn: C) -> GetProceedFut; -} - -/// Output returned from [`Getter::get`]. -#[derive(Debug)] -pub enum GetOutput { - /// The request is already complete in the local store. - Complete(Stats), - /// The request needs a connection to continue. - NeedsConn(N), -} - -/// Concurrency limits for the [`Downloader`]. -#[derive(Debug)] -pub struct ConcurrencyLimits { - /// Maximum number of requests the service performs concurrently. - pub max_concurrent_requests: usize, - /// Maximum number of requests performed by a single node concurrently. - pub max_concurrent_requests_per_node: usize, - /// Maximum number of open connections the service maintains. - pub max_open_connections: usize, - /// Maximum number of nodes to dial concurrently for a single request. - pub max_concurrent_dials_per_hash: usize, -} - -impl Default for ConcurrencyLimits { - fn default() -> Self { - // these numbers should be checked against a running node and might depend on platform - ConcurrencyLimits { - max_concurrent_requests: 50, - max_concurrent_requests_per_node: 4, - max_open_connections: 25, - max_concurrent_dials_per_hash: 5, - } - } -} - -impl ConcurrencyLimits { - /// Checks if the maximum number of concurrent requests has been reached. - fn at_requests_capacity(&self, active_requests: usize) -> bool { - active_requests >= self.max_concurrent_requests - } - - /// Checks if the maximum number of concurrent requests per node has been reached. - fn node_at_request_capacity(&self, active_node_requests: usize) -> bool { - active_node_requests >= self.max_concurrent_requests_per_node - } - - /// Checks if the maximum number of connections has been reached. - fn at_connections_capacity(&self, active_connections: usize) -> bool { - active_connections >= self.max_open_connections - } - - /// Checks if the maximum number of concurrent dials per hash has been reached. - /// - /// Note that this limit is not strictly enforced, and not checked in - /// [`Service::check_invariants`]. A certain hash can exceed this limit in a valid way if some - /// of its providers are dialed for another hash. However, once the limit is reached, - /// no new dials will be initiated for the hash. - fn at_dials_per_hash_capacity(&self, concurrent_dials: usize) -> bool { - concurrent_dials >= self.max_concurrent_dials_per_hash - } -} - -/// Configuration for retry behavior of the [`Downloader`]. -#[derive(Debug)] -pub struct RetryConfig { - /// Maximum number of retry attempts for a node that failed to dial or failed with IO errors. - pub max_retries_per_node: u32, - /// The initial delay to wait before retrying a node. On subsequent failures, the retry delay - /// will be multiplied with the number of failed retries. - pub initial_retry_delay: Duration, -} - -impl Default for RetryConfig { - fn default() -> Self { - Self { - max_retries_per_node: 6, - initial_retry_delay: Duration::from_millis(500), - } - } -} - -/// A download request. -#[derive(Debug, Clone)] -pub struct DownloadRequest { - kind: DownloadKind, - nodes: Vec, - progress: Option, -} - -impl DownloadRequest { - /// Create a new download request. - /// - /// It is the responsibility of the caller to ensure that the data is tagged either with a - /// temp tag or with a persistent tag to make sure the data is not garbage collected during - /// the download. - /// - /// If this is not done, there download will proceed as normal, but there is no guarantee - /// that the data is still available when the download is complete. - pub fn new( - resource: impl Into, - nodes: impl IntoIterator>, - ) -> Self { - Self { - kind: resource.into(), - nodes: nodes.into_iter().map(|n| n.into()).collect(), - progress: None, - } - } - - /// Pass a progress sender to receive progress updates. - pub fn progress_sender(mut self, sender: ProgressSubscriber) -> Self { - self.progress = Some(sender); - self - } -} - -/// The kind of resource to download. -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, derive_more::From, derive_more::Into)] -pub struct DownloadKind(HashAndFormat); - -impl DownloadKind { - /// Get the hash of this download - pub const fn hash(&self) -> Hash { - self.0.hash - } - - /// Get the format of this download - pub const fn format(&self) -> BlobFormat { - self.0.format - } - - /// Get the [`HashAndFormat`] pair of this download - pub const fn hash_and_format(&self) -> HashAndFormat { - self.0 - } -} - -impl fmt::Display for DownloadKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}:{:?}", self.0.hash.fmt_short(), self.0.format) - } -} - -/// The result of a download request, as returned to the application code. -type ExternalDownloadResult = Result; - -/// The result of a download request, as used in this module. -type InternalDownloadResult = Result; - -/// Error returned when a download could not be completed. -#[derive(Debug, Clone, thiserror::Error)] -pub enum DownloadError { - /// Failed to download from any provider - #[error("Failed to complete download")] - DownloadFailed, - /// The download was cancelled by us - #[error("Download cancelled by us")] - Cancelled, - /// No provider nodes found - #[error("No provider nodes found")] - NoProviders, - /// Failed to receive response from service. - #[error("Failed to receive response from download service")] - ActorClosed, -} - -/// Handle to interact with a download request. -#[derive(Debug)] -pub struct DownloadHandle { - /// Id used to identify the request in the [`Downloader`]. - id: IntentId, - /// Kind of download. - kind: DownloadKind, - /// Receiver to retrieve the return value of this download. - receiver: oneshot::Receiver, -} - -impl Future for DownloadHandle { - type Output = ExternalDownloadResult; - - fn poll( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - use std::task::Poll::*; - // make it easier on holders of the handle to poll the result, removing the receiver error - // from the middle - match std::pin::Pin::new(&mut self.receiver).poll(cx) { - Ready(Ok(result)) => Ready(result), - Ready(Err(_recv_err)) => Ready(Err(DownloadError::ActorClosed)), - Pending => Pending, - } - } -} - -/// Handle for the download services. -#[derive(Clone, Debug)] -pub struct Downloader { - /// Next id to use for a download intent. - next_id: Arc, - /// Channel to communicate with the service. - msg_tx: mpsc::Sender, -} - -impl Downloader { - /// Create a new Downloader with the default [`ConcurrencyLimits`] and [`RetryConfig`]. - pub fn new(store: S, endpoint: Endpoint, rt: LocalPoolHandle) -> Self - where - S: Store, - { - Self::with_config(store, endpoint, rt, Default::default(), Default::default()) - } - - /// Create a new Downloader with custom [`ConcurrencyLimits`] and [`RetryConfig`]. - pub fn with_config( - store: S, - endpoint: Endpoint, - rt: LocalPoolHandle, - concurrency_limits: ConcurrencyLimits, - retry_config: RetryConfig, - ) -> Self - where - S: Store, - { - let me = endpoint.node_id().fmt_short(); - let (msg_tx, msg_rx) = mpsc::channel(SERVICE_CHANNEL_CAPACITY); - let dialer = iroh_net::dialer::Dialer::new(endpoint); - - let create_future = move || { - let getter = get::IoGetter { - store: store.clone(), - }; - - let service = Service::new(getter, dialer, concurrency_limits, retry_config, msg_rx); - - service.run().instrument(error_span!("downloader", %me)) - }; - rt.spawn_detached(create_future); - Self { - next_id: Arc::new(AtomicU64::new(0)), - msg_tx, - } - } - - /// Queue a download. - pub async fn queue(&self, request: DownloadRequest) -> DownloadHandle { - let kind = request.kind; - let intent_id = IntentId(self.next_id.fetch_add(1, Ordering::SeqCst)); - let (sender, receiver) = oneshot::channel(); - let handle = DownloadHandle { - id: intent_id, - kind, - receiver, - }; - let msg = Message::Queue { - on_finish: sender, - request, - intent_id, - }; - // if this fails polling the handle will fail as well since the sender side of the oneshot - // will be dropped - if let Err(send_err) = self.msg_tx.send(msg).await { - let msg = send_err.0; - debug!(?msg, "download not sent"); - } - handle - } - - /// Cancel a download. - // NOTE: receiving the handle ensures an intent can't be cancelled twice - pub async fn cancel(&self, handle: DownloadHandle) { - let DownloadHandle { - id, - kind, - receiver: _, - } = handle; - let msg = Message::CancelIntent { id, kind }; - if let Err(send_err) = self.msg_tx.send(msg).await { - let msg = send_err.0; - debug!(?msg, "cancel not sent"); - } - } - - /// Declare that certain nodes can be used to download a hash. - /// - /// Note that this does not start a download, but only provides new nodes to already queued - /// downloads. Use [`Self::queue`] to queue a download. - pub async fn nodes_have(&mut self, hash: Hash, nodes: Vec) { - let msg = Message::NodesHave { hash, nodes }; - if let Err(send_err) = self.msg_tx.send(msg).await { - let msg = send_err.0; - debug!(?msg, "nodes have not been sent") - } - } -} - -/// Messages the service can receive. -#[derive(derive_more::Debug)] -enum Message { - /// Queue a download intent. - Queue { - request: DownloadRequest, - #[debug(skip)] - on_finish: oneshot::Sender, - intent_id: IntentId, - }, - /// Declare that nodes have a certain hash and can be used for downloading. - NodesHave { hash: Hash, nodes: Vec }, - /// Cancel an intent. The associated request will be cancelled when the last intent is - /// cancelled. - CancelIntent { id: IntentId, kind: DownloadKind }, -} - -#[derive(derive_more::Debug)] -struct IntentHandlers { - #[debug("oneshot::Sender")] - on_finish: oneshot::Sender, - on_progress: Option, -} - -/// Information about a request. -#[derive(Debug)] -struct RequestInfo { - /// Registered intents with progress senders and result callbacks. - intents: HashMap, - progress_sender: BroadcastProgressSender, - get_state: Option, -} - -/// Information about a request in progress. -#[derive(derive_more::Debug)] -struct ActiveRequestInfo { - /// Token used to cancel the future doing the request. - #[debug(skip)] - cancellation: CancellationToken, - /// Peer doing this request attempt. - node: NodeId, -} - -#[derive(Debug, Default)] -struct RetryState { - /// How many times did we retry this node? - retry_count: u32, - /// Whether the node is currently queued for retry. - retry_is_queued: bool, -} - -/// State of the connection to this node. -#[derive(derive_more::Debug)] -struct ConnectionInfo { - /// Connection to this node. - #[debug(skip)] - conn: Conn, - /// State of this node. - state: ConnectedState, -} - -impl ConnectionInfo { - /// Create a new idle node. - fn new_idle(connection: Conn, drop_key: delay_queue::Key) -> Self { - ConnectionInfo { - conn: connection, - state: ConnectedState::Idle { drop_key }, - } - } - - /// Count of active requests for the node. - fn active_requests(&self) -> usize { - match self.state { - ConnectedState::Busy { active_requests } => active_requests.get(), - ConnectedState::Idle { .. } => 0, - } - } - - /// Returns `true` if the node is currently idle. - fn is_idle(&self) -> bool { - matches!(self.state, ConnectedState::Idle { .. }) - } -} - -/// State of a connected node. -#[derive(derive_more::Debug)] -enum ConnectedState { - /// Peer is handling at least one request. - Busy { - #[debug("{}", active_requests.get())] - active_requests: NonZeroUsize, - }, - /// Peer is idle. - Idle { - #[debug(skip)] - drop_key: delay_queue::Key, - }, -} - -#[derive(Debug)] -enum NodeState<'a, Conn> { - Connected(&'a ConnectionInfo), - Dialing, - WaitForRetry, - Disconnected, -} - -#[derive(Debug)] -struct Service { - /// The getter performs individual requests. - getter: G, - /// Map to query for nodes that we believe have the data we are looking for. - providers: ProviderMap, - /// Dialer to get connections for required nodes. - dialer: D, - /// Limits to concurrent tasks handled by the service. - concurrency_limits: ConcurrencyLimits, - /// Configuration for retry behavior. - retry_config: RetryConfig, - /// Channel to receive messages from the service's handle. - msg_rx: mpsc::Receiver, - /// Nodes to which we have an active or idle connection. - connected_nodes: HashMap>, - /// We track a retry state for nodes which failed to dial or in a transfer. - retry_node_state: HashMap, - /// Delay queue for retrying failed nodes. - retry_nodes_queue: delay_queue::DelayQueue, - /// Delay queue for dropping idle nodes. - goodbye_nodes_queue: delay_queue::DelayQueue, - /// Queue of pending downloads. - queue: Queue, - /// Information about pending and active requests. - requests: HashMap>, - /// State of running downloads. - active_requests: HashMap, - /// Tasks for currently running downloads. - in_progress_downloads: JoinSet<(DownloadKind, InternalDownloadResult)>, - /// Progress tracker - progress_tracker: ProgressTracker, -} -impl, D: Dialer> Service { - fn new( - getter: G, - dialer: D, - concurrency_limits: ConcurrencyLimits, - retry_config: RetryConfig, - msg_rx: mpsc::Receiver, - ) -> Self { - Service { - getter, - dialer, - msg_rx, - concurrency_limits, - retry_config, - connected_nodes: Default::default(), - retry_node_state: Default::default(), - providers: Default::default(), - requests: Default::default(), - retry_nodes_queue: delay_queue::DelayQueue::default(), - goodbye_nodes_queue: delay_queue::DelayQueue::default(), - active_requests: Default::default(), - in_progress_downloads: Default::default(), - progress_tracker: ProgressTracker::new(), - queue: Default::default(), - } - } - - /// Main loop for the service. - async fn run(mut self) { - loop { - trace!("wait for tick"); - inc!(Metrics, downloader_tick_main); - tokio::select! { - Some((node, conn_result)) = self.dialer.next() => { - trace!(node=%node.fmt_short(), "tick: connection ready"); - inc!(Metrics, downloader_tick_connection_ready); - self.on_connection_ready(node, conn_result); - } - maybe_msg = self.msg_rx.recv() => { - trace!(msg=?maybe_msg, "tick: message received"); - inc!(Metrics, downloader_tick_message_received); - match maybe_msg { - Some(msg) => self.handle_message(msg).await, - None => return self.shutdown().await, - } - } - Some(res) = self.in_progress_downloads.join_next(), if !self.in_progress_downloads.is_empty() => { - match res { - Ok((kind, result)) => { - trace!(%kind, "tick: transfer completed"); - inc!(Metrics, downloader_tick_transfer_completed); - self.on_download_completed(kind, result); - } - Err(err) => { - warn!(?err, "transfer task panicked"); - inc!(Metrics, downloader_tick_transfer_failed); - } - } - } - Some(expired) = self.retry_nodes_queue.next() => { - let node = expired.into_inner(); - trace!(node=%node.fmt_short(), "tick: retry node"); - inc!(Metrics, downloader_tick_retry_node); - self.on_retry_wait_elapsed(node); - } - Some(expired) = self.goodbye_nodes_queue.next() => { - let node = expired.into_inner(); - trace!(node=%node.fmt_short(), "tick: goodbye node"); - inc!(Metrics, downloader_tick_goodbye_node); - self.disconnect_idle_node(node, "idle expired"); - } - } - - self.process_head(); - - #[cfg(any(test, debug_assertions))] - self.check_invariants(); - } - } - - /// Handle receiving a [`Message`]. - /// - // This is called in the actor loop, and only async because subscribing to an existing transfer - // sends the initial state. - async fn handle_message(&mut self, msg: Message) { - match msg { - Message::Queue { - request, - on_finish, - intent_id, - } => { - self.handle_queue_new_download(request, intent_id, on_finish) - .await - } - Message::CancelIntent { id, kind } => self.handle_cancel_download(id, kind).await, - Message::NodesHave { hash, nodes } => { - let updated = self - .providers - .add_nodes_if_hash_exists(hash, nodes.iter().cloned()); - if updated { - self.queue.unpark_hash(hash); - } - } - } - } - - /// Handle a [`Message::Queue`]. - /// - /// If this intent maps to a request that already exists, it will be registered with it. If the - /// request is new it will be scheduled. - async fn handle_queue_new_download( - &mut self, - request: DownloadRequest, - intent_id: IntentId, - on_finish: oneshot::Sender, - ) { - let DownloadRequest { - kind, - nodes, - progress, - } = request; - debug!(%kind, nodes=?nodes.iter().map(|n| n.node_id.fmt_short()).collect::>(), "queue intent"); - - // store the download intent - let intent_handlers = IntentHandlers { - on_finish, - on_progress: progress, - }; - - // add the nodes to the provider map - // (skip the node id of our own node - we should never attempt to download from ourselves) - let node_ids = nodes - .iter() - .map(|n| n.node_id) - .filter(|node_id| *node_id != self.dialer.node_id()); - let updated = self.providers.add_hash_with_nodes(kind.hash(), node_ids); - - // queue the transfer (if not running) or attach to transfer progress (if already running) - match self.requests.entry(kind) { - hash_map::Entry::Occupied(mut entry) => { - if let Some(on_progress) = &intent_handlers.on_progress { - // this is async because it sends the current state over the progress channel - if let Err(err) = self - .progress_tracker - .subscribe(kind, on_progress.clone()) - .await - { - debug!(?err, %kind, "failed to subscribe progress sender to transfer"); - } - } - entry.get_mut().intents.insert(intent_id, intent_handlers); - } - hash_map::Entry::Vacant(entry) => { - let progress_sender = self.progress_tracker.track( - kind, - intent_handlers - .on_progress - .clone() - .into_iter() - .collect::>(), - ); - - let get_state = match self.getter.get(kind, progress_sender.clone()).await { - Err(err) => { - // This prints a "FailureAction" which is somewhat weird, but that's all we get here. - tracing::error!(?err, "failed queuing new download"); - self.finalize_download( - kind, - [(intent_id, intent_handlers)].into(), - // TODO: add better error variant? this is only triggered if the local - // store failed with local IO. - Err(DownloadError::DownloadFailed), - ); - return; - } - Ok(GetOutput::Complete(stats)) => { - self.finalize_download( - kind, - [(intent_id, intent_handlers)].into(), - Ok(stats), - ); - return; - } - Ok(GetOutput::NeedsConn(state)) => { - // early exit if no providers. - if self.providers.get_candidates(&kind.hash()).next().is_none() { - self.finalize_download( - kind, - [(intent_id, intent_handlers)].into(), - Err(DownloadError::NoProviders), - ); - return; - } - state - } - }; - entry.insert(RequestInfo { - intents: [(intent_id, intent_handlers)].into_iter().collect(), - progress_sender, - get_state: Some(get_state), - }); - self.queue.insert(kind); - } - } - - if updated && self.queue.is_parked(&kind) { - // the transfer is on hold for pending retries, and we added new nodes, so move back to queue. - self.queue.unpark(&kind); - } - } - - /// Cancels a download intent. - /// - /// This removes the intent from the list of intents for the `kind`. If the removed intent was - /// the last one for the `kind`, this means that the download is no longer needed. In this - /// case, the `kind` will be removed from the list of pending downloads - and, if the download was - /// already started, the download task will be cancelled. - /// - /// The method is async because it will send a final abort event on the progress sender. - async fn handle_cancel_download(&mut self, intent_id: IntentId, kind: DownloadKind) { - let Entry::Occupied(mut occupied_entry) = self.requests.entry(kind) else { - warn!(%kind, %intent_id, "cancel download called for unknown download"); - return; - }; - - let request_info = occupied_entry.get_mut(); - if let Some(handlers) = request_info.intents.remove(&intent_id) { - handlers.on_finish.send(Err(DownloadError::Cancelled)).ok(); - - if let Some(sender) = handlers.on_progress { - self.progress_tracker.unsubscribe(&kind, &sender); - sender - .send(DownloadProgress::Abort( - anyhow::Error::from(DownloadError::Cancelled).into(), - )) - .await - .ok(); - } - } - - if request_info.intents.is_empty() { - occupied_entry.remove(); - if let Entry::Occupied(occupied_entry) = self.active_requests.entry(kind) { - occupied_entry.remove().cancellation.cancel(); - } else { - self.queue.remove(&kind); - } - self.remove_hash_if_not_queued(&kind.hash()); - } - } - - /// Handle receiving a new connection. - fn on_connection_ready(&mut self, node: NodeId, result: anyhow::Result) { - debug_assert!( - !self.connected_nodes.contains_key(&node), - "newly connected node is not yet connected" - ); - match result { - Ok(connection) => { - trace!(node=%node.fmt_short(), "connected to node"); - let drop_key = self.goodbye_nodes_queue.insert(node, IDLE_PEER_TIMEOUT); - self.connected_nodes - .insert(node, ConnectionInfo::new_idle(connection, drop_key)); - } - Err(err) => { - debug!(%node, %err, "connection to node failed"); - self.disconnect_and_retry(node); - } - } - } - - fn on_download_completed(&mut self, kind: DownloadKind, result: InternalDownloadResult) { - // first remove the request - let active_request_info = self - .active_requests - .remove(&kind) - .expect("request was active"); - - // get general request info - let request_info = self.requests.remove(&kind).expect("request was active"); - - let ActiveRequestInfo { node, .. } = active_request_info; - - // get node info - let node_info = self - .connected_nodes - .get_mut(&node) - .expect("node exists in the mapping"); - - // update node busy/idle state - node_info.state = match NonZeroUsize::new(node_info.active_requests() - 1) { - None => { - // last request of the node was this one, switch to idle - let drop_key = self.goodbye_nodes_queue.insert(node, IDLE_PEER_TIMEOUT); - ConnectedState::Idle { drop_key } - } - Some(active_requests) => ConnectedState::Busy { active_requests }, - }; - - match &result { - Ok(_) => { - debug!(%kind, node=%node.fmt_short(), "download successful"); - // clear retry state if operation was successful - self.retry_node_state.remove(&node); - } - Err(FailureAction::AllIntentsDropped) => { - debug!(%kind, node=%node.fmt_short(), "download cancelled"); - } - Err(FailureAction::AbortRequest(reason)) => { - debug!(%kind, node=%node.fmt_short(), %reason, "download failed: abort request"); - // do not try to download the hash from this node again - self.providers.remove_hash_from_node(&kind.hash(), &node); - } - Err(FailureAction::DropPeer(reason)) => { - debug!(%kind, node=%node.fmt_short(), %reason, "download failed: drop node"); - if node_info.is_idle() { - // remove the node - self.remove_node(node, "explicit drop"); - } else { - // do not try to download the hash from this node again - self.providers.remove_hash_from_node(&kind.hash(), &node); - } - } - Err(FailureAction::RetryLater(reason)) => { - debug!(%kind, node=%node.fmt_short(), %reason, "download failed: retry later"); - if node_info.is_idle() { - self.disconnect_and_retry(node); - } - } - }; - - // we finalize the download if either the download was successful, - // or if it should never proceed because all intents were dropped, - // or if we don't have any candidates to proceed with anymore. - let finalize = match &result { - Ok(_) | Err(FailureAction::AllIntentsDropped) => true, - _ => !self.providers.has_candidates(&kind.hash()), - }; - - if finalize { - let result = result.map_err(|_| DownloadError::DownloadFailed); - self.finalize_download(kind, request_info.intents, result); - } else { - // reinsert the download at the front of the queue to try from the next node - self.requests.insert(kind, request_info); - self.queue.insert_front(kind); - } - } - - /// Finalize a download. - /// - /// This triggers the intent return channels, and removes the download from the progress tracker - /// and provider map. - fn finalize_download( - &mut self, - kind: DownloadKind, - intents: HashMap, - result: ExternalDownloadResult, - ) { - self.progress_tracker.remove(&kind); - self.remove_hash_if_not_queued(&kind.hash()); - for (_id, handlers) in intents.into_iter() { - handlers.on_finish.send(result.clone()).ok(); - } - } - - fn on_retry_wait_elapsed(&mut self, node: NodeId) { - // check if the node is still needed - let Some(hashes) = self.providers.node_hash.get(&node) else { - self.retry_node_state.remove(&node); - return; - }; - let Some(state) = self.retry_node_state.get_mut(&node) else { - warn!(node=%node.fmt_short(), "missing retry state for node ready for retry"); - return; - }; - state.retry_is_queued = false; - for hash in hashes { - self.queue.unpark_hash(*hash); - } - } - - /// Start the next downloads, or dial nodes, if limits permit and the queue is non-empty. - /// - /// This is called after all actions. If there is nothing to do, it will return cheaply. - /// Otherwise, we will check the next hash in the queue, and: - /// * start the transfer if we are connected to a provider and limits are ok - /// * or, connect to a provider, if there is one we are not dialing yet and limits are ok - /// * or, disconnect an idle node if it would allow us to connect to a provider, - /// * or, if all providers are waiting for retry, park the download - /// * or, if our limits are reached, do nothing for now - /// - /// The download requests will only be popped from the queue once we either start the transfer - /// from a connected node [`NextStep::StartTransfer`], or if we abort the download on - /// [`NextStep::OutOfProviders`]. In all other cases, the request is kept at the top of the - /// queue, so the next call to [`Self::process_head`] will evaluate the situation again - and - /// so forth, until either [`NextStep::StartTransfer`] or [`NextStep::OutOfProviders`] is - /// reached. - fn process_head(&mut self) { - // start as many queued downloads as allowed by the request limits. - loop { - let Some(kind) = self.queue.front().cloned() else { - break; - }; - - let next_step = self.next_step(&kind); - trace!(%kind, ?next_step, "process_head"); - - match next_step { - NextStep::Wait => break, - NextStep::StartTransfer(node) => { - let _ = self.queue.pop_front(); - debug!(%kind, node=%node.fmt_short(), "start transfer"); - self.start_download(kind, node); - } - NextStep::Dial(node) => { - debug!(%kind, node=%node.fmt_short(), "dial node"); - self.dialer.queue_dial(node); - } - NextStep::DialQueuedDisconnect(node, key) => { - let idle_node = self.goodbye_nodes_queue.remove(&key).into_inner(); - self.disconnect_idle_node(idle_node, "drop idle for new dial"); - debug!(%kind, node=%node.fmt_short(), idle_node=%idle_node.fmt_short(), "dial node, disconnect idle node)"); - self.dialer.queue_dial(node); - } - NextStep::Park => { - debug!(%kind, "park download: all providers waiting for retry"); - self.queue.park_front(); - } - NextStep::OutOfProviders => { - debug!(%kind, "abort download: out of providers"); - let _ = self.queue.pop_front(); - let info = self.requests.remove(&kind).expect("queued downloads exist"); - self.finalize_download(kind, info.intents, Err(DownloadError::NoProviders)); - } - } - } - } - - /// Drop the connection to a node and insert it into the the retry queue. - fn disconnect_and_retry(&mut self, node: NodeId) { - self.disconnect_idle_node(node, "queue retry"); - let retry_state = self.retry_node_state.entry(node).or_default(); - retry_state.retry_count += 1; - if retry_state.retry_count <= self.retry_config.max_retries_per_node { - // node can be retried - debug!(node=%node.fmt_short(), retry_count=retry_state.retry_count, "queue retry"); - let timeout = self.retry_config.initial_retry_delay * retry_state.retry_count; - self.retry_nodes_queue.insert(node, timeout); - retry_state.retry_is_queued = true; - } else { - // node is dead - self.remove_node(node, "retries exceeded"); - } - } - - /// Calculate the next step needed to proceed the download for `kind`. - /// - /// This is called once `kind` has reached the head of the queue, see [`Self::process_head`]. - /// It can be called repeatedly, and does nothing on itself, only calculate what *should* be - /// done next. - /// - /// See [`NextStep`] for details on the potential next steps returned from this method. - fn next_step(&self, kind: &DownloadKind) -> NextStep { - // If the total requests capacity is reached, we have to wait until an active request - // completes. - if self - .concurrency_limits - .at_requests_capacity(self.active_requests.len()) - { - return NextStep::Wait; - }; - - let mut candidates = self.providers.get_candidates(&kind.hash()).peekable(); - // If we have no provider candidates for this download, there's nothing else we can do. - if candidates.peek().is_none() { - return NextStep::OutOfProviders; - } - - // Track if there is provider node to which we are connected and which is not at its request capacity. - // If there are more than one, take the one with the least amount of running transfers. - let mut best_connected: Option<(NodeId, usize)> = None; - // Track if there is a disconnected provider node to which we can potentially connect. - let mut next_to_dial = None; - // Track the number of provider nodes that are currently being dialed. - let mut currently_dialing = 0; - // Track if we have at least one provider node which is currently at its request capacity. - // If this is the case, we will never return [`NextStep::OutOfProviders`] but [`NextStep::Wait`] - // instead, because we can still try that node once it has finished its work. - let mut has_exhausted_provider = false; - // Track if we have at least one provider node that is currently in the retry queue. - let mut has_retrying_provider = false; - - for node in candidates { - match self.node_state(node) { - NodeState::Connected(info) => { - let active_requests = info.active_requests(); - if self - .concurrency_limits - .node_at_request_capacity(active_requests) - { - has_exhausted_provider = true; - } else { - best_connected = Some(match best_connected.take() { - Some(old) if old.1 <= active_requests => old, - _ => (node, active_requests), - }); - } - } - NodeState::Dialing => { - currently_dialing += 1; - } - NodeState::WaitForRetry => { - has_retrying_provider = true; - } - NodeState::Disconnected => { - if next_to_dial.is_none() { - next_to_dial = Some(node); - } - } - } - } - - let has_dialing = currently_dialing > 0; - - // If we have a connected provider node with free slots, use it! - if let Some((node, _active_requests)) = best_connected { - NextStep::StartTransfer(node) - } - // If we have a node which could be dialed: Check capacity and act accordingly. - else if let Some(node) = next_to_dial { - // We check if the dial capacity for this hash is exceeded: We only start new dials for - // the hash if we are below the limit. - // - // If other requests trigger dials for providers of this hash, the limit may be - // exceeded, but then we just don't start further dials and wait until one completes. - let at_dial_capacity = has_dialing - && self - .concurrency_limits - .at_dials_per_hash_capacity(currently_dialing); - // Check if we reached the global connection limit. - let at_connections_capacity = self.at_connections_capacity(); - - // All slots are free: We can dial our candidate. - if !at_connections_capacity && !at_dial_capacity { - NextStep::Dial(node) - } - // The hash has free dial capacity, but the global connection capacity is reached. - // But if we have idle nodes, we will disconnect the longest idling node, and then dial our - // candidate. - else if at_connections_capacity - && !at_dial_capacity - && !self.goodbye_nodes_queue.is_empty() - { - let key = self.goodbye_nodes_queue.peek().expect("just checked"); - NextStep::DialQueuedDisconnect(node, key) - } - // No dial capacity, and no idling nodes: We have to wait until capacity is freed up. - else { - NextStep::Wait - } - } - // If we have pending dials to candidates, or connected candidates which are busy - // with other work: Wait for one of these to become available. - else if has_exhausted_provider || has_dialing { - NextStep::Wait - } - // All providers are in the retry queue: Park this request until they can be tried again. - else if has_retrying_provider { - NextStep::Park - } - // We have no candidates left: Nothing more to do. - else { - NextStep::OutOfProviders - } - } - - /// Start downloading from the given node. - /// - /// Panics if hash is not in self.requests or node is not in self.nodes. - fn start_download(&mut self, kind: DownloadKind, node: NodeId) { - let node_info = self.connected_nodes.get_mut(&node).expect("node exists"); - let request_info = self.requests.get_mut(&kind).expect("request exists"); - let progress = request_info.progress_sender.clone(); - // .expect("queued state exists"); - - // create the active request state - let cancellation = CancellationToken::new(); - let state = ActiveRequestInfo { - cancellation: cancellation.clone(), - node, - }; - let conn = node_info.conn.clone(); - - // If this is the first provider node we try, we have an initial state - // from starting the generator in Self::handle_queue_new_download. - // If this not the first provider node we try, we have to recreate the generator, because - // we can only resume it once. - let get_state = match request_info.get_state.take() { - Some(state) => Either::Left(async move { Ok(GetOutput::NeedsConn(state)) }), - None => Either::Right(self.getter.get(kind, progress)), - }; - let fut = async move { - // NOTE: it's an open question if we should do timeouts at this point. Considerations from @Frando: - // > at this stage we do not know the size of the download, so the timeout would have - // > to be so large that it won't be useful for non-huge downloads. At the same time, - // > this means that a super slow node would block a download from succeeding for a long - // > time, while faster nodes could be readily available. - // As a conclusion, timeouts should be added only after downloads are known to be bounded - let fut = async move { - match get_state.await? { - GetOutput::Complete(stats) => Ok(stats), - GetOutput::NeedsConn(state) => state.proceed(conn).await, - } - }; - tokio::pin!(fut); - let res = tokio::select! { - _ = cancellation.cancelled() => Err(FailureAction::AllIntentsDropped), - res = &mut fut => res - }; - trace!("transfer finished"); - - (kind, res) - } - .instrument(error_span!("transfer", %kind, node=%node.fmt_short())); - node_info.state = match &node_info.state { - ConnectedState::Busy { active_requests } => ConnectedState::Busy { - active_requests: active_requests.saturating_add(1), - }, - ConnectedState::Idle { drop_key } => { - self.goodbye_nodes_queue.remove(drop_key); - ConnectedState::Busy { - active_requests: NonZeroUsize::new(1).expect("clearly non zero"), - } - } - }; - self.active_requests.insert(kind, state); - self.in_progress_downloads.spawn_local(fut); - } - - fn disconnect_idle_node(&mut self, node: NodeId, reason: &'static str) -> bool { - if let Some(info) = self.connected_nodes.remove(&node) { - match info.state { - ConnectedState::Idle { drop_key } => { - self.goodbye_nodes_queue.try_remove(&drop_key); - true - } - ConnectedState::Busy { .. } => { - warn!("expected removed node to be idle, but is busy (removal reason: {reason:?})"); - self.connected_nodes.insert(node, info); - false - } - } - } else { - true - } - } - - fn remove_node(&mut self, node: NodeId, reason: &'static str) { - debug!(node = %node.fmt_short(), %reason, "remove node"); - if self.disconnect_idle_node(node, reason) { - self.providers.remove_node(&node); - self.retry_node_state.remove(&node); - } - } - - fn node_state(&self, node: NodeId) -> NodeState<'_, D::Connection> { - if let Some(info) = self.connected_nodes.get(&node) { - NodeState::Connected(info) - } else if self.dialer.is_pending(node) { - NodeState::Dialing - } else { - match self.retry_node_state.get(&node) { - Some(state) if state.retry_is_queued => NodeState::WaitForRetry, - _ => NodeState::Disconnected, - } - } - } - - /// Check if we have maxed our connection capacity. - fn at_connections_capacity(&self) -> bool { - self.concurrency_limits - .at_connections_capacity(self.connections_count()) - } - - /// Get the total number of connected and dialing nodes. - fn connections_count(&self) -> usize { - let connected_nodes = self.connected_nodes.values().count(); - let dialing_nodes = self.dialer.pending_count(); - connected_nodes + dialing_nodes - } - - /// Remove a `hash` from the [`ProviderMap`], but only if [`Self::queue`] does not contain the - /// hash at all, even with the other [`BlobFormat`]. - fn remove_hash_if_not_queued(&mut self, hash: &Hash) { - if !self.queue.contains_hash(*hash) { - self.providers.remove_hash(hash); - } - } - - #[allow(clippy::unused_async)] - async fn shutdown(self) { - debug!("shutting down"); - // TODO(@divma): how to make sure the download futures end gracefully? - } -} - -/// The next step needed to continue a download. -/// -/// See [`Service::next_step`] for details. -#[derive(Debug)] -enum NextStep { - /// Provider connection is ready, initiate the transfer. - StartTransfer(NodeId), - /// Start to dial `NodeId`. - /// - /// This means: We have no non-exhausted connection to a provider node, but a free connection slot - /// and a provider node we are not yet connected to. - Dial(NodeId), - /// Start to dial `NodeId`, but first disconnect the idle node behind [`delay_queue::Key`] in - /// [`Service::goodbye_nodes_queue`] to free up a connection slot. - DialQueuedDisconnect(NodeId, delay_queue::Key), - /// Resource limits are exhausted, do nothing for now and wait until a slot frees up. - Wait, - /// All providers are currently in a retry timeout. Park the download aside, and move - /// to the next download in the queue. - Park, - /// We have tried all available providers. There is nothing else to do. - OutOfProviders, -} - -/// Map of potential providers for a hash. -#[derive(Default, Debug)] -struct ProviderMap { - hash_node: HashMap>, - node_hash: HashMap>, -} - -impl ProviderMap { - /// Get candidates to download this hash. - pub fn get_candidates<'a>(&'a self, hash: &Hash) -> impl Iterator + 'a { - self.hash_node - .get(hash) - .map(|nodes| nodes.iter()) - .into_iter() - .flatten() - .copied() - } - - /// Whether we have any candidates to download this hash. - pub fn has_candidates(&self, hash: &Hash) -> bool { - self.hash_node - .get(hash) - .map(|nodes| !nodes.is_empty()) - .unwrap_or(false) - } - - /// Register nodes for a hash. Should only be done for hashes we care to download. - /// - /// Returns `true` if new providers were added. - fn add_hash_with_nodes(&mut self, hash: Hash, nodes: impl Iterator) -> bool { - let mut updated = false; - let hash_entry = self.hash_node.entry(hash).or_default(); - for node in nodes { - updated |= hash_entry.insert(node); - let node_entry = self.node_hash.entry(node).or_default(); - node_entry.insert(hash); - } - updated - } - - /// Register nodes for a hash, but only if the hash is already in our queue. - /// - /// Returns `true` if a new node was added. - fn add_nodes_if_hash_exists( - &mut self, - hash: Hash, - nodes: impl Iterator, - ) -> bool { - let mut updated = false; - if let Some(hash_entry) = self.hash_node.get_mut(&hash) { - for node in nodes { - updated |= hash_entry.insert(node); - let node_entry = self.node_hash.entry(node).or_default(); - node_entry.insert(hash); - } - } - updated - } - - /// Signal the registry that this hash is no longer of interest. - fn remove_hash(&mut self, hash: &Hash) { - if let Some(nodes) = self.hash_node.remove(hash) { - for node in nodes { - if let Some(hashes) = self.node_hash.get_mut(&node) { - hashes.remove(hash); - if hashes.is_empty() { - self.node_hash.remove(&node); - } - } - } - } - } - - fn remove_node(&mut self, node: &NodeId) { - if let Some(hashes) = self.node_hash.remove(node) { - for hash in hashes { - if let Some(nodes) = self.hash_node.get_mut(&hash) { - nodes.remove(node); - if nodes.is_empty() { - self.hash_node.remove(&hash); - } - } - } - } - } - - fn remove_hash_from_node(&mut self, hash: &Hash, node: &NodeId) { - if let Some(nodes) = self.hash_node.get_mut(hash) { - nodes.remove(node); - if nodes.is_empty() { - self.remove_hash(hash); - } - } - if let Some(hashes) = self.node_hash.get_mut(node) { - hashes.remove(hash); - if hashes.is_empty() { - self.remove_node(node); - } - } - } -} - -/// The queue of requested downloads. -/// -/// This manages two datastructures: -/// * The main queue, a FIFO queue where each item can only appear once. -/// New downloads are pushed to the back of the queue, and the next download to process is popped -/// from the front. -/// * The parked set, a hash set. Items can be moved from the main queue into the parked set. -/// Parked items will not be popped unless they are moved back into the main queue. -#[derive(Debug, Default)] -struct Queue { - main: LinkedHashSet, - parked: HashSet, -} - -impl Queue { - /// Peek at the front element of the main queue. - pub fn front(&self) -> Option<&DownloadKind> { - self.main.front() - } - - #[cfg(any(test, debug_assertions))] - pub fn iter_parked(&self) -> impl Iterator { - self.parked.iter() - } - - #[cfg(any(test, debug_assertions))] - pub fn iter(&self) -> impl Iterator { - self.main.iter().chain(self.parked.iter()) - } - - /// Returns `true` if either the main queue or the parked set contain a download. - pub fn contains(&self, kind: &DownloadKind) -> bool { - self.main.contains(kind) || self.parked.contains(kind) - } - - /// Returns `true` if either the main queue or the parked set contain a download for a hash. - pub fn contains_hash(&self, hash: Hash) -> bool { - let as_raw = HashAndFormat::raw(hash).into(); - let as_hash_seq = HashAndFormat::hash_seq(hash).into(); - self.contains(&as_raw) || self.contains(&as_hash_seq) - } - - /// Returns `true` if a download is in the parked set. - pub fn is_parked(&self, kind: &DownloadKind) -> bool { - self.parked.contains(kind) - } - - /// Insert an element at the back of the main queue. - pub fn insert(&mut self, kind: DownloadKind) { - if !self.main.contains(&kind) { - self.main.insert(kind); - } - } - - /// Insert an element at the front of the main queue. - pub fn insert_front(&mut self, kind: DownloadKind) { - if !self.main.contains(&kind) { - self.main.insert(kind); - } - self.main.to_front(&kind); - } - - /// Dequeue the first download of the main queue. - pub fn pop_front(&mut self) -> Option { - self.main.pop_front() - } - - /// Move the front item of the main queue into the parked set. - pub fn park_front(&mut self) { - if let Some(item) = self.pop_front() { - self.parked.insert(item); - } - } - - /// Move a download from the parked set to the front of the main queue. - pub fn unpark(&mut self, kind: &DownloadKind) { - if self.parked.remove(kind) { - self.main.insert(*kind); - self.main.to_front(kind); - } - } - - /// Move any download for a hash from the parked set to the main queue. - pub fn unpark_hash(&mut self, hash: Hash) { - let as_raw = HashAndFormat::raw(hash).into(); - let as_hash_seq = HashAndFormat::hash_seq(hash).into(); - self.unpark(&as_raw); - self.unpark(&as_hash_seq); - } - - /// Remove a download from both the main queue and the parked set. - pub fn remove(&mut self, kind: &DownloadKind) -> bool { - self.main.remove(kind) || self.parked.remove(kind) - } -} - -impl Dialer for iroh_net::dialer::Dialer { - type Connection = endpoint::Connection; - - fn queue_dial(&mut self, node_id: NodeId) { - self.queue_dial(node_id, crate::protocol::ALPN) - } - - fn pending_count(&self) -> usize { - self.pending_count() - } - - fn is_pending(&self, node: NodeId) -> bool { - self.is_pending(node) - } - - fn node_id(&self) -> NodeId { - self.endpoint().node_id() - } -} diff --git a/iroh-blobs/src/downloader/get.rs b/iroh-blobs/src/downloader/get.rs deleted file mode 100644 index 1be8cd39a15..00000000000 --- a/iroh-blobs/src/downloader/get.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! [`Getter`] implementation that performs requests over [`Connection`]s. -//! -//! [`Connection`]: iroh_net::endpoint::Connection - -use futures_lite::FutureExt; -use iroh_net::endpoint; - -use super::{progress::BroadcastProgressSender, DownloadKind, FailureAction, GetStartFut, Getter}; -use crate::{ - get::{db::get_to_db_in_steps, error::GetError}, - store::Store, -}; - -impl From for FailureAction { - fn from(e: GetError) -> Self { - match e { - e @ GetError::NotFound(_) => FailureAction::AbortRequest(e.into()), - e @ GetError::RemoteReset(_) => FailureAction::RetryLater(e.into()), - e @ GetError::NoncompliantNode(_) => FailureAction::DropPeer(e.into()), - e @ GetError::Io(_) => FailureAction::RetryLater(e.into()), - e @ GetError::BadRequest(_) => FailureAction::AbortRequest(e.into()), - // TODO: what do we want to do on local failures? - e @ GetError::LocalFailure(_) => FailureAction::AbortRequest(e.into()), - } - } -} - -/// [`Getter`] implementation that performs requests over [`Connection`]s. -/// -/// [`Connection`]: iroh_net::endpoint::Connection -pub(crate) struct IoGetter { - pub store: S, -} - -impl Getter for IoGetter { - type Connection = endpoint::Connection; - type NeedsConn = crate::get::db::GetStateNeedsConn; - - fn get( - &mut self, - kind: DownloadKind, - progress_sender: BroadcastProgressSender, - ) -> GetStartFut { - let store = self.store.clone(); - async move { - match get_to_db_in_steps(store, kind.hash_and_format(), progress_sender).await { - Err(err) => Err(err.into()), - Ok(crate::get::db::GetState::Complete(stats)) => { - Ok(super::GetOutput::Complete(stats)) - } - Ok(crate::get::db::GetState::NeedsConn(needs_conn)) => { - Ok(super::GetOutput::NeedsConn(needs_conn)) - } - } - } - .boxed_local() - } -} - -impl super::NeedsConn for crate::get::db::GetStateNeedsConn { - fn proceed(self, conn: endpoint::Connection) -> super::GetProceedFut { - async move { - let res = self.proceed(conn).await; - #[cfg(feature = "metrics")] - track_metrics(&res); - match res { - Ok(stats) => Ok(stats), - Err(err) => Err(err.into()), - } - } - .boxed_local() - } -} - -#[cfg(feature = "metrics")] -fn track_metrics(res: &Result) { - use iroh_metrics::{inc, inc_by}; - - use crate::metrics::Metrics; - match res { - Ok(stats) => { - let crate::get::Stats { - bytes_written, - bytes_read: _, - elapsed, - } = stats; - - inc!(Metrics, downloads_success); - inc_by!(Metrics, download_bytes_total, *bytes_written); - inc_by!(Metrics, download_time_total, elapsed.as_millis() as u64); - } - Err(e) => match &e { - GetError::NotFound(_) => inc!(Metrics, downloads_notfound), - _ => inc!(Metrics, downloads_error), - }, - } -} diff --git a/iroh-blobs/src/downloader/invariants.rs b/iroh-blobs/src/downloader/invariants.rs deleted file mode 100644 index 0409e3d9225..00000000000 --- a/iroh-blobs/src/downloader/invariants.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Invariants for the service. - -#![cfg(any(test, debug_assertions))] - -use super::*; - -/// invariants for the service. -impl, D: Dialer> Service { - /// Checks the various invariants the service must maintain - #[track_caller] - pub(in crate::downloader) fn check_invariants(&self) { - self.check_active_request_count(); - self.check_queued_requests_consistency(); - self.check_idle_peer_consistency(); - self.check_concurrency_limits(); - self.check_provider_map_prunning(); - } - - /// Checks concurrency limits are maintained. - #[track_caller] - fn check_concurrency_limits(&self) { - let ConcurrencyLimits { - max_concurrent_requests, - max_concurrent_requests_per_node, - max_open_connections, - max_concurrent_dials_per_hash, - } = &self.concurrency_limits; - - // check the total number of active requests to ensure it stays within the limit - assert!( - self.in_progress_downloads.len() <= *max_concurrent_requests, - "max_concurrent_requests exceeded" - ); - - // check that the open and dialing peers don't exceed the connection capacity - tracing::trace!( - "limits: conns: {}/{} | reqs: {}/{}", - self.connections_count(), - max_open_connections, - self.in_progress_downloads.len(), - max_concurrent_requests - ); - assert!( - self.connections_count() <= *max_open_connections, - "max_open_connections exceeded" - ); - - // check the active requests per peer don't exceed the limit - for (node, info) in self.connected_nodes.iter() { - assert!( - info.active_requests() <= *max_concurrent_requests_per_node, - "max_concurrent_requests_per_node exceeded for {node}" - ) - } - - // check that we do not dial more nodes than allowed for the next pending hashes - if let Some(kind) = self.queue.front() { - let hash = kind.hash(); - let nodes = self.providers.get_candidates(&hash); - let mut dialing = 0; - for node in nodes { - if self.dialer.is_pending(node) { - dialing += 1; - } - } - assert!( - dialing <= *max_concurrent_dials_per_hash, - "max_concurrent_dials_per_hash exceeded for {hash}" - ) - } - } - - /// Checks that the count of active requests per peer is consistent with the active requests, - /// and that active request are consistent with download futures - #[track_caller] - fn check_active_request_count(&self) { - // check that the count of futures we are polling for downloads is consistent with the - // number of requests - assert_eq!( - self.active_requests.len(), - self.in_progress_downloads.len(), - "active_requests and in_progress_downloads are out of sync" - ); - // check that the count of requests per peer matches the number of requests that have that - // peer as active - let mut real_count: HashMap = - HashMap::with_capacity(self.connected_nodes.len()); - for req_info in self.active_requests.values() { - // nothing like some classic word count - *real_count.entry(req_info.node).or_default() += 1; - } - for (peer, info) in self.connected_nodes.iter() { - assert_eq!( - info.active_requests(), - real_count.get(peer).copied().unwrap_or_default(), - "mismatched count of active requests for {peer}" - ) - } - } - - /// Checks that the queued requests all appear in the provider map and request map. - #[track_caller] - fn check_queued_requests_consistency(&self) { - // check that all hashes in the queue have candidates - for entry in self.queue.iter() { - assert!( - self.providers - .get_candidates(&entry.hash()) - .next() - .is_some(), - "all queued requests have providers" - ); - assert!( - self.requests.contains_key(entry), - "all queued requests have request info" - ); - } - - // check that all parked hashes should be parked - for entry in self.queue.iter_parked() { - assert!( - matches!(self.next_step(entry), NextStep::Park), - "all parked downloads evaluate to the correct next step" - ); - assert!( - self.providers - .get_candidates(&entry.hash()) - .all(|node| matches!(self.node_state(node), NodeState::WaitForRetry)), - "all parked downloads have only retrying nodes" - ); - } - } - - /// Check that peers queued to be disconnected are consistent with peers considered idle. - #[track_caller] - fn check_idle_peer_consistency(&self) { - let idle_peers = self - .connected_nodes - .values() - .filter(|info| info.active_requests() == 0) - .count(); - assert_eq!( - self.goodbye_nodes_queue.len(), - idle_peers, - "inconsistent count of idle peers" - ); - } - - /// Check that every hash in the provider map is needed. - #[track_caller] - fn check_provider_map_prunning(&self) { - for hash in self.providers.hash_node.keys() { - let as_raw = DownloadKind(HashAndFormat::raw(*hash)); - let as_hash_seq = DownloadKind(HashAndFormat::hash_seq(*hash)); - assert!( - self.queue.contains_hash(*hash) - || self.active_requests.contains_key(&as_raw) - || self.active_requests.contains_key(&as_hash_seq), - "all hashes in the provider map are in the queue or active" - ) - } - } -} diff --git a/iroh-blobs/src/downloader/progress.rs b/iroh-blobs/src/downloader/progress.rs deleted file mode 100644 index 9b0372976f6..00000000000 --- a/iroh-blobs/src/downloader/progress.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, -}; - -use anyhow::anyhow; -use parking_lot::Mutex; - -use super::DownloadKind; -use crate::{ - get::{db::DownloadProgress, progress::TransferState}, - util::progress::{AsyncChannelProgressSender, IdGenerator, ProgressSendError, ProgressSender}, -}; - -/// The channel that can be used to subscribe to progress updates. -pub type ProgressSubscriber = AsyncChannelProgressSender; - -/// Track the progress of downloads. -/// -/// This struct allows to create [`ProgressSender`] structs to be passed to -/// [`crate::get::db::get_to_db`]. Each progress sender can be subscribed to by any number of -/// [`ProgressSubscriber`] channel senders, which will receive each progress update (if they have -/// capacity). Additionally, the [`ProgressTracker`] maintains a [`TransferState`] for each -/// transfer, applying each progress update to update this state. When subscribing to an already -/// running transfer, the subscriber will receive a [`DownloadProgress::InitialState`] message -/// containing the state at the time of the subscription, and then receive all further progress -/// events directly. -#[derive(Debug, Default)] -pub struct ProgressTracker { - /// Map of shared state for each tracked download. - running: HashMap, - /// Shared [`IdGenerator`] for all progress senders created by the tracker. - id_gen: Arc, -} - -impl ProgressTracker { - pub fn new() -> Self { - Self::default() - } - - /// Track a new download with a list of initial subscribers. - /// - /// Note that this should only be called for *new* downloads. If a download for the `kind` is - /// already tracked in this [`ProgressTracker`], calling `track` will replace all existing - /// state and subscribers (equal to calling [`Self::remove`] first). - pub fn track( - &mut self, - kind: DownloadKind, - subscribers: impl IntoIterator, - ) -> BroadcastProgressSender { - let inner = Inner { - subscribers: subscribers.into_iter().collect(), - state: TransferState::new(kind.hash()), - }; - let shared = Arc::new(Mutex::new(inner)); - self.running.insert(kind, Arc::clone(&shared)); - let id_gen = Arc::clone(&self.id_gen); - BroadcastProgressSender { shared, id_gen } - } - - /// Subscribe to a tracked download. - /// - /// Will return an error if `kind` is not yet tracked. - pub async fn subscribe( - &mut self, - kind: DownloadKind, - sender: ProgressSubscriber, - ) -> anyhow::Result<()> { - let initial_msg = self - .running - .get_mut(&kind) - .ok_or_else(|| anyhow!("state for download {kind:?} not found"))? - .lock() - .subscribe(sender.clone()); - sender.send(initial_msg).await?; - Ok(()) - } - - /// Unsubscribe `sender` from `kind`. - pub fn unsubscribe(&mut self, kind: &DownloadKind, sender: &ProgressSubscriber) { - if let Some(shared) = self.running.get_mut(kind) { - shared.lock().unsubscribe(sender) - } - } - - /// Remove all state for a download. - pub fn remove(&mut self, kind: &DownloadKind) { - self.running.remove(kind); - } -} - -type Shared = Arc>; - -#[derive(Debug)] -struct Inner { - subscribers: Vec, - state: TransferState, -} - -impl Inner { - fn subscribe(&mut self, subscriber: ProgressSubscriber) -> DownloadProgress { - let msg = DownloadProgress::InitialState(self.state.clone()); - self.subscribers.push(subscriber); - msg - } - - fn unsubscribe(&mut self, sender: &ProgressSubscriber) { - self.subscribers.retain(|s| !s.same_channel(sender)); - } - - fn on_progress(&mut self, progress: DownloadProgress) { - self.state.on_progress(progress); - } -} - -#[derive(Debug, Clone)] -pub struct BroadcastProgressSender { - shared: Shared, - id_gen: Arc, -} - -impl IdGenerator for BroadcastProgressSender { - fn new_id(&self) -> u64 { - self.id_gen.fetch_add(1, Ordering::SeqCst) - } -} - -impl ProgressSender for BroadcastProgressSender { - type Msg = DownloadProgress; - - async fn send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { - // making sure that the lock is not held across an await point. - let futs = { - let mut inner = self.shared.lock(); - inner.on_progress(msg.clone()); - let futs = inner - .subscribers - .iter_mut() - .map(|sender| { - let sender = sender.clone(); - let msg = msg.clone(); - async move { - match sender.send(msg).await { - Ok(()) => None, - Err(ProgressSendError::ReceiverDropped) => Some(sender), - } - } - }) - .collect::>(); - drop(inner); - futs - }; - - let failed_senders = futures_buffered::join_all(futs).await; - // remove senders where the receiver is dropped - if failed_senders.iter().any(|s| s.is_some()) { - let mut inner = self.shared.lock(); - for sender in failed_senders.into_iter().flatten() { - inner.unsubscribe(&sender); - } - drop(inner); - } - Ok(()) - } - - fn try_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { - let mut inner = self.shared.lock(); - inner.on_progress(msg.clone()); - // remove senders where the receiver is dropped - inner - .subscribers - .retain_mut(|sender| match sender.try_send(msg.clone()) { - Err(ProgressSendError::ReceiverDropped) => false, - Ok(()) => true, - }); - Ok(()) - } - - fn blocking_send(&self, msg: Self::Msg) -> Result<(), ProgressSendError> { - let mut inner = self.shared.lock(); - inner.on_progress(msg.clone()); - // remove senders where the receiver is dropped - inner - .subscribers - .retain_mut(|sender| match sender.blocking_send(msg.clone()) { - Err(ProgressSendError::ReceiverDropped) => false, - Ok(()) => true, - }); - Ok(()) - } -} diff --git a/iroh-blobs/src/downloader/test.rs b/iroh-blobs/src/downloader/test.rs deleted file mode 100644 index eee6f39cc74..00000000000 --- a/iroh-blobs/src/downloader/test.rs +++ /dev/null @@ -1,531 +0,0 @@ -#![cfg(test)] -use std::{ - sync::atomic::AtomicUsize, - time::{Duration, Instant}, -}; - -use anyhow::anyhow; -use futures_util::future::FutureExt; -use iroh_net::key::SecretKey; - -use super::*; -use crate::{ - get::{ - db::BlobId, - progress::{BlobProgress, TransferState}, - }, - util::{ - local_pool::LocalPool, - progress::{AsyncChannelProgressSender, IdGenerator}, - }, -}; - -mod dialer; -mod getter; - -impl Downloader { - fn spawn_for_test( - dialer: dialer::TestingDialer, - getter: getter::TestingGetter, - concurrency_limits: ConcurrencyLimits, - ) -> (Self, LocalPool) { - Self::spawn_for_test_with_retry_config( - dialer, - getter, - concurrency_limits, - Default::default(), - ) - } - - fn spawn_for_test_with_retry_config( - dialer: dialer::TestingDialer, - getter: getter::TestingGetter, - concurrency_limits: ConcurrencyLimits, - retry_config: RetryConfig, - ) -> (Self, LocalPool) { - let (msg_tx, msg_rx) = mpsc::channel(super::SERVICE_CHANNEL_CAPACITY); - - let lp = LocalPool::default(); - lp.spawn_detached(move || async move { - // we want to see the logs of the service - let _guard = iroh_test::logging::setup(); - - let service = Service::new(getter, dialer, concurrency_limits, retry_config, msg_rx); - service.run().await - }); - - ( - Downloader { - next_id: Arc::new(AtomicU64::new(0)), - msg_tx, - }, - lp, - ) - } -} - -/// Tests that receiving a download request and performing it doesn't explode. -#[tokio::test] -async fn smoke_test() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let concurrency_limits = ConcurrencyLimits::default(); - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - // send a request and make sure the peer is requested the corresponding download - let peer = SecretKey::generate().public(); - let kind: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into(); - let req = DownloadRequest::new(kind, vec![peer]); - let handle = downloader.queue(req).await; - // wait for the download result to be reported - handle.await.expect("should report success"); - // verify that the peer was dialed - dialer.assert_history(&[peer]); - // verify that the request was sent - getter.assert_history(&[(kind, peer)]); -} - -/// Tests that multiple intents produce a single request. -#[tokio::test] -async fn deduplication() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - // make request take some time to ensure the intents are received before completion - getter.set_request_duration(Duration::from_secs(1)); - let concurrency_limits = ConcurrencyLimits::default(); - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - let peer = SecretKey::generate().public(); - let kind: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into(); - let mut handles = Vec::with_capacity(10); - for _ in 0..10 { - let req = DownloadRequest::new(kind, vec![peer]); - let h = downloader.queue(req).await; - handles.push(h); - } - assert!( - futures_buffered::join_all(handles) - .await - .into_iter() - .all(|r| r.is_ok()), - "all downloads should succeed" - ); - // verify that the request was sent just once - getter.assert_history(&[(kind, peer)]); -} - -/// Tests that the request is cancelled only when all intents are cancelled. -#[tokio::test] -async fn cancellation() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - // make request take some time to ensure cancellations are received on time - getter.set_request_duration(Duration::from_millis(500)); - let concurrency_limits = ConcurrencyLimits::default(); - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - let peer = SecretKey::generate().public(); - let kind_1: DownloadKind = HashAndFormat::raw(Hash::new([0u8; 32])).into(); - let req = DownloadRequest::new(kind_1, vec![peer]); - let handle_a = downloader.queue(req.clone()).await; - let handle_b = downloader.queue(req).await; - downloader.cancel(handle_a).await; - - // create a request with two intents and cancel them both - let kind_2 = HashAndFormat::raw(Hash::new([1u8; 32])); - let req = DownloadRequest::new(kind_2, vec![peer]); - let handle_c = downloader.queue(req.clone()).await; - let handle_d = downloader.queue(req).await; - downloader.cancel(handle_c).await; - downloader.cancel(handle_d).await; - - // wait for the download result to be reported, a was cancelled but b should continue - handle_b.await.expect("should report success"); - // verify that the request was sent just once, and that the second request was never sent - getter.assert_history(&[(kind_1, peer)]); -} - -/// Test that when the downloader receives a flood of requests, they are scheduled so that the -/// maximum number of concurrent requests is not exceed. -/// NOTE: This is internally tested by [`Service::check_invariants`]. -#[tokio::test] -async fn max_concurrent_requests_total() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - // make request take some time to ensure concurreny limits are hit - getter.set_request_duration(Duration::from_millis(500)); - // set the concurreny limit very low to ensure it's hit - let concurrency_limits = ConcurrencyLimits { - max_concurrent_requests: 2, - ..Default::default() - }; - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - // send the downloads - let peer = SecretKey::generate().public(); - let mut handles = Vec::with_capacity(5); - let mut expected_history = Vec::with_capacity(5); - for i in 0..5 { - let kind: DownloadKind = HashAndFormat::raw(Hash::new([i; 32])).into(); - let req = DownloadRequest::new(kind, vec![peer]); - let h = downloader.queue(req).await; - expected_history.push((kind, peer)); - handles.push(h); - } - - assert!( - futures_buffered::join_all(handles) - .await - .into_iter() - .all(|r| r.is_ok()), - "all downloads should succeed" - ); - - // verify that the request was sent just once - getter.assert_history(&expected_history); -} - -/// Test that when the downloader receives a flood of requests, with only one peer to handle them, -/// the maximum number of requests per peer is still respected. -/// NOTE: This is internally tested by [`Service::check_invariants`]. -#[tokio::test] -async fn max_concurrent_requests_per_peer() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - // make request take some time to ensure concurreny limits are hit - getter.set_request_duration(Duration::from_millis(500)); - // set the concurreny limit very low to ensure it's hit - let concurrency_limits = ConcurrencyLimits { - max_concurrent_requests_per_node: 1, - max_concurrent_requests: 10000, // all requests can be performed at the same time - ..Default::default() - }; - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - // send the downloads - let peer = SecretKey::generate().public(); - let mut handles = Vec::with_capacity(5); - for i in 0..5 { - let kind = HashAndFormat::raw(Hash::new([i; 32])); - let req = DownloadRequest::new(kind, vec![peer]); - let h = downloader.queue(req).await; - handles.push(h); - } - - futures_buffered::join_all(handles).await; -} - -/// Tests concurrent progress reporting for multiple intents. -/// -/// This first registers two intents for a download, and then proceeds until the `Found` event is -/// emitted, and verifies that both intents received the event. -/// It then registers a third intent mid-download, and makes sure it receives a correct ìnitial -/// state. The download then finishes, and we make sure that all events are emitted properly, and -/// the progress state of the handles converges. -#[tokio::test] -async fn concurrent_progress() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - - let (start_tx, start_rx) = oneshot::channel(); - let start_rx = start_rx.shared(); - - let (done_tx, done_rx) = oneshot::channel(); - let done_rx = done_rx.shared(); - - getter.set_handler(Arc::new(move |hash, _peer, progress, _duration| { - let start_rx = start_rx.clone(); - let done_rx = done_rx.clone(); - async move { - let hash = hash.hash(); - start_rx.await.unwrap(); - let id = progress.new_id(); - progress - .send(DownloadProgress::Found { - id, - child: BlobId::Root, - hash, - size: 100, - }) - .await - .unwrap(); - done_rx.await.unwrap(); - progress.send(DownloadProgress::Done { id }).await.unwrap(); - Ok(Stats::default()) - } - .boxed() - })); - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default()); - - let peer = SecretKey::generate().public(); - let hash = Hash::new([0u8; 32]); - let kind_1 = HashAndFormat::raw(hash); - - let (prog_a_tx, prog_a_rx) = async_channel::bounded(64); - let prog_a_tx = AsyncChannelProgressSender::new(prog_a_tx); - let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_a_tx); - let handle_a = downloader.queue(req).await; - - let (prog_b_tx, prog_b_rx) = async_channel::bounded(64); - let prog_b_tx = AsyncChannelProgressSender::new(prog_b_tx); - let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_b_tx); - let handle_b = downloader.queue(req).await; - - let mut state_a = TransferState::new(hash); - let mut state_b = TransferState::new(hash); - let mut state_c = TransferState::new(hash); - - let prog0_b = prog_b_rx.recv().await.unwrap(); - assert!(matches!( - prog0_b, - DownloadProgress::InitialState(state) if state.root.hash == hash && state.root.progress == BlobProgress::Pending, - )); - - start_tx.send(()).unwrap(); - - let prog1_a = prog_a_rx.recv().await.unwrap(); - let prog1_b = prog_b_rx.recv().await.unwrap(); - assert!( - matches!(prog1_a, DownloadProgress::Found { hash: found_hash, size: 100, ..} if found_hash == hash) - ); - assert!( - matches!(prog1_b, DownloadProgress::Found { hash: found_hash, size: 100, ..} if found_hash == hash) - ); - - state_a.on_progress(prog1_a); - state_b.on_progress(prog1_b); - assert_eq!(state_a, state_b); - - let (prog_c_tx, prog_c_rx) = async_channel::bounded(64); - let prog_c_tx = AsyncChannelProgressSender::new(prog_c_tx); - let req = DownloadRequest::new(kind_1, vec![peer]).progress_sender(prog_c_tx); - let handle_c = downloader.queue(req).await; - - let prog1_c = prog_c_rx.recv().await.unwrap(); - assert!(matches!(&prog1_c, DownloadProgress::InitialState(state) if state == &state_a)); - state_c.on_progress(prog1_c); - - done_tx.send(()).unwrap(); - - let (res_a, res_b, res_c) = tokio::join!(handle_a, handle_b, handle_c); - res_a.unwrap(); - res_b.unwrap(); - res_c.unwrap(); - - let prog_a: Vec<_> = prog_a_rx.collect().await; - let prog_b: Vec<_> = prog_b_rx.collect().await; - let prog_c: Vec<_> = prog_c_rx.collect().await; - - assert_eq!(prog_a.len(), 1); - assert_eq!(prog_b.len(), 1); - assert_eq!(prog_c.len(), 1); - - assert!(matches!(prog_a[0], DownloadProgress::Done { .. })); - assert!(matches!(prog_b[0], DownloadProgress::Done { .. })); - assert!(matches!(prog_c[0], DownloadProgress::Done { .. })); - - for p in prog_a { - state_a.on_progress(p); - } - for p in prog_b { - state_b.on_progress(p); - } - for p in prog_c { - state_c.on_progress(p); - } - assert_eq!(state_a, state_b); - assert_eq!(state_a, state_c); -} - -#[tokio::test] -async fn long_queue() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let concurrency_limits = ConcurrencyLimits { - max_open_connections: 2, - max_concurrent_requests_per_node: 2, - max_concurrent_requests: 4, // all requests can be performed at the same time - ..Default::default() - }; - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - // send the downloads - let nodes = [ - SecretKey::generate().public(), - SecretKey::generate().public(), - SecretKey::generate().public(), - ]; - let mut handles = vec![]; - for i in 0..100usize { - let kind = HashAndFormat::raw(Hash::new(i.to_be_bytes())); - let peer = nodes[i % 3]; - let req = DownloadRequest::new(kind, vec![peer]); - let h = downloader.queue(req).await; - handles.push(h); - } - - let res = futures_buffered::join_all(handles).await; - for res in res { - res.expect("all downloads to succeed"); - } -} - -/// If a download errors with [`FailureAction::DropPeer`], make sure that the peer is not dropped -/// while other transfers are still running. -#[tokio::test] -async fn fail_while_running() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default()); - let blob_fail = HashAndFormat::raw(Hash::new([1u8; 32])); - let blob_success = HashAndFormat::raw(Hash::new([2u8; 32])); - - getter.set_handler(Arc::new(move |kind, _node, _progress_sender, _duration| { - async move { - if kind == blob_fail.into() { - tokio::time::sleep(Duration::from_millis(10)).await; - Err(FailureAction::DropPeer(anyhow!("bad!"))) - } else if kind == blob_success.into() { - tokio::time::sleep(Duration::from_millis(20)).await; - Ok(Default::default()) - } else { - unreachable!("invalid blob") - } - } - .boxed() - })); - - let node = SecretKey::generate().public(); - let req_success = DownloadRequest::new(blob_success, vec![node]); - let req_fail = DownloadRequest::new(blob_fail, vec![node]); - let handle_success = downloader.queue(req_success).await; - let handle_fail = downloader.queue(req_fail).await; - - let res_fail = handle_fail.await; - let res_success = handle_success.await; - - assert!(res_fail.is_err()); - assert!(res_success.is_ok()); -} - -#[tokio::test] -async fn retry_nodes_simple() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), Default::default()); - let node = SecretKey::generate().public(); - let dial_attempts = Arc::new(AtomicUsize::new(0)); - let dial_attempts2 = dial_attempts.clone(); - // fail on first dial, then succeed - dialer.set_dial_outcome(move |_node| dial_attempts2.fetch_add(1, Ordering::SeqCst) != 0); - let kind = HashAndFormat::raw(Hash::EMPTY); - let req = DownloadRequest::new(kind, vec![node]); - let handle = downloader.queue(req).await; - - assert!(handle.await.is_ok()); - assert_eq!(dial_attempts.load(Ordering::SeqCst), 2); - dialer.assert_history(&[node, node]); -} - -#[tokio::test] -async fn retry_nodes_fail() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let config = RetryConfig { - initial_retry_delay: Duration::from_millis(10), - max_retries_per_node: 3, - }; - - let (downloader, _lp) = Downloader::spawn_for_test_with_retry_config( - dialer.clone(), - getter.clone(), - Default::default(), - config, - ); - let node = SecretKey::generate().public(); - // fail always - dialer.set_dial_outcome(move |_node| false); - - // queue a download - let kind = HashAndFormat::raw(Hash::EMPTY); - let req = DownloadRequest::new(kind, vec![node]); - let now = Instant::now(); - let handle = downloader.queue(req).await; - - // assert that the download failed - assert!(handle.await.is_err()); - - // assert the dial history: we dialed 4 times - dialer.assert_history(&[node, node, node, node]); - - // assert that the retry timeouts were uphold - let expected_dial_duration = Duration::from_millis(10 * 4); - let expected_retry_wait_duration = Duration::from_millis(10 + 2 * 10 + 3 * 10); - assert!(now.elapsed() >= expected_dial_duration + expected_retry_wait_duration); -} - -#[tokio::test] -async fn retry_nodes_jump_queue() { - let _guard = iroh_test::logging::setup(); - let dialer = dialer::TestingDialer::default(); - let getter = getter::TestingGetter::default(); - let concurrency_limits = ConcurrencyLimits { - max_open_connections: 2, - max_concurrent_requests_per_node: 2, - max_concurrent_requests: 4, // all requests can be performed at the same time - ..Default::default() - }; - - let (downloader, _lp) = - Downloader::spawn_for_test(dialer.clone(), getter.clone(), concurrency_limits); - - let good_node = SecretKey::generate().public(); - let bad_node = SecretKey::generate().public(); - - dialer.set_dial_outcome(move |node| node == good_node); - let kind1 = HashAndFormat::raw(Hash::new([0u8; 32])); - let kind2 = HashAndFormat::raw(Hash::new([2u8; 32])); - - let req1 = DownloadRequest::new(kind1, vec![bad_node]); - let h1 = downloader.queue(req1).await; - - let req2 = DownloadRequest::new(kind2, vec![bad_node, good_node]); - let h2 = downloader.queue(req2).await; - - // wait for req2 to complete - this tests that the "queue is jumped" and we are not - // waiting for req1 to elapse all retries - assert!(h2.await.is_ok()); - - dialer.assert_history(&[bad_node, good_node]); - - // now we make download1 succeed! - dialer.set_dial_outcome(move |_node| true); - assert!(h1.await.is_ok()); - - // assert history - dialer.assert_history(&[bad_node, good_node, bad_node]); -} diff --git a/iroh-blobs/src/downloader/test/dialer.rs b/iroh-blobs/src/downloader/test/dialer.rs deleted file mode 100644 index fc5a9399592..00000000000 --- a/iroh-blobs/src/downloader/test/dialer.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Implementation of [`super::Dialer`] used for testing. - -use std::task::{Context, Poll}; - -use parking_lot::RwLock; - -use super::*; - -/// Dialer for testing that keeps track of the dialing history. -#[derive(Default, Clone)] -pub(super) struct TestingDialer(Arc>); - -struct TestingDialerInner { - /// Peers that are being dialed. - dialing: HashSet, - /// Queue of dials. - dial_futs: delay_queue::DelayQueue, - /// History of attempted dials. - dial_history: Vec, - /// How long does a dial last. - dial_duration: Duration, - /// Fn deciding if a dial is successful. - dial_outcome: Box bool + Send + Sync + 'static>, - /// Our own node id - node_id: NodeId, -} - -impl Default for TestingDialerInner { - fn default() -> Self { - TestingDialerInner { - dialing: HashSet::default(), - dial_futs: delay_queue::DelayQueue::default(), - dial_history: Vec::default(), - dial_duration: Duration::from_millis(10), - dial_outcome: Box::new(|_| true), - node_id: NodeId::from_bytes(&[0u8; 32]).unwrap(), - } - } -} - -impl Dialer for TestingDialer { - type Connection = NodeId; - - fn queue_dial(&mut self, node_id: NodeId) { - let mut inner = self.0.write(); - inner.dial_history.push(node_id); - // for now assume every dial works - let dial_duration = inner.dial_duration; - if inner.dialing.insert(node_id) { - inner.dial_futs.insert(node_id, dial_duration); - } - } - - fn pending_count(&self) -> usize { - self.0.read().dialing.len() - } - - fn is_pending(&self, node: NodeId) -> bool { - self.0.read().dialing.contains(&node) - } - - fn node_id(&self) -> NodeId { - self.0.read().node_id - } -} - -impl Stream for TestingDialer { - type Item = (NodeId, anyhow::Result); - - fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut inner = self.0.write(); - match inner.dial_futs.poll_expired(cx) { - Poll::Ready(Some(expired)) => { - let node = expired.into_inner(); - let report_ok = (inner.dial_outcome)(node); - let result = report_ok - .then_some(node) - .ok_or_else(|| anyhow::anyhow!("dialing test set to fail")); - inner.dialing.remove(&node); - Poll::Ready(Some((node, result))) - } - _ => Poll::Pending, - } - } -} - -impl TestingDialer { - #[track_caller] - pub(super) fn assert_history(&self, history: &[NodeId]) { - assert_eq!(self.0.read().dial_history, history) - } - - pub(super) fn set_dial_outcome( - &self, - dial_outcome: impl Fn(NodeId) -> bool + Send + Sync + 'static, - ) { - let mut inner = self.0.write(); - inner.dial_outcome = Box::new(dial_outcome); - } -} diff --git a/iroh-blobs/src/downloader/test/getter.rs b/iroh-blobs/src/downloader/test/getter.rs deleted file mode 100644 index 0ea200caa47..00000000000 --- a/iroh-blobs/src/downloader/test/getter.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Implementation of [`super::Getter`] used for testing. - -use futures_lite::{future::Boxed as BoxFuture, FutureExt}; -use parking_lot::RwLock; - -use super::*; -use crate::downloader; - -#[derive(Default, Clone, derive_more::Debug)] -#[debug("TestingGetter")] -pub(super) struct TestingGetter(Arc>); - -pub(super) type RequestHandlerFn = Arc< - dyn Fn( - DownloadKind, - NodeId, - BroadcastProgressSender, - Duration, - ) -> BoxFuture - + Send - + Sync - + 'static, ->; - -#[derive(Default)] -struct TestingGetterInner { - /// How long requests take. - request_duration: Duration, - /// History of requests performed by the [`Getter`] and if they were successful. - request_history: Vec<(DownloadKind, NodeId)>, - /// Set a handler function which actually handles the requests. - request_handler: Option, -} - -impl Getter for TestingGetter { - // since for testing we don't need a real connection, just keep track of what peer is the - // request being sent to - type Connection = NodeId; - type NeedsConn = GetStateNeedsConn; - - fn get( - &mut self, - kind: DownloadKind, - progress_sender: BroadcastProgressSender, - ) -> GetStartFut { - std::future::ready(Ok(downloader::GetOutput::NeedsConn(GetStateNeedsConn( - self.clone(), - kind, - progress_sender, - )))) - .boxed_local() - } -} - -#[derive(Debug)] -pub(super) struct GetStateNeedsConn(TestingGetter, DownloadKind, BroadcastProgressSender); - -impl downloader::NeedsConn for GetStateNeedsConn { - fn proceed(self, peer: NodeId) -> super::GetProceedFut { - let GetStateNeedsConn(getter, kind, progress_sender) = self; - let mut inner = getter.0.write(); - inner.request_history.push((kind, peer)); - let request_duration = inner.request_duration; - let handler = inner.request_handler.clone(); - async move { - if let Some(f) = handler { - f(kind, peer, progress_sender, request_duration).await - } else { - tokio::time::sleep(request_duration).await; - Ok(Stats::default()) - } - } - .boxed_local() - } -} - -impl TestingGetter { - pub(super) fn set_handler(&self, handler: RequestHandlerFn) { - self.0.write().request_handler = Some(handler); - } - pub(super) fn set_request_duration(&self, request_duration: Duration) { - self.0.write().request_duration = request_duration; - } - /// Verify that the request history is as expected - #[track_caller] - pub(super) fn assert_history(&self, history: &[(DownloadKind, NodeId)]) { - assert_eq!(self.0.read().request_history, history); - } -} diff --git a/iroh-blobs/src/export.rs b/iroh-blobs/src/export.rs deleted file mode 100644 index 38008251ba8..00000000000 --- a/iroh-blobs/src/export.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Functions to export data from a store - -use std::path::PathBuf; - -use anyhow::Context; -use bytes::Bytes; -use iroh_base::rpc::RpcError; -use serde::{Deserialize, Serialize}; -use tracing::trace; - -use crate::{ - format::collection::Collection, - store::{BaoBlobSize, ExportFormat, ExportMode, MapEntry, Store as BaoStore}, - util::progress::{IdGenerator, ProgressSender}, - Hash, -}; - -/// Export a hash to the local file system. -/// -/// This exports a single hash, or a collection `recursive` is true, from the `db` store to the -/// local filesystem. Depending on `mode` the data is either copied or reflinked (if possible). -/// -/// Progress is reported as [`ExportProgress`] through a [`ProgressSender`]. Note that the -/// [`ExportProgress::AllDone`] event is not emitted from here, but left to an upper layer to send, -/// if desired. -pub async fn export( - db: &D, - hash: Hash, - outpath: PathBuf, - format: ExportFormat, - mode: ExportMode, - progress: impl ProgressSender + IdGenerator, -) -> anyhow::Result<()> { - match format { - ExportFormat::Blob => export_blob(db, hash, outpath, mode, progress).await, - ExportFormat::Collection => export_collection(db, hash, outpath, mode, progress).await, - } -} - -/// Export all entries of a collection, recursively, to files on the local filesystem. -pub async fn export_collection( - db: &D, - hash: Hash, - outpath: PathBuf, - mode: ExportMode, - progress: impl ProgressSender + IdGenerator, -) -> anyhow::Result<()> { - tokio::fs::create_dir_all(&outpath).await?; - let collection = Collection::load_db(db, &hash).await?; - for (name, hash) in collection.into_iter() { - #[allow(clippy::needless_borrow)] - let path = outpath.join(pathbuf_from_name(&name)); - export_blob(db, hash, path, mode, progress.clone()).await?; - } - Ok(()) -} - -/// Export a single blob to a file on the local filesystem. -pub async fn export_blob( - db: &D, - hash: Hash, - outpath: PathBuf, - mode: ExportMode, - progress: impl ProgressSender + IdGenerator, -) -> anyhow::Result<()> { - if let Some(parent) = outpath.parent() { - tokio::fs::create_dir_all(parent).await?; - } - trace!("exporting blob {} to {}", hash, outpath.display()); - let id = progress.new_id(); - let entry = db.get(&hash).await?.context("entry not there")?; - progress - .send(ExportProgress::Found { - id, - hash, - outpath: outpath.clone(), - size: entry.size(), - meta: None, - }) - .await?; - let progress1 = progress.clone(); - db.export( - hash, - outpath, - mode, - Box::new(move |offset| Ok(progress1.try_send(ExportProgress::Progress { id, offset })?)), - ) - .await?; - progress.send(ExportProgress::Done { id }).await?; - Ok(()) -} - -/// Progress events for an export operation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum ExportProgress { - /// The download part is done for this id, we are now exporting the data - /// to the specified out path. - Found { - /// Unique id of the entry. - id: u64, - /// The hash of the entry. - hash: Hash, - /// The size of the entry in bytes. - size: BaoBlobSize, - /// The path to the file where the data is exported. - outpath: PathBuf, - /// Operation-specific metadata. - meta: Option, - }, - /// We have made progress exporting the data. - /// - /// This is only sent for large blobs. - Progress { - /// Unique id of the entry that is being exported. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We finished exporting a blob - Done { - /// Unique id of the entry that is being exported. - id: u64, - }, - /// We are done with the whole operation. - AllDone, - /// We got an error and need to abort. - Abort(RpcError), -} - -fn pathbuf_from_name(name: &str) -> PathBuf { - let mut path = PathBuf::new(); - for part in name.split('/') { - path.push(part); - } - path -} diff --git a/iroh-blobs/src/format.rs b/iroh-blobs/src/format.rs deleted file mode 100644 index 2ccb8f3baa3..00000000000 --- a/iroh-blobs/src/format.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Defines data formats for HashSeq. -//! -//! The exact details how to use a HashSeq for specific purposes is up to the -//! user. However, the following approach is used by iroh formats: -//! -//! The first child blob is a metadata blob. It starts with a header, followed -//! by serialized metadata. We mostly use [postcard] for serialization. The -//! metadata either implicitly or explicitly refers to the other blobs in the -//! HashSeq by index. -//! -//! In a very simple case, the metadata just an array of items, where each item -//! is the metadata for the corresponding blob. The metadata array will have -//! n-1 items, where n is the number of blobs in the HashSeq. -//! -//! [postcard]: https://docs.rs/postcard/latest/postcard/ -pub mod collection; diff --git a/iroh-blobs/src/format/collection.rs b/iroh-blobs/src/format/collection.rs deleted file mode 100644 index ca4f95548ea..00000000000 --- a/iroh-blobs/src/format/collection.rs +++ /dev/null @@ -1,347 +0,0 @@ -//! The collection type used by iroh -use std::{collections::BTreeMap, future::Future}; - -use anyhow::Context; -use bao_tree::blake3; -use bytes::Bytes; -use iroh_io::AsyncSliceReaderExt; -use serde::{Deserialize, Serialize}; - -use crate::{ - get::{fsm, Stats}, - hashseq::HashSeq, - store::MapEntry, - util::TempTag, - BlobFormat, Hash, -}; - -/// A collection of blobs -/// -/// Note that the format is subject to change. -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Default)] -pub struct Collection { - /// Links to the blobs in this collection - blobs: Vec<(String, Hash)>, -} - -impl std::ops::Index for Collection { - type Output = (String, Hash); - - fn index(&self, index: usize) -> &Self::Output { - &self.blobs[index] - } -} - -impl Extend<(K, V)> for Collection -where - K: Into, - V: Into, -{ - fn extend>(&mut self, iter: T) { - self.blobs - .extend(iter.into_iter().map(|(k, v)| (k.into(), v.into()))); - } -} - -impl FromIterator<(K, V)> for Collection -where - K: Into, - V: Into, -{ - fn from_iter>(iter: T) -> Self { - let mut res = Self::default(); - res.extend(iter); - res - } -} - -impl IntoIterator for Collection { - type Item = (String, Hash); - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.blobs.into_iter() - } -} - -/// A simple store trait for loading blobs -pub trait SimpleStore { - /// Load a blob from the store - fn load(&self, hash: Hash) -> impl Future> + Send + '_; -} - -/// Metadata for a collection -/// -/// This is the wire format for the metadata blob. -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -struct CollectionMeta { - header: [u8; 13], // Must contain "CollectionV0." - names: Vec, -} - -impl Collection { - /// The header for the collection format. - /// - /// This is the start of the metadata blob. - pub const HEADER: &'static [u8; 13] = b"CollectionV0."; - - /// Convert the collection to an iterator of blobs, with the last being the - /// root blob. - /// - /// To persist the collection, write all the blobs to storage, and use the - /// hash of the last blob as the collection hash. - pub fn to_blobs(&self) -> impl DoubleEndedIterator { - let meta = CollectionMeta { - header: *Self::HEADER, - names: self.names(), - }; - let meta_bytes = postcard::to_stdvec(&meta).unwrap(); - let meta_bytes_hash = blake3::hash(&meta_bytes).into(); - let links = std::iter::once(meta_bytes_hash) - .chain(self.links()) - .collect::(); - let links_bytes = links.into_inner(); - [meta_bytes.into(), links_bytes].into_iter() - } - - /// Read the collection from a get fsm. - /// - /// Returns the fsm at the start of the first child blob (if any), - /// the links array, and the collection. - pub async fn read_fsm( - fsm_at_start_root: fsm::AtStartRoot, - ) -> anyhow::Result<(fsm::EndBlobNext, HashSeq, Collection)> { - let (next, links) = { - let curr = fsm_at_start_root.next(); - let (curr, data) = curr.concatenate_into_vec().await?; - let links = HashSeq::new(data.into()).context("links could not be parsed")?; - (curr.next(), links) - }; - let fsm::EndBlobNext::MoreChildren(at_meta) = next else { - anyhow::bail!("expected meta"); - }; - let (next, collection) = { - let mut children = links.clone(); - let meta_link = children.pop_front().context("meta link not found")?; - let curr = at_meta.next(meta_link); - let (curr, names) = curr.concatenate_into_vec().await?; - let names = postcard::from_bytes::(&names)?; - anyhow::ensure!( - names.header == *Self::HEADER, - "expected header {:?}, got {:?}", - Self::HEADER, - names.header - ); - let collection = Collection::from_parts(children, names); - (curr.next(), collection) - }; - Ok((next, links, collection)) - } - - /// Read the collection and all it's children from a get fsm. - /// - /// Returns the collection, a map from blob offsets to bytes, and the stats. - pub async fn read_fsm_all( - fsm_at_start_root: crate::get::fsm::AtStartRoot, - ) -> anyhow::Result<(Collection, BTreeMap, Stats)> { - let (next, links, collection) = Self::read_fsm(fsm_at_start_root).await?; - let mut res = BTreeMap::new(); - let mut curr = next; - let end = loop { - match curr { - fsm::EndBlobNext::MoreChildren(more) => { - let child_offset = more.child_offset(); - let Some(hash) = links.get(usize::try_from(child_offset)?) else { - break more.finish(); - }; - let header = more.next(hash); - let (next, blob) = header.concatenate_into_vec().await?; - res.insert(child_offset - 1, blob.into()); - curr = next.next(); - } - fsm::EndBlobNext::Closing(closing) => break closing, - } - }; - let stats = end.next().await?; - Ok((collection, res, stats)) - } - - /// Create a new collection from a hash sequence and metadata. - pub async fn load(root: Hash, store: &impl SimpleStore) -> anyhow::Result { - let hs = store.load(root).await?; - let hs = HashSeq::try_from(hs)?; - let meta_hash = hs.iter().next().context("empty hash seq")?; - let meta = store.load(meta_hash).await?; - let meta: CollectionMeta = postcard::from_bytes(&meta)?; - anyhow::ensure!( - meta.names.len() + 1 == hs.len(), - "names and links length mismatch" - ); - Ok(Self::from_parts(hs.into_iter().skip(1), meta)) - } - - /// Load a collection from a store given a root hash - /// - /// This assumes that both the links and the metadata of the collection is stored in the store. - /// It does not require that all child blobs are stored in the store. - pub async fn load_db(db: &D, root: &Hash) -> anyhow::Result - where - D: crate::store::Map, - { - let links_entry = db.get(root).await?.context("links not found")?; - anyhow::ensure!(links_entry.is_complete(), "links not complete"); - let links_bytes = links_entry.data_reader().await?.read_to_end().await?; - let mut links = HashSeq::try_from(links_bytes)?; - let meta_hash = links.pop_front().context("meta hash not found")?; - let meta_entry = db.get(&meta_hash).await?.context("meta not found")?; - anyhow::ensure!(links_entry.is_complete(), "links not complete"); - let meta_bytes = meta_entry.data_reader().await?.read_to_end().await?; - let meta: CollectionMeta = postcard::from_bytes(&meta_bytes)?; - anyhow::ensure!( - meta.names.len() == links.len(), - "names and links length mismatch" - ); - Ok(Self::from_parts(links, meta)) - } - - /// Store a collection in a store. returns the root hash of the collection - /// as a TempTag. - pub async fn store(self, db: &D) -> anyhow::Result - where - D: crate::store::Store, - { - let (links, meta) = self.into_parts(); - let meta_bytes = postcard::to_stdvec(&meta)?; - let meta_tag = db.import_bytes(meta_bytes.into(), BlobFormat::Raw).await?; - let links_bytes = std::iter::once(*meta_tag.hash()) - .chain(links) - .collect::(); - let links_tag = db - .import_bytes(links_bytes.into(), BlobFormat::HashSeq) - .await?; - Ok(links_tag) - } - - /// Split a collection into a sequence of links and metadata - fn into_parts(self) -> (Vec, CollectionMeta) { - let mut names = Vec::with_capacity(self.blobs.len()); - let mut links = Vec::with_capacity(self.blobs.len()); - for (name, hash) in self.blobs { - names.push(name); - links.push(hash); - } - let meta = CollectionMeta { - header: *Self::HEADER, - names, - }; - (links, meta) - } - - /// Create a new collection from a list of hashes and metadata - fn from_parts(links: impl IntoIterator, meta: CollectionMeta) -> Self { - meta.names.into_iter().zip(links).collect() - } - - /// Get the links to the blobs in this collection - fn links(&self) -> impl Iterator + '_ { - self.blobs.iter().map(|(_name, hash)| *hash) - } - - /// Get the names of the blobs in this collection - fn names(&self) -> Vec { - self.blobs.iter().map(|(name, _)| name.clone()).collect() - } - - /// Iterate over the blobs in this collection - pub fn iter(&self) -> impl Iterator { - self.blobs.iter() - } - - /// Get the number of blobs in this collection - pub fn len(&self) -> usize { - self.blobs.len() - } - - /// Check if this collection is empty - pub fn is_empty(&self) -> bool { - self.blobs.is_empty() - } - - /// Add the given blob to the collection. - pub fn push(&mut self, name: String, hash: Hash) { - self.blobs.push((name, hash)); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn roundtrip_blob() { - let b = ( - "test".to_string(), - blake3::Hash::from_hex( - "3aa61c409fd7717c9d9c639202af2fae470c0ef669be7ba2caea5779cb534e9d", - ) - .unwrap() - .into(), - ); - - let mut buf = bytes::BytesMut::zeroed(1024); - postcard::to_slice(&b, &mut buf).unwrap(); - let deserialize_b: (String, Hash) = postcard::from_bytes(&buf).unwrap(); - assert_eq!(b, deserialize_b); - } - - #[test] - fn roundtrip_collection_meta() { - let expected = CollectionMeta { - header: *Collection::HEADER, - names: vec!["test".to_string(), "a".to_string(), "b".to_string()], - }; - let mut buf = bytes::BytesMut::zeroed(1024); - postcard::to_slice(&expected, &mut buf).unwrap(); - let actual: CollectionMeta = postcard::from_bytes(&buf).unwrap(); - assert_eq!(expected, actual); - } - - #[tokio::test] - async fn collection_store_load() -> testresult::TestResult { - let collection = (0..3) - .map(|i| { - ( - format!("blob{}", i), - crate::Hash::from(blake3::hash(&[i as u8])), - ) - }) - .collect::(); - let mut root = None; - let store = collection - .to_blobs() - .map(|data| { - let hash = crate::Hash::from(blake3::hash(&data)); - root = Some(hash); - (hash, data) - }) - .collect::(); - let collection2 = Collection::load(root.unwrap(), &store).await?; - assert_eq!(collection, collection2); - Ok(()) - } - - /// An implementation of a [SimpleStore] for testing - struct TestStore(BTreeMap); - - impl FromIterator<(Hash, Bytes)> for TestStore { - fn from_iter>(iter: T) -> Self { - Self(iter.into_iter().collect()) - } - } - - impl SimpleStore for TestStore { - async fn load(&self, hash: Hash) -> anyhow::Result { - self.0.get(&hash).cloned().context("not found") - } - } -} diff --git a/iroh-blobs/src/get.rs b/iroh-blobs/src/get.rs deleted file mode 100644 index 38ef9237794..00000000000 --- a/iroh-blobs/src/get.rs +++ /dev/null @@ -1,945 +0,0 @@ -//! The client side API -//! -//! To get data, create a connection using [iroh-net] or use any quinn -//! connection that was obtained in another way. -//! -//! Create a request describing the data you want to get. -//! -//! Then create a state machine using [fsm::start] and -//! drive it to completion by calling next on each state. -//! -//! For some states you have to provide additional arguments when calling next, -//! or you can choose to finish early. -//! -//! [iroh-net]: https://docs.rs/iroh-net -use std::{ - error::Error, - fmt::{self, Debug}, - time::{Duration, Instant}, -}; - -use anyhow::Result; -use bao_tree::{io::fsm::BaoContentItem, ChunkNum}; -use iroh_net::endpoint::{self, RecvStream, SendStream}; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error}; - -use crate::{ - protocol::RangeSpecSeq, - util::io::{TrackingReader, TrackingWriter}, - Hash, IROH_BLOCK_SIZE, -}; - -pub mod db; -pub mod error; -pub mod progress; -pub mod request; - -/// Stats about the transfer. -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Stats { - /// The number of bytes written - pub bytes_written: u64, - /// The number of bytes read - pub bytes_read: u64, - /// The time it took to transfer the data - pub elapsed: Duration, -} - -impl Stats { - /// Transfer rate in megabits per second - pub fn mbits(&self) -> f64 { - let data_len_bit = self.bytes_read * 8; - data_len_bit as f64 / (1000. * 1000.) / self.elapsed.as_secs_f64() - } -} - -/// Finite state machine for get responses. -/// -/// This is the low level API for getting data from a peer. -#[doc = include_str!("../docs/img/get_machine.drawio.svg")] -pub mod fsm { - use std::{io, result}; - - use bao_tree::{ - io::fsm::{OutboardMut, ResponseDecoder, ResponseDecoderNext}, - BaoTree, ChunkRanges, TreeNode, - }; - use derive_more::From; - use iroh_io::{AsyncSliceWriter, AsyncStreamReader, TokioStreamReader}; - use iroh_net::endpoint::Connection; - use tokio::io::AsyncWriteExt; - - use super::*; - use crate::{ - protocol::{GetRequest, NonEmptyRequestRangeSpecIter, Request, MAX_MESSAGE_SIZE}, - store::BaoBatchWriter, - }; - - type WrappedRecvStream = TrackingReader>; - - self_cell::self_cell! { - struct RangesIterInner { - owner: RangeSpecSeq, - #[covariant] - dependent: NonEmptyRequestRangeSpecIter, - } - } - - /// The entry point of the get response machine - pub fn start(connection: Connection, request: GetRequest) -> AtInitial { - AtInitial::new(connection, request) - } - - /// Owned iterator for the ranges in a request - /// - /// We need an owned iterator for a fsm style API, otherwise we would have - /// to drag a lifetime around every single state. - struct RangesIter(RangesIterInner); - - impl fmt::Debug for RangesIter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RangesIter").finish() - } - } - - impl RangesIter { - pub fn new(owner: RangeSpecSeq) -> Self { - Self(RangesIterInner::new(owner, |owner| owner.iter_non_empty())) - } - - pub fn offset(&self) -> u64 { - self.0.with_dependent(|_owner, iter| iter.offset()) - } - } - - impl Iterator for RangesIter { - type Item = (u64, ChunkRanges); - - fn next(&mut self) -> Option { - self.0.with_dependent_mut(|_owner, iter| { - iter.next() - .map(|(offset, ranges)| (offset, ranges.to_chunk_ranges())) - }) - } - } - - /// Initial state of the get response machine - #[derive(Debug)] - pub struct AtInitial { - connection: Connection, - request: GetRequest, - } - - impl AtInitial { - /// Create a new get response - /// - /// `connection` is an existing connection - /// `request` is the request to be sent - pub fn new(connection: Connection, request: GetRequest) -> Self { - Self { - connection, - request, - } - } - - /// Initiate a new bidi stream to use for the get response - pub async fn next(self) -> Result { - let start = Instant::now(); - let (writer, reader) = self.connection.open_bi().await?; - let reader = TrackingReader::new(TokioStreamReader::new(reader)); - let writer = TrackingWriter::new(writer); - Ok(AtConnected { - start, - reader, - writer, - request: self.request, - }) - } - } - - /// State of the get response machine after the handshake has been sent - #[derive(Debug)] - pub struct AtConnected { - start: Instant, - reader: WrappedRecvStream, - writer: TrackingWriter, - request: GetRequest, - } - - /// Possible next states after the handshake has been sent - #[derive(Debug, From)] - pub enum ConnectedNext { - /// First response is either a collection or a single blob - StartRoot(AtStartRoot), - /// First response is a child - StartChild(AtStartChild), - /// Request is empty - Closing(AtClosing), - } - - /// Error that you can get from [`AtConnected::next`] - #[derive(Debug, thiserror::Error)] - pub enum ConnectedNextError { - /// Error when serializing the request - #[error("postcard ser: {0}")] - PostcardSer(postcard::Error), - /// The serialized request is too long to be sent - #[error("request too big")] - RequestTooBig, - /// Error when writing the request to the [`SendStream`]. - #[error("write: {0}")] - Write(#[from] quinn::WriteError), - /// Quic connection is closed. - #[error("closed")] - Closed(#[from] quinn::ClosedStream), - /// A generic io error - #[error("io {0}")] - Io(io::Error), - } - - impl ConnectedNextError { - fn from_io(cause: io::Error) -> Self { - if let Some(inner) = cause.get_ref() { - if let Some(e) = inner.downcast_ref::() { - Self::Write(e.clone()) - } else { - Self::Io(cause) - } - } else { - Self::Io(cause) - } - } - } - - impl From for io::Error { - fn from(cause: ConnectedNextError) -> Self { - match cause { - ConnectedNextError::Write(cause) => cause.into(), - ConnectedNextError::Io(cause) => cause, - ConnectedNextError::PostcardSer(cause) => { - io::Error::new(io::ErrorKind::Other, cause) - } - _ => io::Error::new(io::ErrorKind::Other, cause), - } - } - } - - impl AtConnected { - /// Send the request and move to the next state - /// - /// The next state will be either `StartRoot` or `StartChild` depending on whether - /// the request requests part of the collection or not. - /// - /// If the request is empty, this can also move directly to `Finished`. - pub async fn next(self) -> Result { - let Self { - start, - reader, - mut writer, - mut request, - } = self; - // 1. Send Request - { - debug!("sending request"); - let wrapped = Request::Get(request); - let request_bytes = - postcard::to_stdvec(&wrapped).map_err(ConnectedNextError::PostcardSer)?; - let Request::Get(x) = wrapped; - request = x; - - if request_bytes.len() > MAX_MESSAGE_SIZE { - return Err(ConnectedNextError::RequestTooBig); - } - - // write the request itself - writer - .write_all(&request_bytes) - .await - .map_err(ConnectedNextError::from_io)?; - } - - // 2. Finish writing before expecting a response - let (mut writer, bytes_written) = writer.into_parts(); - writer.finish()?; - - let hash = request.hash; - let ranges_iter = RangesIter::new(request.ranges); - // this is in a box so we don't have to memcpy it on every state transition - let mut misc = Box::new(Misc { - start, - bytes_written, - ranges_iter, - }); - Ok(match misc.ranges_iter.next() { - Some((offset, ranges)) => { - if offset == 0 { - AtStartRoot { - reader, - ranges, - misc, - hash, - } - .into() - } else { - AtStartChild { - reader, - ranges, - misc, - child_offset: offset - 1, - } - .into() - } - } - None => AtClosing::new(misc, reader, true).into(), - }) - } - } - - /// State of the get response when we start reading a collection - #[derive(Debug)] - pub struct AtStartRoot { - ranges: ChunkRanges, - reader: TrackingReader>, - misc: Box, - hash: Hash, - } - - /// State of the get response when we start reading a child - #[derive(Debug)] - pub struct AtStartChild { - ranges: ChunkRanges, - reader: TrackingReader>, - misc: Box, - child_offset: u64, - } - - impl AtStartChild { - /// The offset of the child we are currently reading - /// - /// This must be used to determine the hash needed to call next. - /// If this is larger than the number of children in the collection, - /// you can call finish to stop reading the response. - pub fn child_offset(&self) -> u64 { - self.child_offset - } - - /// The ranges we have requested for the child - pub fn ranges(&self) -> &ChunkRanges { - &self.ranges - } - - /// Go into the next state, reading the header - /// - /// This requires passing in the hash of the child for validation - pub fn next(self, hash: Hash) -> AtBlobHeader { - AtBlobHeader { - reader: self.reader, - ranges: self.ranges, - misc: self.misc, - hash, - } - } - - /// Finish the get response without reading further - /// - /// This is used if you know that there are no more children from having - /// read the collection, or when you want to stop reading the response - /// early. - pub fn finish(self) -> AtClosing { - AtClosing::new(self.misc, self.reader, false) - } - } - - impl AtStartRoot { - /// The ranges we have requested for the child - pub fn ranges(&self) -> &ChunkRanges { - &self.ranges - } - - /// Hash of the root blob - pub fn hash(&self) -> Hash { - self.hash - } - - /// Go into the next state, reading the header - /// - /// For the collection we already know the hash, since it was part of the request - pub fn next(self) -> AtBlobHeader { - AtBlobHeader { - reader: self.reader, - ranges: self.ranges, - hash: self.hash, - misc: self.misc, - } - } - - /// Finish the get response without reading further - pub fn finish(self) -> AtClosing { - AtClosing::new(self.misc, self.reader, false) - } - } - - /// State before reading a size header - #[derive(Debug)] - pub struct AtBlobHeader { - ranges: ChunkRanges, - reader: TrackingReader>, - misc: Box, - hash: Hash, - } - - /// Error that you can get from [`AtBlobHeader::next`] - #[derive(Debug, thiserror::Error)] - pub enum AtBlobHeaderNextError { - /// Eof when reading the size header - /// - /// This indicates that the provider does not have the requested data. - #[error("not found")] - NotFound, - /// Quinn read error when reading the size header - #[error("read: {0}")] - Read(endpoint::ReadError), - /// Generic io error - #[error("io: {0}")] - Io(io::Error), - } - - impl From for io::Error { - fn from(cause: AtBlobHeaderNextError) -> Self { - match cause { - AtBlobHeaderNextError::NotFound => { - io::Error::new(io::ErrorKind::UnexpectedEof, cause) - } - AtBlobHeaderNextError::Read(cause) => cause.into(), - AtBlobHeaderNextError::Io(cause) => cause, - } - } - } - - impl AtBlobHeader { - /// Read the size header, returning it and going into the `Content` state. - pub async fn next(mut self) -> Result<(AtBlobContent, u64), AtBlobHeaderNextError> { - let size = self.reader.read::<8>().await.map_err(|cause| { - if cause.kind() == io::ErrorKind::UnexpectedEof { - AtBlobHeaderNextError::NotFound - } else if let Some(e) = cause - .get_ref() - .and_then(|x| x.downcast_ref::()) - { - AtBlobHeaderNextError::Read(e.clone()) - } else { - AtBlobHeaderNextError::Io(cause) - } - })?; - let size = u64::from_le_bytes(size); - let stream = ResponseDecoder::new( - self.hash.into(), - self.ranges, - BaoTree::new(size, IROH_BLOCK_SIZE), - self.reader, - ); - Ok(( - AtBlobContent { - stream, - misc: self.misc, - }, - size, - )) - } - - /// Drain the response and throw away the result - pub async fn drain(self) -> result::Result { - let (content, _size) = self.next().await?; - content.drain().await - } - - /// Concatenate the entire response into a vec - /// - /// For a request that does not request the complete blob, this will just - /// concatenate the ranges that were requested. - pub async fn concatenate_into_vec( - self, - ) -> result::Result<(AtEndBlob, Vec), DecodeError> { - let (content, _size) = self.next().await?; - content.concatenate_into_vec().await - } - - /// Write the entire blob to a slice writer. - pub async fn write_all( - self, - data: D, - ) -> result::Result { - let (content, _size) = self.next().await?; - let res = content.write_all(data).await?; - Ok(res) - } - - /// Write the entire blob to a slice writer and to an optional outboard. - /// - /// The outboard is only written to if the blob is larger than a single - /// chunk group. - pub async fn write_all_with_outboard( - self, - outboard: Option, - data: D, - ) -> result::Result - where - D: AsyncSliceWriter, - O: OutboardMut, - { - let (content, _size) = self.next().await?; - let res = content.write_all_with_outboard(outboard, data).await?; - Ok(res) - } - - /// Write the entire stream for this blob to a batch writer. - pub async fn write_all_batch(self, batch: B) -> result::Result - where - B: BaoBatchWriter, - { - let (content, _size) = self.next().await?; - let res = content.write_all_batch(batch).await?; - Ok(res) - } - - /// The hash of the blob we are reading. - pub fn hash(&self) -> Hash { - self.hash - } - - /// The ranges we have requested for the current hash. - pub fn ranges(&self) -> &ChunkRanges { - &self.ranges - } - - /// The current offset of the blob we are reading. - pub fn offset(&self) -> u64 { - self.misc.ranges_iter.offset() - } - } - - /// State while we are reading content - #[derive(Debug)] - pub struct AtBlobContent { - stream: ResponseDecoder, - misc: Box, - } - - /// Decode error that you can get once you have sent the request and are - /// decoding the response, e.g. from [`AtBlobContent::next`]. - /// - /// This is similar to [`bao_tree::io::DecodeError`], but takes into account - /// that we are reading from a [`RecvStream`], so read errors will be - /// propagated as [`DecodeError::Read`], containing a [`ReadError`]. - /// This carries more concrete information about the error than an [`io::Error`]. - /// - /// When the provider finds that it does not have a chunk that we requested, - /// or that the chunk is invalid, it will stop sending data without producing - /// an error. This is indicated by either the [`DecodeError::ParentNotFound`] or - /// [`DecodeError::LeafNotFound`] variant, which can be used to detect that data - /// is missing but the connection as well that the provider is otherwise healthy. - /// - /// The [`DecodeError::ParentHashMismatch`] and [`DecodeError::LeafHashMismatch`] - /// variants indicate that the provider has sent us invalid data. A well-behaved - /// provider should never do this, so this is an indication that the provider is - /// not behaving correctly. - /// - /// The [`DecodeError::Io`] variant is just a fallback for any other io error that - /// is not actually a [`ReadError`]. - /// - /// [`ReadError`]: endpoint::ReadError - #[derive(Debug, thiserror::Error)] - pub enum DecodeError { - /// A chunk was not found or invalid, so the provider stopped sending data - #[error("not found")] - NotFound, - /// A parent was not found or invalid, so the provider stopped sending data - #[error("parent not found {0:?}")] - ParentNotFound(TreeNode), - /// A parent was not found or invalid, so the provider stopped sending data - #[error("chunk not found {0}")] - LeafNotFound(ChunkNum), - /// The hash of a parent did not match the expected hash - #[error("parent hash mismatch: {0:?}")] - ParentHashMismatch(TreeNode), - /// The hash of a leaf did not match the expected hash - #[error("leaf hash mismatch: {0}")] - LeafHashMismatch(ChunkNum), - /// Error when reading from the stream - #[error("read: {0}")] - Read(endpoint::ReadError), - /// A generic io error - #[error("io: {0}")] - Io(#[from] io::Error), - } - - impl From for DecodeError { - fn from(cause: AtBlobHeaderNextError) -> Self { - match cause { - AtBlobHeaderNextError::NotFound => Self::NotFound, - AtBlobHeaderNextError::Read(cause) => Self::Read(cause), - AtBlobHeaderNextError::Io(cause) => Self::Io(cause), - } - } - } - - impl From for io::Error { - fn from(cause: DecodeError) -> Self { - match cause { - DecodeError::ParentNotFound(_) => { - io::Error::new(io::ErrorKind::UnexpectedEof, cause) - } - DecodeError::LeafNotFound(_) => io::Error::new(io::ErrorKind::UnexpectedEof, cause), - DecodeError::Read(cause) => cause.into(), - DecodeError::Io(cause) => cause, - _ => io::Error::new(io::ErrorKind::Other, cause), - } - } - } - - impl From for DecodeError { - fn from(value: bao_tree::io::DecodeError) -> Self { - match value { - bao_tree::io::DecodeError::ParentNotFound(x) => Self::ParentNotFound(x), - bao_tree::io::DecodeError::LeafNotFound(x) => Self::LeafNotFound(x), - bao_tree::io::DecodeError::ParentHashMismatch(node) => { - Self::ParentHashMismatch(node) - } - bao_tree::io::DecodeError::LeafHashMismatch(chunk) => Self::LeafHashMismatch(chunk), - bao_tree::io::DecodeError::Io(cause) => { - if let Some(inner) = cause.get_ref() { - if let Some(e) = inner.downcast_ref::() { - Self::Read(e.clone()) - } else { - Self::Io(cause) - } - } else { - Self::Io(cause) - } - } - } - } - } - - /// The next state after reading a content item - #[derive(Debug, From)] - pub enum BlobContentNext { - /// We expect more content - More((AtBlobContent, result::Result)), - /// We are done with this blob - Done(AtEndBlob), - } - - impl AtBlobContent { - /// Read the next item, either content, an error, or the end of the blob - pub async fn next(self) -> BlobContentNext { - match self.stream.next().await { - ResponseDecoderNext::More((stream, res)) => { - let next = Self { stream, ..self }; - let res = res.map_err(DecodeError::from); - BlobContentNext::More((next, res)) - } - ResponseDecoderNext::Done(stream) => BlobContentNext::Done(AtEndBlob { - stream, - misc: self.misc, - }), - } - } - - /// The geometry of the tree we are currently reading. - pub fn tree(&self) -> bao_tree::BaoTree { - self.stream.tree() - } - - /// The hash of the blob we are reading. - pub fn hash(&self) -> Hash { - (*self.stream.hash()).into() - } - - /// The current offset of the blob we are reading. - pub fn offset(&self) -> u64 { - self.misc.ranges_iter.offset() - } - - /// Drain the response and throw away the result - pub async fn drain(self) -> result::Result { - let mut content = self; - loop { - match content.next().await { - BlobContentNext::More((content1, res)) => { - let _ = res?; - content = content1; - } - BlobContentNext::Done(end) => { - break Ok(end); - } - } - } - } - - /// Concatenate the entire response into a vec - pub async fn concatenate_into_vec( - self, - ) -> result::Result<(AtEndBlob, Vec), DecodeError> { - let mut res = Vec::with_capacity(1024); - let mut curr = self; - let done = loop { - match curr.next().await { - BlobContentNext::More((next, data)) => { - if let BaoContentItem::Leaf(leaf) = data? { - res.extend_from_slice(&leaf.data); - } - curr = next; - } - BlobContentNext::Done(done) => { - // we are done with the root blob - break done; - } - } - }; - Ok((done, res)) - } - - /// Write the entire stream for this blob to a batch writer. - pub async fn write_all_batch(self, writer: B) -> result::Result - where - B: BaoBatchWriter, - { - let mut writer = writer; - let mut buf = Vec::new(); - let mut content = self; - let size = content.tree().size(); - loop { - match content.next().await { - BlobContentNext::More((next, item)) => { - let item = item?; - match &item { - BaoContentItem::Parent(_) => { - buf.push(item); - } - BaoContentItem::Leaf(_) => { - buf.push(item); - let batch = std::mem::take(&mut buf); - writer.write_batch(size, batch).await?; - } - } - content = next; - } - BlobContentNext::Done(end) => { - assert!(buf.is_empty()); - return Ok(end); - } - } - } - } - - /// Write the entire blob to a slice writer and to an optional outboard. - /// - /// The outboard is only written to if the blob is larger than a single - /// chunk group. - pub async fn write_all_with_outboard( - self, - mut outboard: Option, - mut data: D, - ) -> result::Result - where - D: AsyncSliceWriter, - O: OutboardMut, - { - let mut content = self; - loop { - match content.next().await { - BlobContentNext::More((content1, item)) => { - content = content1; - match item? { - BaoContentItem::Parent(parent) => { - if let Some(outboard) = outboard.as_mut() { - outboard.save(parent.node, &parent.pair).await?; - } - } - BaoContentItem::Leaf(leaf) => { - data.write_bytes_at(leaf.offset, leaf.data).await?; - } - } - } - BlobContentNext::Done(end) => { - return Ok(end); - } - } - } - } - - /// Write the entire blob to a slice writer. - pub async fn write_all(self, mut data: D) -> result::Result - where - D: AsyncSliceWriter, - { - let mut content = self; - loop { - match content.next().await { - BlobContentNext::More((content1, item)) => { - content = content1; - match item? { - BaoContentItem::Parent(_) => {} - BaoContentItem::Leaf(leaf) => { - data.write_bytes_at(leaf.offset, leaf.data).await?; - } - } - } - BlobContentNext::Done(end) => { - return Ok(end); - } - } - } - } - - /// Immediately finish the get response without reading further - pub fn finish(self) -> AtClosing { - AtClosing::new(self.misc, self.stream.finish(), false) - } - } - - /// State after we have read all the content for a blob - #[derive(Debug)] - pub struct AtEndBlob { - stream: WrappedRecvStream, - misc: Box, - } - - /// The next state after the end of a blob - #[derive(Debug, From)] - pub enum EndBlobNext { - /// Response is expected to have more children - MoreChildren(AtStartChild), - /// No more children expected - Closing(AtClosing), - } - - impl AtEndBlob { - /// Read the next child, or finish - pub fn next(mut self) -> EndBlobNext { - if let Some((offset, ranges)) = self.misc.ranges_iter.next() { - AtStartChild { - reader: self.stream, - child_offset: offset - 1, - ranges, - misc: self.misc, - } - .into() - } else { - AtClosing::new(self.misc, self.stream, true).into() - } - } - } - - /// State when finishing the get response - #[derive(Debug)] - pub struct AtClosing { - misc: Box, - reader: WrappedRecvStream, - check_extra_data: bool, - } - - impl AtClosing { - fn new(misc: Box, reader: WrappedRecvStream, check_extra_data: bool) -> Self { - Self { - misc, - reader, - check_extra_data, - } - } - - /// Finish the get response, returning statistics - pub async fn next(self) -> result::Result { - // Shut down the stream - let (reader, bytes_read) = self.reader.into_parts(); - let mut reader = reader.into_inner(); - if self.check_extra_data { - if let Some(chunk) = reader.read_chunk(8, false).await? { - reader.stop(0u8.into()).ok(); - error!("Received unexpected data from the provider: {chunk:?}"); - } - } else { - reader.stop(0u8.into()).ok(); - } - Ok(Stats { - elapsed: self.misc.start.elapsed(), - bytes_written: self.misc.bytes_written, - bytes_read, - }) - } - } - - /// Stuff we need to hold on to while going through the machine states - #[derive(Debug)] - struct Misc { - /// start time for statistics - start: Instant, - /// bytes written for statistics - bytes_written: u64, - /// iterator over the ranges of the collection and the children - ranges_iter: RangesIter, - } -} - -/// Error when processing a response -#[derive(thiserror::Error, Debug)] -pub enum GetResponseError { - /// Error when opening a stream - #[error("connection: {0}")] - Connection(#[from] endpoint::ConnectionError), - /// Error when writing the handshake or request to the stream - #[error("write: {0}")] - Write(#[from] endpoint::WriteError), - /// Error when reading from the stream - #[error("read: {0}")] - Read(#[from] endpoint::ReadError), - /// Error when decoding, e.g. hash mismatch - #[error("decode: {0}")] - Decode(bao_tree::io::DecodeError), - /// A generic error - #[error("generic: {0}")] - Generic(anyhow::Error), -} - -impl From for GetResponseError { - fn from(cause: postcard::Error) -> Self { - Self::Generic(cause.into()) - } -} - -impl From for GetResponseError { - fn from(cause: bao_tree::io::DecodeError) -> Self { - match cause { - bao_tree::io::DecodeError::Io(cause) => { - // try to downcast to specific quinn errors - if let Some(source) = cause.source() { - if let Some(error) = source.downcast_ref::() { - return Self::Connection(error.clone()); - } - if let Some(error) = source.downcast_ref::() { - return Self::Read(error.clone()); - } - if let Some(error) = source.downcast_ref::() { - return Self::Write(error.clone()); - } - } - Self::Generic(cause.into()) - } - _ => Self::Decode(cause), - } - } -} - -impl From for GetResponseError { - fn from(cause: anyhow::Error) -> Self { - Self::Generic(cause) - } -} - -impl From for std::io::Error { - fn from(cause: GetResponseError) -> Self { - Self::new(std::io::ErrorKind::Other, cause) - } -} diff --git a/iroh-blobs/src/get/db.rs b/iroh-blobs/src/get/db.rs deleted file mode 100644 index 7682188f0c1..00000000000 --- a/iroh-blobs/src/get/db.rs +++ /dev/null @@ -1,699 +0,0 @@ -//! Functions that use the iroh-blobs protocol in conjunction with a bao store. - -use std::{future::Future, io, num::NonZeroU64, pin::Pin}; - -use anyhow::anyhow; -use bao_tree::{ChunkNum, ChunkRanges}; -use futures_lite::StreamExt; -use genawaiter::{ - rc::{Co, Gen}, - GeneratorState, -}; -use iroh_base::{hash::Hash, rpc::RpcError}; -use iroh_io::AsyncSliceReader; -use iroh_net::endpoint::Connection; -use serde::{Deserialize, Serialize}; -use tokio::sync::oneshot; -use tracing::trace; - -use crate::{ - get::{ - self, - error::GetError, - fsm::{AtBlobHeader, AtEndBlob, ConnectedNext, EndBlobNext}, - progress::TransferState, - Stats, - }, - hashseq::parse_hash_seq, - protocol::{GetRequest, RangeSpec, RangeSpecSeq}, - store::{ - BaoBatchWriter, BaoBlobSize, FallibleProgressBatchWriter, MapEntry, MapEntryMut, MapMut, - Store as BaoStore, - }, - util::progress::{IdGenerator, ProgressSender}, - BlobFormat, HashAndFormat, -}; - -type GetGenerator = Gen>>>>; -type GetFuture = Pin> + 'static>>; - -/// Get a blob or collection into a store. -/// -/// This considers data that is already in the store, and will only request -/// the remaining data. -/// -/// Progress is reported as [`DownloadProgress`] through a [`ProgressSender`]. Note that the -/// [`DownloadProgress::AllDone`] event is not emitted from here, but left to an upper layer to send, -/// if desired. -pub async fn get_to_db< - D: BaoStore, - C: FnOnce() -> F, - F: Future>, ->( - db: &D, - get_conn: C, - hash_and_format: &HashAndFormat, - progress_sender: impl ProgressSender + IdGenerator, -) -> Result { - match get_to_db_in_steps(db.clone(), *hash_and_format, progress_sender).await? { - GetState::Complete(res) => Ok(res), - GetState::NeedsConn(state) => { - let conn = get_conn().await.map_err(GetError::Io)?; - state.proceed(conn).await - } - } -} - -/// Get a blob or collection into a store, yielding if a connection is needed. -/// -/// This checks a get request against a local store, and returns [`GetState`], -/// which is either `Complete` in case the requested data is fully available in the local store, or -/// `NeedsConn`, once a connection is needed to proceed downloading the missing data. -/// -/// In the latter case, call [`GetStateNeedsConn::proceed`] with a connection to a provider to -/// proceed with the download. -/// -/// Progress reporting works in the same way as documented in [`get_to_db`]. -pub async fn get_to_db_in_steps< - D: BaoStore, - P: ProgressSender + IdGenerator, ->( - db: D, - hash_and_format: HashAndFormat, - progress_sender: P, -) -> Result { - let mut gen: GetGenerator = genawaiter::rc::Gen::new(move |co| { - let fut = async move { producer(co, &db, &hash_and_format, progress_sender).await }; - let fut: GetFuture = Box::pin(fut); - fut - }); - match gen.async_resume().await { - GeneratorState::Yielded(Yield::NeedConn(reply)) => { - Ok(GetState::NeedsConn(GetStateNeedsConn(gen, reply))) - } - GeneratorState::Complete(res) => res.map(GetState::Complete), - } -} - -/// Intermediary state returned from [`get_to_db_in_steps`] for a download request that needs a -/// connection to proceed. -#[derive(derive_more::Debug)] -#[debug("GetStateNeedsConn")] -pub struct GetStateNeedsConn(GetGenerator, oneshot::Sender); - -impl GetStateNeedsConn { - /// Proceed with the download by providing a connection to a provider. - pub async fn proceed(mut self, conn: Connection) -> Result { - self.1.send(conn).expect("receiver is not dropped"); - match self.0.async_resume().await { - GeneratorState::Yielded(y) => match y { - Yield::NeedConn(_) => panic!("NeedsConn may only be yielded once"), - }, - GeneratorState::Complete(res) => res, - } - } -} - -/// Output of [`get_to_db_in_steps`]. -#[derive(Debug)] -pub enum GetState { - /// The requested data is completely available in the local store, no network requests are - /// needed. - Complete(Stats), - /// The requested data is not fully available in the local store, we need a connection to - /// proceed. - /// - /// Once a connection is available, call [`GetStateNeedsConn::proceed`] to continue. - NeedsConn(GetStateNeedsConn), -} - -struct GetCo(Co); - -impl GetCo { - async fn get_conn(&self) -> Connection { - let (tx, rx) = oneshot::channel(); - self.0.yield_(Yield::NeedConn(tx)).await; - rx.await.expect("sender may not be dropped") - } -} - -enum Yield { - NeedConn(oneshot::Sender), -} - -async fn producer( - co: Co, - db: &D, - hash_and_format: &HashAndFormat, - progress: impl ProgressSender + IdGenerator, -) -> Result { - let HashAndFormat { hash, format } = hash_and_format; - let co = GetCo(co); - match format { - BlobFormat::Raw => get_blob(db, co, hash, progress).await, - BlobFormat::HashSeq => get_hash_seq(db, co, hash, progress).await, - } -} - -/// Get a blob that was requested completely. -/// -/// We need to create our own files and handle the case where an outboard -/// is not needed. -async fn get_blob( - db: &D, - co: GetCo, - hash: &Hash, - progress: impl ProgressSender + IdGenerator, -) -> Result { - let end = match db.get_mut(hash).await? { - Some(entry) if entry.is_complete() => { - tracing::info!("already got entire blob"); - progress - .send(DownloadProgress::FoundLocal { - child: BlobId::Root, - hash: *hash, - size: entry.size(), - valid_ranges: RangeSpec::all(), - }) - .await?; - return Ok(Stats::default()); - } - Some(entry) => { - trace!("got partial data for {}", hash); - let valid_ranges = valid_ranges::(&entry) - .await - .ok() - .unwrap_or_else(ChunkRanges::all); - progress - .send(DownloadProgress::FoundLocal { - child: BlobId::Root, - hash: *hash, - size: entry.size(), - valid_ranges: RangeSpec::new(&valid_ranges), - }) - .await?; - let required_ranges: ChunkRanges = ChunkRanges::all().difference(&valid_ranges); - - let request = GetRequest::new(*hash, RangeSpecSeq::from_ranges([required_ranges])); - // full request - let conn = co.get_conn().await; - let request = get::fsm::start(conn, request); - // create a new bidi stream - let connected = request.next().await?; - // next step. we have requested a single hash, so this must be StartRoot - let ConnectedNext::StartRoot(start) = connected.next().await? else { - return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot"))); - }; - // move to the header - let header = start.next(); - // do the ceremony of getting the blob and adding it to the database - - get_blob_inner_partial(db, header, entry, progress).await? - } - None => { - // full request - let conn = co.get_conn().await; - let request = get::fsm::start(conn, GetRequest::single(*hash)); - // create a new bidi stream - let connected = request.next().await?; - // next step. we have requested a single hash, so this must be StartRoot - let ConnectedNext::StartRoot(start) = connected.next().await? else { - return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot"))); - }; - // move to the header - let header = start.next(); - // do the ceremony of getting the blob and adding it to the database - get_blob_inner(db, header, progress).await? - } - }; - - // we have requested a single hash, so we must be at closing - let EndBlobNext::Closing(end) = end.next() else { - return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot"))); - }; - // this closes the bidi stream. Do something with the stats? - let stats = end.next().await?; - Ok(stats) -} - -/// Given a partial entry, get the valid ranges. -pub async fn valid_ranges(entry: &D::EntryMut) -> anyhow::Result { - use tracing::trace as log; - // compute the valid range from just looking at the data file - let mut data_reader = entry.data_reader().await?; - let data_size = data_reader.size().await?; - let valid_from_data = ChunkRanges::from(..ChunkNum::full_chunks(data_size)); - // compute the valid range from just looking at the outboard file - let mut outboard = entry.outboard().await?; - let all = ChunkRanges::all(); - let mut stream = bao_tree::io::fsm::valid_outboard_ranges(&mut outboard, &all); - let mut valid_from_outboard = ChunkRanges::empty(); - while let Some(range) = stream.next().await { - valid_from_outboard |= ChunkRanges::from(range?); - } - let valid: ChunkRanges = valid_from_data.intersection(&valid_from_outboard); - log!("valid_from_data: {:?}", valid_from_data); - log!("valid_from_outboard: {:?}", valid_from_data); - Ok(valid) -} - -/// Get a blob that was requested completely. -/// -/// We need to create our own files and handle the case where an outboard -/// is not needed. -async fn get_blob_inner( - db: &D, - at_header: AtBlobHeader, - sender: impl ProgressSender + IdGenerator, -) -> Result { - // read the size. The size we get here is not verified, but since we use - // it for the tree traversal we are guaranteed not to get more than size. - let (at_content, size) = at_header.next().await?; - let hash = at_content.hash(); - let child_offset = at_content.offset(); - // get or create the partial entry - let entry = db.get_or_create(hash, size).await?; - // open the data file in any case - let bw = entry.batch_writer().await?; - // allocate a new id for progress reports for this transfer - let id = sender.new_id(); - sender - .send(DownloadProgress::Found { - id, - hash, - size, - child: BlobId::from_offset(child_offset), - }) - .await?; - let sender2 = sender.clone(); - let on_write = move |offset: u64, _length: usize| { - // if try send fails it means that the receiver has been dropped. - // in that case we want to abort the write_all_with_outboard. - sender2 - .try_send(DownloadProgress::Progress { id, offset }) - .inspect_err(|_| { - tracing::info!("aborting download of {}", hash); - })?; - Ok(()) - }; - let mut bw = FallibleProgressBatchWriter::new(bw, on_write); - // use the convenience method to write all to the batch writer - let end = at_content.write_all_batch(&mut bw).await?; - // sync the underlying storage, if needed - bw.sync().await?; - drop(bw); - db.insert_complete(entry).await?; - // notify that we are done - sender.send(DownloadProgress::Done { id }).await?; - Ok(end) -} - -/// Get a blob that was requested partially. -/// -/// We get passed the data and outboard ids. Partial downloads are only done -/// for large blobs where the outboard is present. -async fn get_blob_inner_partial( - db: &D, - at_header: AtBlobHeader, - entry: D::EntryMut, - sender: impl ProgressSender + IdGenerator, -) -> Result { - // read the size. The size we get here is not verified, but since we use - // it for the tree traversal we are guaranteed not to get more than size. - let (at_content, size) = at_header.next().await?; - // create a batch writer for the bao file - let bw = entry.batch_writer().await?; - // allocate a new id for progress reports for this transfer - let id = sender.new_id(); - let hash = at_content.hash(); - let child_offset = at_content.offset(); - sender - .send(DownloadProgress::Found { - id, - hash, - size, - child: BlobId::from_offset(child_offset), - }) - .await?; - let sender2 = sender.clone(); - let on_write = move |offset: u64, _length: usize| { - // if try send fails it means that the receiver has been dropped. - // in that case we want to abort the write_all_with_outboard. - sender2 - .try_send(DownloadProgress::Progress { id, offset }) - .inspect_err(|_| { - tracing::info!("aborting download of {}", hash); - })?; - Ok(()) - }; - let mut bw = FallibleProgressBatchWriter::new(bw, on_write); - // use the convenience method to write all to the batch writer - let at_end = at_content.write_all_batch(&mut bw).await?; - // sync the underlying storage, if needed - bw.sync().await?; - drop(bw); - // we got to the end without error, so we can mark the entry as complete - // - // caution: this assumes that the request filled all the gaps in our local - // data. We can't re-check this here since that would be very expensive. - db.insert_complete(entry).await?; - // notify that we are done - sender.send(DownloadProgress::Done { id }).await?; - Ok(at_end) -} - -/// Get information about a blob in a store. -/// -/// This will compute the valid ranges for partial blobs, so it is somewhat expensive for those. -pub async fn blob_info(db: &D, hash: &Hash) -> io::Result> { - io::Result::Ok(match db.get_mut(hash).await? { - Some(entry) if entry.is_complete() => BlobInfo::Complete { - size: entry.size().value(), - }, - Some(entry) => { - let valid_ranges = valid_ranges::(&entry) - .await - .ok() - .unwrap_or_else(ChunkRanges::all); - BlobInfo::Partial { - entry, - valid_ranges, - } - } - None => BlobInfo::Missing, - }) -} - -/// Like `get_blob_info`, but for multiple hashes -async fn blob_infos(db: &D, hash_seq: &[Hash]) -> io::Result>> { - let items = futures_lite::stream::iter(hash_seq) - .then(|hash| blob_info(db, hash)) - .collect::>(); - items.await.into_iter().collect() -} - -/// Get a sequence of hashes -async fn get_hash_seq( - db: &D, - co: GetCo, - root_hash: &Hash, - sender: impl ProgressSender + IdGenerator, -) -> Result { - use tracing::info as log; - let finishing = match db.get_mut(root_hash).await? { - Some(entry) if entry.is_complete() => { - log!("already got collection - doing partial download"); - // send info that we have the hashseq itself entirely - sender - .send(DownloadProgress::FoundLocal { - child: BlobId::Root, - hash: *root_hash, - size: entry.size(), - valid_ranges: RangeSpec::all(), - }) - .await?; - // got the collection - let reader = entry.data_reader().await?; - let (mut hash_seq, children) = parse_hash_seq(reader).await.map_err(|err| { - GetError::NoncompliantNode(anyhow!("Failed to parse downloaded HashSeq: {err}")) - })?; - sender - .send(DownloadProgress::FoundHashSeq { - hash: *root_hash, - children, - }) - .await?; - let mut children: Vec = vec![]; - while let Some(hash) = hash_seq.next().await? { - children.push(hash); - } - let missing_info = blob_infos(db, &children).await?; - // send the info about what we have - for (i, info) in missing_info.iter().enumerate() { - if let Some(size) = info.size() { - sender - .send(DownloadProgress::FoundLocal { - child: BlobId::from_offset((i as u64) + 1), - hash: children[i], - size, - valid_ranges: RangeSpec::new(info.valid_ranges()), - }) - .await?; - } - } - if missing_info - .iter() - .all(|x| matches!(x, BlobInfo::Complete { .. })) - { - log!("nothing to do"); - return Ok(Stats::default()); - } - - let missing_iter = std::iter::once(ChunkRanges::empty()) - .chain(missing_info.iter().map(|x| x.missing_ranges())) - .collect::>(); - log!("requesting chunks {:?}", missing_iter); - let request = GetRequest::new(*root_hash, RangeSpecSeq::from_ranges(missing_iter)); - let conn = co.get_conn().await; - let request = get::fsm::start(conn, request); - // create a new bidi stream - let connected = request.next().await?; - log!("connected"); - // we have not requested the root, so this must be StartChild - let ConnectedNext::StartChild(start) = connected.next().await? else { - return Err(GetError::NoncompliantNode(anyhow!("expected StartChild"))); - }; - let mut next = EndBlobNext::MoreChildren(start); - // read all the children - loop { - let start = match next { - EndBlobNext::MoreChildren(start) => start, - EndBlobNext::Closing(finish) => break finish, - }; - let child_offset = usize::try_from(start.child_offset()) - .map_err(|_| GetError::NoncompliantNode(anyhow!("child offset too large")))?; - let (child_hash, info) = - match (children.get(child_offset), missing_info.get(child_offset)) { - (Some(blob), Some(info)) => (*blob, info), - _ => break start.finish(), - }; - tracing::info!( - "requesting child {} {:?}", - child_hash, - info.missing_ranges() - ); - let header = start.next(child_hash); - let end_blob = match info { - BlobInfo::Missing => get_blob_inner(db, header, sender.clone()).await?, - BlobInfo::Partial { entry, .. } => { - get_blob_inner_partial(db, header, entry.clone(), sender.clone()).await? - } - BlobInfo::Complete { .. } => { - return Err(GetError::NoncompliantNode(anyhow!( - "got data we have not requested" - ))); - } - }; - next = end_blob.next(); - } - } - _ => { - tracing::debug!("don't have collection - doing full download"); - // don't have the collection, so probably got nothing - let conn = co.get_conn().await; - let request = get::fsm::start(conn, GetRequest::all(*root_hash)); - // create a new bidi stream - let connected = request.next().await?; - // next step. we have requested a single hash, so this must be StartRoot - let ConnectedNext::StartRoot(start) = connected.next().await? else { - return Err(GetError::NoncompliantNode(anyhow!("expected StartRoot"))); - }; - // move to the header - let header = start.next(); - // read the blob and add it to the database - let end_root = get_blob_inner(db, header, sender.clone()).await?; - // read the collection fully for now - let entry = db - .get(root_hash) - .await? - .ok_or_else(|| GetError::LocalFailure(anyhow!("just downloaded but not in db")))?; - let reader = entry.data_reader().await?; - let (mut collection, count) = parse_hash_seq(reader).await.map_err(|err| { - GetError::NoncompliantNode(anyhow!("Failed to parse downloaded HashSeq: {err}")) - })?; - sender - .send(DownloadProgress::FoundHashSeq { - hash: *root_hash, - children: count, - }) - .await?; - let mut children = vec![]; - while let Some(hash) = collection.next().await? { - children.push(hash); - } - let mut next = end_root.next(); - // read all the children - loop { - let start = match next { - EndBlobNext::MoreChildren(start) => start, - EndBlobNext::Closing(finish) => break finish, - }; - let child_offset = usize::try_from(start.child_offset()) - .map_err(|_| GetError::NoncompliantNode(anyhow!("child offset too large")))?; - - let child_hash = match children.get(child_offset) { - Some(blob) => *blob, - None => break start.finish(), - }; - let header = start.next(child_hash); - let end_blob = get_blob_inner(db, header, sender.clone()).await?; - next = end_blob.next(); - } - } - }; - // this closes the bidi stream. Do something with the stats? - let stats = finishing.next().await?; - Ok(stats) -} - -/// Information about a the status of a blob in a store. -#[derive(Debug, Clone)] -pub enum BlobInfo { - /// we have the blob completely - Complete { - /// The size of the entry in bytes. - size: u64, - }, - /// we have the blob partially - Partial { - /// The partial entry. - entry: D::EntryMut, - /// The ranges that are available locally. - valid_ranges: ChunkRanges, - }, - /// we don't have the blob at all - Missing, -} - -impl BlobInfo { - /// The size of the blob, if known. - pub fn size(&self) -> Option { - match self { - BlobInfo::Complete { size } => Some(BaoBlobSize::Verified(*size)), - BlobInfo::Partial { entry, .. } => Some(entry.size()), - BlobInfo::Missing => None, - } - } - - /// Ranges that are valid locally. - /// - /// This will be all for complete blobs, empty for missing blobs, - /// and a set with possibly open last range for partial blobs. - pub fn valid_ranges(&self) -> ChunkRanges { - match self { - BlobInfo::Complete { .. } => ChunkRanges::all(), - BlobInfo::Partial { valid_ranges, .. } => valid_ranges.clone(), - BlobInfo::Missing => ChunkRanges::empty(), - } - } - - /// Ranges that are missing locally and need to be requested. - /// - /// This will be empty for complete blobs, all for missing blobs, and - /// a set with possibly open last range for partial blobs. - pub fn missing_ranges(&self) -> ChunkRanges { - match self { - BlobInfo::Complete { .. } => ChunkRanges::empty(), - BlobInfo::Partial { valid_ranges, .. } => ChunkRanges::all().difference(valid_ranges), - BlobInfo::Missing => ChunkRanges::all(), - } - } -} - -/// Progress updates for the get operation. -// TODO: Move to super::progress -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum DownloadProgress { - /// Initial state if subscribing to a running or queued transfer. - InitialState(TransferState), - /// Data was found locally. - FoundLocal { - /// child offset - child: BlobId, - /// The hash of the entry. - hash: Hash, - /// The size of the entry in bytes. - size: BaoBlobSize, - /// The ranges that are available locally. - valid_ranges: RangeSpec, - }, - /// A new connection was established. - Connected, - /// An item was found with hash `hash`, from now on referred to via `id`. - Found { - /// A new unique progress id for this entry. - id: u64, - /// Identifier for this blob within this download. - /// - /// Will always be [`BlobId::Root`] unless a hashseq is downloaded, in which case this - /// allows to identify the children by their offset in the hashseq. - child: BlobId, - /// The hash of the entry. - hash: Hash, - /// The size of the entry in bytes. - size: u64, - }, - /// An item was found with hash `hash`, from now on referred to via `id`. - FoundHashSeq { - /// The name of the entry. - hash: Hash, - /// Number of children in the collection, if known. - children: u64, - }, - /// We got progress ingesting item `id`. - Progress { - /// The unique id of the entry. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done with `id`. - Done { - /// The unique id of the entry. - id: u64, - }, - /// All operations finished. - /// - /// This will be the last message in the stream. - AllDone(Stats), - /// We got an error and need to abort. - /// - /// This will be the last message in the stream. - Abort(RpcError), -} - -/// The id of a blob in a transfer -#[derive( - Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, std::hash::Hash, Serialize, Deserialize, -)] -pub enum BlobId { - /// The root blob (child id 0) - Root, - /// A child blob (child id > 0) - Child(NonZeroU64), -} - -impl BlobId { - fn from_offset(id: u64) -> Self { - NonZeroU64::new(id).map(Self::Child).unwrap_or(Self::Root) - } -} - -impl From for u64 { - fn from(value: BlobId) -> Self { - match value { - BlobId::Root => 0, - BlobId::Child(id) => id.into(), - } - } -} diff --git a/iroh-blobs/src/get/error.rs b/iroh-blobs/src/get/error.rs deleted file mode 100644 index 8980d04a9d6..00000000000 --- a/iroh-blobs/src/get/error.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Error returned from get operations - -use iroh_net::endpoint; - -use crate::util::progress::ProgressSendError; - -/// Failures for a get operation -#[derive(Debug, thiserror::Error)] -pub enum GetError { - /// Hash not found. - #[error("Hash not found")] - NotFound(#[source] anyhow::Error), - /// Remote has reset the connection. - #[error("Remote has reset the connection")] - RemoteReset(#[source] anyhow::Error), - /// Remote behaved in a non-compliant way. - #[error("Remote behaved in a non-compliant way")] - NoncompliantNode(#[source] anyhow::Error), - - /// Network or IO operation failed. - #[error("A network or IO operation failed")] - Io(#[source] anyhow::Error), - - /// Our download request is invalid. - #[error("Our download request is invalid")] - BadRequest(#[source] anyhow::Error), - /// Operation failed on the local node. - #[error("Operation failed on the local node")] - LocalFailure(#[source] anyhow::Error), -} - -impl From for GetError { - fn from(value: ProgressSendError) -> Self { - Self::LocalFailure(value.into()) - } -} - -impl From for GetError { - fn from(value: endpoint::ConnectionError) -> Self { - // explicit match just to be sure we are taking everything into account - use endpoint::ConnectionError; - match value { - e @ ConnectionError::VersionMismatch => { - // > The peer doesn't implement any supported version - // unsupported version is likely a long time error, so this peer is not usable - GetError::NoncompliantNode(e.into()) - } - e @ ConnectionError::TransportError(_) => { - // > The peer violated the QUIC specification as understood by this implementation - // bad peer we don't want to keep around - GetError::NoncompliantNode(e.into()) - } - e @ ConnectionError::ConnectionClosed(_) => { - // > The peer's QUIC stack aborted the connection automatically - // peer might be disconnecting or otherwise unavailable, drop it - GetError::Io(e.into()) - } - e @ ConnectionError::ApplicationClosed(_) => { - // > The peer closed the connection - // peer might be disconnecting or otherwise unavailable, drop it - GetError::Io(e.into()) - } - e @ ConnectionError::Reset => { - // > The peer is unable to continue processing this connection, usually due to having restarted - GetError::RemoteReset(e.into()) - } - e @ ConnectionError::TimedOut => { - // > Communication with the peer has lapsed for longer than the negotiated idle timeout - GetError::Io(e.into()) - } - e @ ConnectionError::LocallyClosed => { - // > The local application closed the connection - // TODO(@divma): don't see how this is reachable but let's just not use the peer - GetError::Io(e.into()) - } - e @ quinn::ConnectionError::CidsExhausted => { - // > The connection could not be created because not enough of the CID space - // > is available - GetError::Io(e.into()) - } - } - } -} - -impl From for GetError { - fn from(value: endpoint::ReadError) -> Self { - use endpoint::ReadError; - match value { - e @ ReadError::Reset(_) => GetError::RemoteReset(e.into()), - ReadError::ConnectionLost(conn_error) => conn_error.into(), - ReadError::ClosedStream - | ReadError::IllegalOrderedRead - | ReadError::ZeroRttRejected => { - // all these errors indicate the peer is not usable at this moment - GetError::Io(value.into()) - } - } - } -} -impl From for GetError { - fn from(value: quinn::ClosedStream) -> Self { - GetError::Io(value.into()) - } -} - -impl From for GetError { - fn from(value: endpoint::WriteError) -> Self { - use endpoint::WriteError; - match value { - e @ WriteError::Stopped(_) => GetError::RemoteReset(e.into()), - WriteError::ConnectionLost(conn_error) => conn_error.into(), - WriteError::ClosedStream | WriteError::ZeroRttRejected => { - // all these errors indicate the peer is not usable at this moment - GetError::Io(value.into()) - } - } - } -} - -impl From for GetError { - fn from(value: crate::get::fsm::ConnectedNextError) -> Self { - use crate::get::fsm::ConnectedNextError::*; - match value { - e @ PostcardSer(_) => { - // serialization errors indicate something wrong with the request itself - GetError::BadRequest(e.into()) - } - e @ RequestTooBig => { - // request will never be sent, drop it - GetError::BadRequest(e.into()) - } - Write(e) => e.into(), - Closed(e) => e.into(), - e @ Io(_) => { - // io errors are likely recoverable - GetError::Io(e.into()) - } - } - } -} - -impl From for GetError { - fn from(value: crate::get::fsm::AtBlobHeaderNextError) -> Self { - use crate::get::fsm::AtBlobHeaderNextError::*; - match value { - e @ NotFound => { - // > This indicates that the provider does not have the requested data. - // peer might have the data later, simply retry it - GetError::NotFound(e.into()) - } - Read(e) => e.into(), - e @ Io(_) => { - // io errors are likely recoverable - GetError::Io(e.into()) - } - } - } -} - -impl From for GetError { - fn from(value: crate::get::fsm::DecodeError) -> Self { - use crate::get::fsm::DecodeError::*; - - match value { - e @ NotFound => GetError::NotFound(e.into()), - e @ ParentNotFound(_) => GetError::NotFound(e.into()), - e @ LeafNotFound(_) => GetError::NotFound(e.into()), - e @ ParentHashMismatch(_) => { - // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong - // request? - GetError::NoncompliantNode(e.into()) - } - e @ LeafHashMismatch(_) => { - // TODO(@divma): did the peer sent wrong data? is it corrupted? did we sent a wrong - // request? - GetError::NoncompliantNode(e.into()) - } - Read(e) => e.into(), - Io(e) => e.into(), - } - } -} - -impl From for GetError { - fn from(value: std::io::Error) -> Self { - // generally consider io errors recoverable - // we might want to revisit this at some point - GetError::Io(value.into()) - } -} diff --git a/iroh-blobs/src/get/progress.rs b/iroh-blobs/src/get/progress.rs deleted file mode 100644 index d4025e5c3b8..00000000000 --- a/iroh-blobs/src/get/progress.rs +++ /dev/null @@ -1,185 +0,0 @@ -//! Types for get progress state management. - -use std::{collections::HashMap, num::NonZeroU64}; - -use serde::{Deserialize, Serialize}; -use tracing::warn; - -use super::db::{BlobId, DownloadProgress}; -use crate::{protocol::RangeSpec, store::BaoBlobSize, Hash}; - -/// The identifier for progress events. -pub type ProgressId = u64; - -/// Accumulated progress state of a transfer. -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct TransferState { - /// The root blob of this transfer (may be a hash seq), - pub root: BlobState, - /// Whether we are connected to a node - pub connected: bool, - /// Children if the root blob is a hash seq, empty for raw blobs - pub children: HashMap, - /// Child being transferred at the moment. - pub current: Option, - /// Progress ids for individual blobs. - pub progress_id_to_blob: HashMap, -} - -impl TransferState { - /// Create a new, empty transfer state. - pub fn new(root_hash: Hash) -> Self { - Self { - root: BlobState::new(root_hash), - connected: false, - children: Default::default(), - current: None, - progress_id_to_blob: Default::default(), - } - } -} - -/// State of a single blob in transfer -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct BlobState { - /// The hash of this blob. - pub hash: Hash, - /// The size of this blob. Only known if the blob is partially present locally, or after having - /// received the size from the remote. - pub size: Option, - /// The current state of the blob transfer. - pub progress: BlobProgress, - /// Ranges already available locally at the time of starting the transfer. - pub local_ranges: Option, - /// Number of children (only applies to hashseqs, None for raw blobs). - pub child_count: Option, -} - -/// Progress state for a single blob -#[derive(Debug, Default, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub enum BlobProgress { - /// Download is pending - #[default] - Pending, - /// Download is in progress - Progressing(u64), - /// Download has finished - Done, -} - -impl BlobState { - /// Create a new [`BlobState`]. - pub fn new(hash: Hash) -> Self { - Self { - hash, - size: None, - local_ranges: None, - child_count: None, - progress: BlobProgress::default(), - } - } -} - -impl TransferState { - /// Get state of the root blob of this transfer. - pub fn root(&self) -> &BlobState { - &self.root - } - - /// Get a blob state by its [`BlobId`] in this transfer. - pub fn get_blob(&self, blob_id: &BlobId) -> Option<&BlobState> { - match blob_id { - BlobId::Root => Some(&self.root), - BlobId::Child(id) => self.children.get(id), - } - } - - /// Get the blob state currently being transferred. - pub fn get_current(&self) -> Option<&BlobState> { - self.current.as_ref().and_then(|id| self.get_blob(id)) - } - - fn get_or_insert_blob(&mut self, blob_id: BlobId, hash: Hash) -> &mut BlobState { - match blob_id { - BlobId::Root => &mut self.root, - BlobId::Child(id) => self - .children - .entry(id) - .or_insert_with(|| BlobState::new(hash)), - } - } - fn get_blob_mut(&mut self, blob_id: &BlobId) -> Option<&mut BlobState> { - match blob_id { - BlobId::Root => Some(&mut self.root), - BlobId::Child(id) => self.children.get_mut(id), - } - } - - fn get_by_progress_id(&mut self, progress_id: ProgressId) -> Option<&mut BlobState> { - let blob_id = *self.progress_id_to_blob.get(&progress_id)?; - self.get_blob_mut(&blob_id) - } - - /// Update the state with a new [`DownloadProgress`] event for this transfer. - pub fn on_progress(&mut self, event: DownloadProgress) { - match event { - DownloadProgress::InitialState(s) => { - *self = s; - } - DownloadProgress::FoundLocal { - child, - hash, - size, - valid_ranges, - } => { - let blob = self.get_or_insert_blob(child, hash); - blob.size = Some(size); - blob.local_ranges = Some(valid_ranges); - } - DownloadProgress::Connected => self.connected = true, - DownloadProgress::Found { - id: progress_id, - child: blob_id, - hash, - size, - } => { - let blob = self.get_or_insert_blob(blob_id, hash); - blob.size = match blob.size { - // If we don't have a verified size for this blob yet: Use the size as reported - // by the remote. - None | Some(BaoBlobSize::Unverified(_)) => Some(BaoBlobSize::Unverified(size)), - // Otherwise, keep the existing verified size. - value @ Some(BaoBlobSize::Verified(_)) => value, - }; - blob.progress = BlobProgress::Progressing(0); - self.progress_id_to_blob.insert(progress_id, blob_id); - self.current = Some(blob_id); - } - DownloadProgress::FoundHashSeq { hash, children } => { - if hash == self.root.hash { - self.root.child_count = Some(children); - } else { - // I think it is an invariant of the protocol that `FoundHashSeq` is only - // triggered for the root hash. - warn!("Received `FoundHashSeq` event for a hash which is not the download's root hash.") - } - } - DownloadProgress::Progress { id, offset } => { - if let Some(blob) = self.get_by_progress_id(id) { - blob.progress = BlobProgress::Progressing(offset); - } else { - warn!(%id, "Received `Progress` event for unknown progress id.") - } - } - DownloadProgress::Done { id } => { - if let Some(blob) = self.get_by_progress_id(id) { - blob.progress = BlobProgress::Done; - self.progress_id_to_blob.remove(&id); - } else { - warn!(%id, "Received `Done` event for unknown progress id.") - } - } - DownloadProgress::AllDone(_) | DownloadProgress::Abort(_) => {} - } - } -} diff --git a/iroh-blobs/src/get/request.rs b/iroh-blobs/src/get/request.rs deleted file mode 100644 index 871b8501e1a..00000000000 --- a/iroh-blobs/src/get/request.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Utilities for complex get requests. -use std::sync::Arc; - -use bao_tree::{ChunkNum, ChunkRanges}; -use bytes::Bytes; -use iroh_net::endpoint::Connection; -use rand::Rng; - -use super::{fsm, Stats}; -use crate::{ - hashseq::HashSeq, - protocol::{GetRequest, RangeSpecSeq}, - Hash, HashAndFormat, -}; - -/// Get the claimed size of a blob from a peer. -/// -/// This is just reading the size header and then immediately closing the connection. -/// It can be used to check if a peer has any data at all. -pub async fn get_unverified_size( - connection: &Connection, - hash: &Hash, -) -> anyhow::Result<(u64, Stats)> { - let request = GetRequest::new( - *hash, - RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), - ); - let request = fsm::start(connection.clone(), request); - let connected = request.next().await?; - let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { - unreachable!("expected start root"); - }; - let at_blob_header = start.next(); - let (curr, size) = at_blob_header.next().await?; - let stats = curr.finish().next().await?; - Ok((size, stats)) -} - -/// Get the verified size of a blob from a peer. -/// -/// This asks for the last chunk of the blob and validates the response. -/// Note that this does not validate that the peer has all the data. -pub async fn get_verified_size( - connection: &Connection, - hash: &Hash, -) -> anyhow::Result<(u64, Stats)> { - tracing::trace!("Getting verified size of {}", hash.to_hex()); - let request = GetRequest::new( - *hash, - RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), - ); - let request = fsm::start(connection.clone(), request); - let connected = request.next().await?; - let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { - unreachable!("expected start root"); - }; - let header = start.next(); - let (mut curr, size) = header.next().await?; - let end = loop { - match curr.next().await { - fsm::BlobContentNext::More((next, res)) => { - let _ = res?; - curr = next; - } - fsm::BlobContentNext::Done(end) => { - break end; - } - } - }; - let fsm::EndBlobNext::Closing(closing) = end.next() else { - unreachable!("expected closing"); - }; - let stats = closing.next().await?; - tracing::trace!( - "Got verified size of {}, {:.6}s", - hash.to_hex(), - stats.elapsed.as_secs_f64() - ); - Ok((size, stats)) -} - -/// Given a hash of a hash seq, get the hash seq and the verified sizes of its -/// children. -/// -/// This can be used to compute the total size when requesting a hash seq. -pub async fn get_hash_seq_and_sizes( - connection: &Connection, - hash: &Hash, - max_size: u64, -) -> anyhow::Result<(HashSeq, Arc<[u64]>)> { - let content = HashAndFormat::hash_seq(*hash); - tracing::debug!("Getting hash seq and children sizes of {}", content); - let request = GetRequest::new( - *hash, - RangeSpecSeq::from_ranges_infinite([ - ChunkRanges::all(), - ChunkRanges::from(ChunkNum(u64::MAX)..), - ]), - ); - let at_start = fsm::start(connection.clone(), request); - let at_connected = at_start.next().await?; - let fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { - unreachable!("query includes root"); - }; - let at_start_root = start.next(); - let (at_blob_content, size) = at_start_root.next().await?; - // check the size to avoid parsing a maliciously large hash seq - if size > max_size { - anyhow::bail!("size too large"); - } - let (mut curr, hash_seq) = at_blob_content.concatenate_into_vec().await?; - let hash_seq = HashSeq::try_from(Bytes::from(hash_seq))?; - let mut sizes = Vec::with_capacity(hash_seq.len()); - let closing = loop { - match curr.next() { - fsm::EndBlobNext::MoreChildren(more) => { - let hash = match hash_seq.get(sizes.len()) { - Some(hash) => hash, - None => break more.finish(), - }; - let at_header = more.next(hash); - let (at_content, size) = at_header.next().await?; - let next = at_content.drain().await?; - sizes.push(size); - curr = next; - } - fsm::EndBlobNext::Closing(closing) => break closing, - } - }; - let _stats = closing.next().await?; - tracing::debug!( - "Got hash seq and children sizes of {}: {:?}", - content, - sizes - ); - Ok((hash_seq, sizes.into())) -} - -/// Probe for a single chunk of a blob. -/// -/// This is used to check if a peer has a specific chunk. -pub async fn get_chunk_probe( - connection: &Connection, - hash: &Hash, - chunk: ChunkNum, -) -> anyhow::Result { - let ranges = ChunkRanges::from(chunk..chunk + 1); - let ranges = RangeSpecSeq::from_ranges([ranges]); - let request = GetRequest::new(*hash, ranges); - let request = fsm::start(connection.clone(), request); - let connected = request.next().await?; - let fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { - unreachable!("query includes root"); - }; - let header = start.next(); - let (mut curr, _size) = header.next().await?; - let end = loop { - match curr.next().await { - fsm::BlobContentNext::More((next, res)) => { - res?; - curr = next; - } - fsm::BlobContentNext::Done(end) => { - break end; - } - } - }; - let fsm::EndBlobNext::Closing(closing) = end.next() else { - unreachable!("query contains only one blob"); - }; - let stats = closing.next().await?; - Ok(stats) -} - -/// Given a sequence of sizes of children, generate a range spec that selects a -/// random chunk of a random child. -/// -/// The random chunk is chosen uniformly from the chunks of the children, so -/// larger children are more likely to be selected. -pub fn random_hash_seq_ranges(sizes: &[u64], mut rng: impl Rng) -> RangeSpecSeq { - let total_chunks = sizes - .iter() - .map(|size| ChunkNum::full_chunks(*size).0) - .sum::(); - let random_chunk = rng.gen_range(0..total_chunks); - let mut remaining = random_chunk; - let mut ranges = vec![]; - ranges.push(ChunkRanges::empty()); - for size in sizes.iter() { - let chunks = ChunkNum::full_chunks(*size).0; - if remaining < chunks { - ranges.push(ChunkRanges::from( - ChunkNum(remaining)..ChunkNum(remaining + 1), - )); - break; - } else { - remaining -= chunks; - ranges.push(ChunkRanges::empty()); - } - } - RangeSpecSeq::from_ranges(ranges) -} diff --git a/iroh-blobs/src/hashseq.rs b/iroh-blobs/src/hashseq.rs deleted file mode 100644 index f77f8b81dfa..00000000000 --- a/iroh-blobs/src/hashseq.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! traits related to collections of blobs -use std::{fmt::Debug, io}; - -use bytes::Bytes; -use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; - -use crate::Hash; - -/// A sequence of links, backed by a [`Bytes`] object. -#[derive(Debug, Clone, derive_more::Into)] -pub struct HashSeq(Bytes); - -impl FromIterator for HashSeq { - fn from_iter>(iter: T) -> Self { - let iter = iter.into_iter(); - let (lower, _upper) = iter.size_hint(); - let mut bytes = Vec::with_capacity(lower * 32); - for hash in iter { - bytes.extend_from_slice(hash.as_ref()); - } - Self(bytes.into()) - } -} - -impl TryFrom for HashSeq { - type Error = anyhow::Error; - - fn try_from(bytes: Bytes) -> Result { - Self::new(bytes).ok_or_else(|| anyhow::anyhow!("invalid hash sequence")) - } -} - -impl IntoIterator for HashSeq { - type Item = Hash; - type IntoIter = HashSeqIter; - - fn into_iter(self) -> Self::IntoIter { - HashSeqIter(self) - } -} - -/// Stream over the hashes in a [`HashSeq`]. -/// -/// todo: make this wrap a reader instead of a [`HashSeq`]. -#[derive(Debug, Clone)] -pub struct HashSeqStream(HashSeq); - -impl HashSeqStream { - /// Get the next hash in the sequence. - #[allow(clippy::should_implement_trait, clippy::unused_async)] - pub async fn next(&mut self) -> io::Result> { - Ok(self.0.pop_front()) - } - - /// Skip a number of hashes in the sequence. - #[allow(clippy::unused_async)] - pub async fn skip(&mut self, n: u64) -> io::Result<()> { - let ok = self.0.drop_front(n as usize); - if !ok { - Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "end of sequence", - )) - } else { - Ok(()) - } - } -} - -impl HashSeq { - /// Create a new sequence of hashes. - pub fn new(bytes: Bytes) -> Option { - if bytes.len() % 32 == 0 { - Some(Self(bytes)) - } else { - None - } - } - - fn drop_front(&mut self, n: usize) -> bool { - let start = n * 32; - if start > self.0.len() { - false - } else { - self.0 = self.0.slice(start..); - true - } - } - - /// Iterate over the hashes in this sequence. - pub fn iter(&self) -> impl Iterator + '_ { - self.0.chunks_exact(32).map(|chunk| { - let hash: [u8; 32] = chunk.try_into().unwrap(); - hash.into() - }) - } - - /// Get the number of hashes in this sequence. - pub fn len(&self) -> usize { - self.0.len() / 32 - } - - /// Check if this sequence is empty. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Get the hash at the given index. - pub fn get(&self, index: usize) -> Option { - if index < self.len() { - let hash: [u8; 32] = self.0[index * 32..(index + 1) * 32].try_into().unwrap(); - Some(hash.into()) - } else { - None - } - } - - /// Get and remove the first hash in this sequence. - pub fn pop_front(&mut self) -> Option { - if self.is_empty() { - None - } else { - let hash = self.get(0).unwrap(); - self.0 = self.0.slice(32..); - Some(hash) - } - } - - /// Get the underlying bytes. - pub fn into_inner(self) -> Bytes { - self.0 - } -} - -/// Iterator over the hashes in a [`HashSeq`]. -#[derive(Debug, Clone)] -pub struct HashSeqIter(HashSeq); - -impl Iterator for HashSeqIter { - type Item = Hash; - - fn next(&mut self) -> Option { - self.0.pop_front() - } -} - -/// Parse a sequence of hashes. -pub async fn parse_hash_seq<'a, R: AsyncSliceReader + 'a>( - mut reader: R, -) -> anyhow::Result<(HashSeqStream, u64)> { - let bytes = reader.read_to_end().await?; - let hashes = HashSeq::try_from(bytes)?; - let num_hashes = hashes.len() as u64; - let stream = HashSeqStream(hashes); - Ok((stream, num_hashes)) -} diff --git a/iroh-blobs/src/lib.rs b/iroh-blobs/src/lib.rs deleted file mode 100644 index 2821cdf3ef6..00000000000 --- a/iroh-blobs/src/lib.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Blobs layer for iroh. -//! -//! The crate is designed to be used from the [iroh] crate, which provides a -//! [high level interface](https://docs.rs/iroh/latest/iroh/client/blobs/index.html), -//! but can also be used standalone. -//! -//! It implements a [protocol] for streaming content-addressed data transfer using -//! [BLAKE3] verified streaming. -//! -//! It also provides a [store] interface for storage of blobs and outboards, -//! as well as a [persistent](crate::store::fs) and a [memory](crate::store::mem) -//! store implementation. -//! -//! To implement a server, the [provider] module provides helpers for handling -//! connections and individual requests given a store. -//! -//! To perform get requests, the [get] module provides utilities to perform -//! requests and store the result in a store, as well as a low level state -//! machine for executing requests. -//! -//! The [downloader] module provides a component to download blobs from -//! multiple sources and store them in a store. -//! -//! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf -//! [iroh]: https://docs.rs/iroh -#![deny(missing_docs, rustdoc::broken_intra_doc_links)] -#![recursion_limit = "256"] -#![cfg_attr(iroh_docsrs, feature(doc_cfg))] - -#[cfg(feature = "downloader")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "downloader")))] -pub mod downloader; -pub mod export; -pub mod format; -pub mod get; -pub mod hashseq; -pub mod metrics; -pub mod protocol; -pub mod provider; -pub mod store; -pub mod util; - -use bao_tree::BlockSize; -pub use iroh_base::hash::{BlobFormat, Hash, HashAndFormat}; - -pub use crate::util::{Tag, TempTag}; - -/// Block size used by iroh, 2^4*1024 = 16KiB -pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4); diff --git a/iroh-blobs/src/metrics.rs b/iroh-blobs/src/metrics.rs deleted file mode 100644 index d44f70cba6b..00000000000 --- a/iroh-blobs/src/metrics.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Metrics for iroh-blobs - -use iroh_metrics::{ - core::{Counter, Metric}, - struct_iterable::Iterable, -}; - -/// Enum of metrics for the module -#[allow(missing_docs)] -#[derive(Debug, Clone, Iterable)] -pub struct Metrics { - pub download_bytes_total: Counter, - pub download_time_total: Counter, - pub downloads_success: Counter, - pub downloads_error: Counter, - pub downloads_notfound: Counter, - - pub downloader_tick_main: Counter, - pub downloader_tick_connection_ready: Counter, - pub downloader_tick_message_received: Counter, - pub downloader_tick_transfer_completed: Counter, - pub downloader_tick_transfer_failed: Counter, - pub downloader_tick_retry_node: Counter, - pub downloader_tick_goodbye_node: Counter, -} - -impl Default for Metrics { - fn default() -> Self { - Self { - download_bytes_total: Counter::new("Total number of content bytes downloaded"), - download_time_total: Counter::new("Total time in ms spent downloading content bytes"), - downloads_success: Counter::new("Total number of successful downloads"), - downloads_error: Counter::new("Total number of downloads failed with error"), - downloads_notfound: Counter::new("Total number of downloads failed with not found"), - - downloader_tick_main: Counter::new( - "Number of times the main downloader actor loop ticked", - ), - downloader_tick_connection_ready: Counter::new( - "Number of times the downloader actor ticked for a connection ready", - ), - downloader_tick_message_received: Counter::new( - "Number of times the downloader actor ticked for a message received", - ), - downloader_tick_transfer_completed: Counter::new( - "Number of times the downloader actor ticked for a transfer completed", - ), - downloader_tick_transfer_failed: Counter::new( - "Number of times the downloader actor ticked for a transfer failed", - ), - downloader_tick_retry_node: Counter::new( - "Number of times the downloader actor ticked for a retry node", - ), - downloader_tick_goodbye_node: Counter::new( - "Number of times the downloader actor ticked for a goodbye node", - ), - } - } -} - -impl Metric for Metrics { - fn name() -> &'static str { - "iroh-blobs" - } -} diff --git a/iroh-blobs/src/protocol.rs b/iroh-blobs/src/protocol.rs deleted file mode 100644 index 9f24b72177a..00000000000 --- a/iroh-blobs/src/protocol.rs +++ /dev/null @@ -1,516 +0,0 @@ -//! Protocol for transferring content-addressed blobs and collections over quic -//! connections. This can be used either with normal quic connections when using -//! the [quinn](https://crates.io/crates/quinn) crate or with magicsock connections -//! when using the [iroh-net](https://crates.io/crates/iroh-net) crate. -//! -//! # Participants -//! -//! The protocol is a request/response protocol with two parties, a *provider* that -//! serves blobs and a *getter* that requests blobs. -//! -//! # Goals -//! -//! - Be paranoid about data integrity. -//! -//! Data integrity is considered more important than performance. Data will be validated both on -//! the provider and getter side. A well behaved provider will never send invalid data. Responses -//! to range requests contain sufficient information to validate the data. -//! -//! Note: Validation using blake3 is extremely fast, so in almost all scenarios the validation -//! will not be the bottleneck even if we validate both on the provider and getter side. -//! -//! - Do not limit the size of blobs or collections. -//! -//! Blobs can be of arbitrary size, up to terabytes. Likewise, collections can contain an -//! arbitrary number of links. A well behaved implementation will not require the entire blob or -//! collection to be in memory at once. -//! -//! - Be efficient when transferring large blobs, including range requests. -//! -//! It is possible to request entire blobs or ranges of blobs, where the minimum granularity is a -//! chunk group of 16KiB or 16 blake3 chunks. The worst case overhead when doing range requests -//! is about two chunk groups per range. -//! -//! - Be efficient when transferring multiple tiny blobs. -//! -//! For tiny blobs the overhead of sending the blob hashes and the round-trip time for each blob -//! would be prohibitive. -//! -//! To avoid roundtrips, the protocol allows grouping multiple blobs into *collections*. -//! The semantic meaning of a collection is up to the application. For the purpose -//! of this protocol, a collection is just a grouping of related blobs. -//! -//! # Non-goals -//! -//! - Do not attempt to be generic in terms of the used hash function. -//! -//! The protocol makes extensive use of the [blake3](https://crates.io/crates/blake3) hash -//! function and it's special properties such as blake3 verified streaming. -//! -//! - Do not support graph traversal. -//! -//! The protocol only supports collections that directly contain blobs. If you have deeply nested -//! graph data, you will need to either do multiple requests or flatten the graph into a single -//! temporary collection. -//! -//! - Do not support discovery. -//! -//! The protocol does not yet have a discovery mechanism for asking the provider what ranges are -//! available for a given blob. Currently you have to have some out-of-band knowledge about what -//! node has data for a given hash, or you can just try to retrieve the data and see if it is -//! available. -//! -//! A discovery protocol is planned in the future though. -//! -//! # Requests -//! -//! ## Getter defined requests -//! -//! In this case the getter knows the hash of the blob it wants to retrieve and -//! whether it wants to retrieve a single blob or a collection. -//! -//! The getter needs to define exactly what it wants to retrieve and send the -//! request to the provider. -//! -//! The provider will then respond with the bao encoded bytes for the requested -//! data and then close the connection. It will immediately close the connection -//! in case some data is not available or invalid. -//! -//! ## Provider defined requests -//! -//! In this case the getter sends a blob to the provider. This blob can contain -//! some kind of query. The exact details of the query are up to the application. -//! -//! The provider evaluates the query and responds with a serialized request in -//! the same format as the getter defined requests, followed by the bao encoded -//! data. From then on the protocol is the same as for getter defined requests. -//! -//! ## Specifying the required data -//! -//! A [`GetRequest`] contains a hash and a specification of what data related to -//! that hash is required. The specification is using a [`RangeSpecSeq`] which -//! has a compact representation on the wire but is otherwise identical to a -//! sequence of sets of ranges. -//! -//! In the following, we describe how the [`RangeSpecSeq`] is to be created for -//! different common scenarios. -//! -//! Ranges are always given in terms of 1024 byte blake3 chunks, *not* in terms -//! of bytes or chunk groups. The reason for this is that chunks are the fundamental -//! unit of hashing in blake3. Addressing anything smaller than a chunk is not -//! possible, and combining multiple chunks is merely an optimization to reduce -//! metadata overhead. -//! -//! ### Individual blobs -//! -//! In the easiest case, the getter just wants to retrieve a single blob. In this -//! case, the getter specifies [`RangeSpecSeq`] that contains a single element. -//! This element is the set of all chunks to indicate that we -//! want the entire blob, no matter how many chunks it has. -//! -//! Since this is a very common case, there is a convenience method -//! [`GetRequest::single`] that only requires the hash of the blob. -//! -//! ```rust -//! # use iroh_blobs::protocol::GetRequest; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let request = GetRequest::single(hash); -//! ``` -//! -//! ### Ranges of blobs -//! -//! In this case, we have a (possibly large) blob and we want to retrieve only -//! some ranges of chunks. This is useful in similar cases as HTTP range requests. -//! -//! We still need just a single element in the [`RangeSpecSeq`], since we are -//! still only interested in a single blob. However, this element contains all -//! the chunk ranges we want to retrieve. -//! -//! For example, if we want to retrieve chunks 0-10 of a blob, we would -//! create a [`RangeSpecSeq`] like this: -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let spec = RangeSpecSeq::from_ranges([ChunkRanges::from(..ChunkNum(10))]); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! Here `ChunkNum` is a newtype wrapper around `u64` that is used to indicate -//! that we are talking about chunk numbers, not bytes. -//! -//! While not that common, it is also possible to request multiple ranges of a -//! single blob. For example, if we want to retrieve chunks `0-10` and `100-110` -//! of a large file, we would create a [`RangeSpecSeq`] like this: -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let ranges = &ChunkRanges::from(..ChunkNum(10)) | &ChunkRanges::from(ChunkNum(100)..ChunkNum(110)); -//! let spec = RangeSpecSeq::from_ranges([ranges]); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! To specify chunk ranges, we use the [`ChunkRanges`] type alias. -//! This is actually the [`RangeSet`] type from the -//! [range_collections](https://crates.io/crates/range_collections) crate. This -//! type supports efficient boolean operations on sets of non-overlapping ranges. -//! -//! The [`RangeSet2`] type is a type alias for [`RangeSet`] that can store up to -//! 2 boundaries without allocating. This is sufficient for most use cases. -//! -//! [`RangeSet`]: range_collections::range_set::RangeSet -//! [`RangeSet2`]: range_collections::range_set::RangeSet2 -//! -//! ### Collections -//! -//! In this case the provider has a collection that contains multiple blobs. -//! We want to retrieve all blobs in the collection. -//! -//! When used for collections, the first element of a [`RangeSpecSeq`] refers -//! to the collection itself, and all subsequent elements refer to the blobs -//! in the collection. When a [`RangeSpecSeq`] specifies ranges for more than -//! one blob, the provider will interpret this as a request for a collection. -//! -//! One thing to note is that we might not yet know how many blobs are in the -//! collection. Therefore, it is not possible to download an entire collection -//! by just specifying [`ChunkRanges::all()`] for all children. -//! -//! Instead, [`RangeSpecSeq`] allows defining infinite sequences of range sets. -//! The [`RangeSpecSeq::all()`] method returns a [`RangeSpecSeq`] that, when iterated -//! over, will yield [`ChunkRanges::all()`] forever. -//! -//! So specifying a collection would work like this: -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let spec = RangeSpecSeq::all(); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! Downloading an entire collection is also a very common case, so there is a -//! convenience method [`GetRequest::all`] that only requires the hash of the -//! collection. -//! -//! ### Parts of collections -//! -//! The most complex common case is when we have retrieved a collection and -//! it's children, but were interrupted before we could retrieve all children. -//! -//! In this case we need to specify the collection we want to retrieve, but -//! exclude the children and parts of children that we already have. -//! -//! For example, if we have a collection with 3 children, and we already have -//! the first child and the first 1000000 chunks of the second child. -//! -//! We would create a [`GetRequest`] like this: -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let spec = RangeSpecSeq::from_ranges([ -//! ChunkRanges::empty(), // we don't need the collection itself -//! ChunkRanges::empty(), // we don't need the first child either -//! ChunkRanges::from(ChunkNum(1000000)..), // we need the second child from chunk 1000000 onwards -//! ChunkRanges::all(), // we need the third child completely -//! ]); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! ### Requesting chunks for each child -//! -//! The RangeSpecSeq allows some scenarios that are not covered above. E.g. you -//! might want to request a collection and the first chunk of each child blob to -//! do something like mime type detection. -//! -//! You do not know how many children the collection has, so you need to use -//! an infinite sequence. -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let spec = RangeSpecSeq::from_ranges_infinite([ -//! ChunkRanges::all(), // the collection itself -//! ChunkRanges::from(..ChunkNum(1)), // the first chunk of each child -//! ]); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! ### Requesting a single child -//! -//! It is of course possible to request a single child of a collection. E.g. -//! the following would download the second child of a collection: -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let hash: iroh_blobs::Hash = [0; 32].into(); -//! let spec = RangeSpecSeq::from_ranges([ -//! ChunkRanges::empty(), // we don't need the collection itself -//! ChunkRanges::empty(), // we don't need the first child either -//! ChunkRanges::all(), // we need the second child completely -//! ]); -//! let request = GetRequest::new(hash, spec); -//! ``` -//! -//! However, if you already have the collection, you might as well locally -//! look up the hash of the child and request it directly. -//! -//! ```rust -//! # use bao_tree::{ChunkNum, ChunkRanges}; -//! # use iroh_blobs::protocol::{GetRequest, RangeSpecSeq}; -//! # let child_hash: iroh_blobs::Hash = [0; 32].into(); -//! let request = GetRequest::single(child_hash); -//! ``` -//! -//! ### Why RangeSpec and RangeSpecSeq? -//! -//! You might wonder why we have [`RangeSpec`] and [`RangeSpecSeq`], when a simple -//! sequence of [`ChunkRanges`] might also do. -//! -//! The [`RangeSpec`] and [`RangeSpecSeq`] types exist to provide an efficient -//! representation of the request on the wire. In the [`RangeSpec`] type, -//! sequences of ranges are encoded alternating intervals of selected and -//! non-selected chunks. This results in smaller numbers that will result in fewer bytes -//! on the wire when using the [postcard](https://crates.io/crates/postcard) encoding -//! format that uses variable length integers. -//! -//! Likewise, the [`RangeSpecSeq`] type is a sequence of [`RangeSpec`]s that -//! does run length encoding to remove repeating elements. It also allows infinite -//! sequences of [`RangeSpec`]s to be encoded, unlike a simple sequence of -//! [`ChunkRanges`]s. -//! -//! [`RangeSpecSeq`] should be efficient even in case of very fragmented availability -//! of chunks, like a download from multiple providers that was frequently interrupted. -//! -//! # Responses -//! -//! The response stream contains the bao encoded bytes for the requested data. -//! The data will be sent in the order in which it was requested, so ascending -//! chunks for each blob, and blobs in the order in which they appear in the -//! collection. -//! -//! For details on the bao encoding, see the [bao specification](https://github.com/oconnor663/bao/blob/master/docs/spec.md) -//! and the [bao-tree](https://crates.io/crates/bao-tree) crate. The bao-tree crate -//! is identical to the bao crate, except that it allows combining multiple blake3 -//! chunks to chunk groups for efficiency. -//! -//! As a consequence of the chunk group optimization, chunk ranges in the response -//! will be rounded up to chunk groups ranges, so e.g. if you ask for chunks 0..10, -//! you will get chunks 0-16. This is done to reduce metadata overhead, and might -//! change in the future. -//! -//! For a complete response, the chunks are guaranteed to completely cover the -//! requested ranges. -//! -//! Reasons for not retrieving a complete response are two-fold: -//! -//! - the connection to the provider was interrupted, or the provider encountered -//! an internal error. In this case the provider will close the entire quinn connection. -//! -//! - the provider does not have the requested data, or discovered on send that the -//! requested data is not valid. -//! -//! In this case the provider will close just the stream used to send the response. -//! The exact location of the missing data can be retrieved from the error. -//! -//! # Requesting multiple unrelated blobs -//! -//! Currently, the protocol does not support requesting multiple unrelated blobs -//! in a single request. As an alternative, you can create a collection -//! on the provider side and use that to efficiently retrieve the blobs. -//! -//! If that is not possible, you can create a custom request handler that -//! accepts a custom request struct that contains the hashes of the blobs. -//! -//! If neither of these options are possible, you have no choice but to do -//! multiple requests. However, note that multiple requests will be multiplexed -//! over a single connection, and the overhead of a new QUIC stream on an existing -//! connection is very low. -//! -//! In case nodes are permanently exchanging data, it is probably valuable to -//! keep a connection open and reuse it for multiple requests. -use bao_tree::{ChunkNum, ChunkRanges}; -use derive_more::From; -use iroh_net::endpoint::VarInt; -use serde::{Deserialize, Serialize}; -mod range_spec; -pub use range_spec::{NonEmptyRequestRangeSpecIter, RangeSpec, RangeSpecSeq}; - -use crate::Hash; - -/// Maximum message size is limited to 100MiB for now. -pub const MAX_MESSAGE_SIZE: usize = 1024 * 1024 * 100; - -/// The ALPN used with quic for the iroh bytes protocol. -pub const ALPN: &[u8] = b"/iroh-bytes/4"; - -#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, From)] -/// A request to the provider -pub enum Request { - /// A get request for a blob or collection - Get(GetRequest), -} - -/// A request -#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] -pub struct GetRequest { - /// blake3 hash - pub hash: Hash, - /// The range of data to request - /// - /// The first element is the parent, all subsequent elements are children. - pub ranges: RangeSpecSeq, -} - -impl GetRequest { - /// Request a blob or collection with specified ranges - pub fn new(hash: Hash, ranges: RangeSpecSeq) -> Self { - Self { hash, ranges } - } - - /// Request a collection and all its children - pub fn all(hash: Hash) -> Self { - Self { - hash, - ranges: RangeSpecSeq::all(), - } - } - - /// Request just a single blob - pub fn single(hash: Hash) -> Self { - Self { - hash, - ranges: RangeSpecSeq::from_ranges([ChunkRanges::all()]), - } - } - - /// Request the last chunk of a single blob - /// - /// This can be used to get the verified size of a blob. - pub fn last_chunk(hash: Hash) -> Self { - Self { - hash, - ranges: RangeSpecSeq::from_ranges([ChunkRanges::from(ChunkNum(u64::MAX)..)]), - } - } - - /// Request the last chunk for all children - /// - /// This can be used to get the verified size of all children. - pub fn last_chunks(hash: Hash) -> Self { - Self { - hash, - ranges: RangeSpecSeq::from_ranges_infinite([ - ChunkRanges::all(), - ChunkRanges::from(ChunkNum(u64::MAX)..), - ]), - } - } -} - -/// Reasons to close connections or stop streams. -/// -/// A QUIC **connection** can be *closed* and a **stream** can request the other side to -/// *stop* sending data. Both closing and stopping have an associated `error_code`, closing -/// also adds a `reason` as some arbitrary bytes. -/// -/// This enum exists so we have a single namespace for `error_code`s used. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum Closed { - /// The [`RecvStream`] was dropped. - /// - /// Used implicitly when a [`RecvStream`] is dropped without explicit call to - /// [`RecvStream::stop`]. We don't use this explicitly but this is here as - /// documentation as to what happened to `0`. - /// - /// [`RecvStream`]: iroh_net::endpoint::RecvStream - /// [`RecvStream::stop`]: iroh_net::endpoint::RecvStream::stop - StreamDropped = 0, - /// The provider is terminating. - /// - /// When a provider terminates all connections and associated streams are closed. - ProviderTerminating = 1, - /// The provider has received the request. - /// - /// Only a single request is allowed on a stream, if more data is received after this a - /// provider may send this error code in a STOP_STREAM frame. - RequestReceived = 2, -} - -impl Closed { - /// The close reason as bytes. This is a valid utf8 string describing the reason. - pub fn reason(&self) -> &'static [u8] { - match self { - Closed::StreamDropped => b"stream dropped", - Closed::ProviderTerminating => b"provider terminating", - Closed::RequestReceived => b"request received", - } - } -} - -impl From for VarInt { - fn from(source: Closed) -> Self { - VarInt::from(source as u16) - } -} - -/// Unknown error_code, can not be converted into [`Closed`]. -#[derive(thiserror::Error, Debug)] -#[error("Unknown error_code: {0}")] -pub struct UnknownErrorCode(u64); - -impl TryFrom for Closed { - type Error = UnknownErrorCode; - - fn try_from(value: VarInt) -> std::result::Result { - match value.into_inner() { - 0 => Ok(Self::StreamDropped), - 1 => Ok(Self::ProviderTerminating), - 2 => Ok(Self::RequestReceived), - val => Err(UnknownErrorCode(val)), - } - } -} - -#[cfg(test)] -mod tests { - use iroh_test::{assert_eq_hex, hexdump::parse_hexdump}; - - use super::{GetRequest, Request}; - - #[test] - fn request_wire_format() { - let hash = [0xda; 32].into(); - let cases = [ - ( - Request::from(GetRequest::single(hash)), - r" - 00 # enum variant for GetRequest - dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada # the hash - 020001000100 # the RangeSpecSeq - ", - ), - ( - Request::from(GetRequest::all(hash)), - r" - 00 # enum variant for GetRequest - dadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadada # the hash - 01000100 # the RangeSpecSeq - ", - ), - ]; - for (case, expected_hex) in cases { - let expected = parse_hexdump(expected_hex).unwrap(); - let bytes = postcard::to_stdvec(&case).unwrap(); - assert_eq_hex!(bytes, expected); - } - } -} diff --git a/iroh-blobs/src/protocol/range_spec.rs b/iroh-blobs/src/protocol/range_spec.rs deleted file mode 100644 index fcf608aa1d2..00000000000 --- a/iroh-blobs/src/protocol/range_spec.rs +++ /dev/null @@ -1,560 +0,0 @@ -//! Specifications for ranges selection in blobs and sequences of blobs. -//! -//! The [`RangeSpec`] allows specifying which BAO chunks inside a single blob should be -//! selected. -//! -//! The [`RangeSpecSeq`] builds on top of this to select blob chunks in an entire -//! collection. -use std::fmt; - -use bao_tree::{ChunkNum, ChunkRanges, ChunkRangesRef}; -use serde::{Deserialize, Serialize}; -use smallvec::{smallvec, SmallVec}; - -/// A chunk range specification as a sequence of chunk offsets. -/// -/// Offsets encode alternating spans starting on 0, where the first span is always -/// deselected. -/// -/// ## Examples: -/// -/// - `[2, 5, 3, 1]` encodes five spans, of which two are selected: -/// - `[0, 0+2) = [0, 2)` is not selected. -/// - `[2, 2+5) = [2, 7)` is selected. -/// - `[7, 7+3) = [7, 10)` is not selected. -/// - `[10, 10+1) = [10, 11)` is selected. -/// - `[11, inf)` is deselected. -/// -/// Such a [`RangeSpec`] can be converted to a [`ChunkRanges`] using containing just the -/// selected ranges: `ChunkRanges{2..7, 10..11}` using [`RangeSpec::to_chunk_ranges`]. -/// -/// - An empty range selects no spans, encoded as `[]`. This means nothing of the blob is -/// selected. -/// -/// - To select an entire blob create a single half-open span starting at the first chunk: -/// `[0]`. -/// -/// - To select the tail of a blob, create a single half-open span: `[15]`. -/// -/// This is a SmallVec so we can avoid allocations for the very common case of a single -/// chunk range. -#[derive(Deserialize, Serialize, PartialEq, Eq, Clone, Hash)] -#[repr(transparent)] -pub struct RangeSpec(SmallVec<[u64; 2]>); - -impl RangeSpec { - /// Creates a new [`RangeSpec`] from a range set. - pub fn new(ranges: impl AsRef) -> Self { - let ranges = ranges.as_ref().boundaries(); - let mut res = SmallVec::new(); - if let Some((start, rest)) = ranges.split_first() { - let mut prev = start.0; - res.push(prev); - for v in rest { - res.push(v.0 - prev); - prev = v.0; - } - } - Self(res) - } - - /// A [`RangeSpec`] selecting nothing from the blob. - /// - /// This is called "empty" because the representation is an empty set. - pub const EMPTY: Self = Self(SmallVec::new_const()); - - /// Creates a [`RangeSpec`] selecting the entire blob. - pub fn all() -> Self { - Self(smallvec![0]) - } - - /// Checks if this [`RangeSpec`] does not select any chunks in the blob. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Checks if this [`RangeSpec`] selects all chunks in the blob. - pub fn is_all(&self) -> bool { - self.0.len() == 1 && self.0[0] == 0 - } - - /// Creates a [`ChunkRanges`] from this [`RangeSpec`]. - pub fn to_chunk_ranges(&self) -> ChunkRanges { - // this is zero allocation for single ranges - // todo: optimize this in range collections - let mut ranges = ChunkRanges::empty(); - let mut current = ChunkNum(0); - let mut on = false; - for &width in self.0.iter() { - let next = current + width; - if on { - ranges |= ChunkRanges::from(current..next); - } - current = next; - on = !on; - } - if on { - ranges |= ChunkRanges::from(current..); - } - ranges - } -} - -impl fmt::Debug for RangeSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if f.alternate() { - f.debug_list() - .entries(self.to_chunk_ranges().iter()) - .finish() - } else if self.is_all() { - write!(f, "all") - } else if self.is_empty() { - write!(f, "empty") - } else { - f.debug_list().entries(self.0.iter()).finish() - } - } -} - -/// A chunk range specification for a sequence of blobs. -/// -/// To select chunks in a sequence of blobs this is encoded as a sequence of `(blob_offset, -/// range_spec)` tuples. Offsets are interpreted in an accumulating fashion. -/// -/// ## Example: -/// -/// Suppose two [`RangeSpec`]s `range_a` and `range_b`. -/// -/// - `[(0, range_a), (2, empty), (3, range_b), (1, empty)]` encodes: -/// - Select `range_a` for children in the range `[0, 2)` -/// - do no selection (empty) for children in the range `[2, 2+3) = [2, 5)` (3 children) -/// - Select `range_b` for children in the range `[5, 5+1) = [5, 6)` (1 children) -/// - do no selection (empty) for children in the open range `[6, inf)` -/// -/// Another way to understand this is that offsets represent the number of times the -/// previous range appears. -/// -/// Other examples: -/// -/// - Select `range_a` from all blobs after the 5th one in the sequence: `[(5, range_a)]`. -/// -/// - Select `range_a` from all blobs in the sequence: `[(0, range_a)]`. -/// -/// - Select `range_a` from blob 1234: `[(1234, range_a), (1, empty)]`. -/// -/// - Select nothing: `[]`. -/// -/// This is a smallvec so that we can avoid allocations in the common case of a single child -/// range. -#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Hash)] -#[repr(transparent)] -pub struct RangeSpecSeq(SmallVec<[(u64, RangeSpec); 2]>); - -impl RangeSpecSeq { - #[allow(dead_code)] - /// A [`RangeSpecSeq`] containing no chunks from any blobs in the sequence. - /// - /// [`RangeSpecSeq::iter`], will return an empty range forever. - pub const fn empty() -> Self { - Self(SmallVec::new_const()) - } - - /// If this range seq describes a range for a single item, returns the offset - /// and range spec for that item - pub fn as_single(&self) -> Option<(u64, &RangeSpec)> { - // we got two elements, - // the first element starts at offset 0, - // and the second element is empty - if self.0.len() != 2 { - return None; - } - let (fst_ofs, fst_val) = &self.0[0]; - let (snd_ofs, snd_val) = &self.0[1]; - if *snd_ofs == 1 && snd_val.is_empty() { - Some((*fst_ofs, fst_val)) - } else { - None - } - } - - /// A [`RangeSpecSeq`] containing all chunks from all blobs. - /// - /// [`RangeSpecSeq::iter`], will return a full range forever. - pub fn all() -> Self { - Self(smallvec![(0, RangeSpec::all())]) - } - - /// Convenience function to create a [`RangeSpecSeq`] from a finite sequence of range sets. - pub fn from_ranges(ranges: impl IntoIterator>) -> Self { - Self::new( - ranges - .into_iter() - .map(RangeSpec::new) - .chain(std::iter::once(RangeSpec::EMPTY)), - ) - } - - /// Convenience function to create a [`RangeSpecSeq`] from a sequence of range sets. - /// - /// Compared to [`RangeSpecSeq::from_ranges`], this will not add an empty range spec at the end, so the final - /// range spec will repeat forever. - pub fn from_ranges_infinite( - ranges: impl IntoIterator>, - ) -> Self { - Self::new(ranges.into_iter().map(RangeSpec::new)) - } - - /// Creates a new range spec sequence from a sequence of range specs. - /// - /// This will merge adjacent range specs with the same value and thus make - /// sure that the resulting sequence is as compact as possible. - pub fn new(children: impl IntoIterator) -> Self { - let mut count = 0; - let mut res = SmallVec::new(); - let before_all = RangeSpec::EMPTY; - for v in children.into_iter() { - let prev = res.last().map(|(_count, spec)| spec).unwrap_or(&before_all); - if &v == prev { - count += 1; - } else { - res.push((count, v.clone())); - count = 1; - } - } - Self(res) - } - - /// An infinite iterator of range specs for blobs in the sequence. - /// - /// Each item yielded by the iterator is the [`RangeSpec`] for a blob in the sequence. - /// Thus the first call to `.next()` returns the range spec for the first blob, the next - /// call returns the range spec of the second blob, etc. - pub fn iter(&self) -> RequestRangeSpecIter<'_> { - let before_first = self.0.first().map(|(c, _)| *c).unwrap_or_default(); - RequestRangeSpecIter { - current: &EMPTY_RANGE_SPEC, - count: before_first, - remaining: &self.0, - } - } - - /// An iterator over blobs in the sequence with a non-empty range spec. - /// - /// This iterator will only yield items for blobs which have at least one chunk - /// selected. - /// - /// This iterator is infinite if the [`RangeSpecSeq`] ends on a non-empty [`RangeSpec`], - /// that is all further blobs have selected chunks spans. - pub fn iter_non_empty(&self) -> NonEmptyRequestRangeSpecIter<'_> { - NonEmptyRequestRangeSpecIter::new(self.iter()) - } -} - -static EMPTY_RANGE_SPEC: RangeSpec = RangeSpec::EMPTY; - -/// An infinite iterator yielding [`RangeSpec`]s for each blob in a sequence. -/// -/// The first item yielded is the [`RangeSpec`] for the first blob in the sequence, the -/// next item is the [`RangeSpec`] for the next blob, etc. -#[derive(Debug)] -pub struct RequestRangeSpecIter<'a> { - /// current value - current: &'a RangeSpec, - /// number of times to emit current before grabbing next value - /// if remaining is empty, this is ignored and current is emitted forever - count: u64, - /// remaining ranges - remaining: &'a [(u64, RangeSpec)], -} - -impl<'a> RequestRangeSpecIter<'a> { - pub fn new(ranges: &'a [(u64, RangeSpec)]) -> Self { - let before_first = ranges.first().map(|(c, _)| *c).unwrap_or_default(); - RequestRangeSpecIter { - current: &EMPTY_RANGE_SPEC, - count: before_first, - remaining: ranges, - } - } - - /// True if we are at the end of the iterator. - /// - /// This does not mean that the iterator is terminated, it just means that - /// it will repeat the same value forever. - pub fn is_at_end(&self) -> bool { - self.count == 0 && self.remaining.is_empty() - } -} - -impl<'a> Iterator for RequestRangeSpecIter<'a> { - type Item = &'a RangeSpec; - - fn next(&mut self) -> Option { - Some(loop { - break if self.count > 0 { - // emit current value count times - self.count -= 1; - self.current - } else if let Some(((_, new), rest)) = self.remaining.split_first() { - // get next current value, new count, and set remaining - self.current = new; - self.count = rest.first().map(|(c, _)| *c).unwrap_or_default(); - self.remaining = rest; - continue; - } else { - // no more values, just repeat current forever - self.current - }; - }) - } -} - -/// An iterator over blobs in the sequence with a non-empty range specs. -/// -/// default is what to use if the children of this RequestRangeSpec are empty. -#[derive(Debug)] -pub struct NonEmptyRequestRangeSpecIter<'a> { - inner: RequestRangeSpecIter<'a>, - count: u64, -} - -impl<'a> NonEmptyRequestRangeSpecIter<'a> { - fn new(inner: RequestRangeSpecIter<'a>) -> Self { - Self { inner, count: 0 } - } - - pub(crate) fn offset(&self) -> u64 { - self.count - } -} - -impl<'a> Iterator for NonEmptyRequestRangeSpecIter<'a> { - type Item = (u64, &'a RangeSpec); - - fn next(&mut self) -> Option { - loop { - // unwrapping is safe because we know that the inner iterator will never terminate - let curr = self.inner.next().unwrap(); - let count = self.count; - // increase count in any case until we are at the end of possible u64 values - // we are unlikely to ever reach this limit. - self.count = self.count.checked_add(1)?; - // yield only if the current value is non-empty - if !curr.is_empty() { - break Some((count, curr)); - } else if self.inner.is_at_end() { - // terminate instead of looping until we run out of u64 values - break None; - } - } - } -} - -#[cfg(test)] -mod tests { - use std::ops::Range; - - use iroh_test::{assert_eq_hex, hexdump::parse_hexdump}; - use proptest::prelude::*; - - use super::*; - - fn ranges(value_range: Range) -> impl Strategy { - prop::collection::vec((value_range.clone(), value_range), 0..16).prop_map(|v| { - let mut res = ChunkRanges::empty(); - for (a, b) in v { - let start = a.min(b); - let end = a.max(b); - res |= ChunkRanges::from(ChunkNum(start)..ChunkNum(end)); - } - res - }) - } - - fn range_spec_seq_roundtrip_impl(ranges: &[ChunkRanges]) -> Vec { - let spec = RangeSpecSeq::from_ranges(ranges.iter().cloned()); - spec.iter() - .map(|x| x.to_chunk_ranges()) - .take(ranges.len()) - .collect::>() - } - - fn range_spec_seq_bytes_roundtrip_impl(ranges: &[ChunkRanges]) -> Vec { - let spec = RangeSpecSeq::from_ranges(ranges.iter().cloned()); - let bytes = postcard::to_allocvec(&spec).unwrap(); - let spec2: RangeSpecSeq = postcard::from_bytes(&bytes).unwrap(); - spec2 - .iter() - .map(|x| x.to_chunk_ranges()) - .take(ranges.len()) - .collect::>() - } - - fn mk_case(case: Vec>) -> Vec { - case.iter() - .map(|x| ChunkRanges::from(ChunkNum(x.start)..ChunkNum(x.end))) - .collect::>() - } - - #[test] - fn range_spec_wire_format() { - // a list of commented hex dumps and the corresponding range spec - let cases = [ - (RangeSpec::EMPTY, "00"), - ( - RangeSpec::all(), - r" - 01 # length prefix - 1 element - 00 # span width - 0. everything stating from 0 is included - ", - ), - ( - RangeSpec::new(ChunkRanges::from(ChunkNum(64)..)), - r" - 01 # length prefix - 1 element - 40 # span width - 64. everything starting from 64 is included - ", - ), - ( - RangeSpec::new(ChunkRanges::from(ChunkNum(10000)..)), - r" - 01 # length prefix - 1 element - 904E # span width - 10000, 904E in postcard varint encoding. everything starting from 10000 is included - ", - ), - ( - RangeSpec::new(ChunkRanges::from(..ChunkNum(64))), - r" - 02 # length prefix - 2 elements - 00 # span width - 0. everything stating from 0 is included - 40 # span width - 64. everything starting from 64 is excluded - ", - ), - ( - RangeSpec::new( - &ChunkRanges::from(ChunkNum(1)..ChunkNum(3)) - | &ChunkRanges::from(ChunkNum(9)..ChunkNum(13)), - ), - r" - 04 # length prefix - 4 elements - 01 # span width - 1 - 02 # span width - 2 (3 - 1) - 06 # span width - 6 (9 - 3) - 04 # span width - 4 (13 - 9) - ", - ), - ]; - for (case, expected_hex) in cases { - let expected = parse_hexdump(expected_hex).unwrap(); - assert_eq_hex!(expected, postcard::to_stdvec(&case).unwrap()); - } - } - - #[test] - fn range_spec_seq_wire_format() { - let cases = [ - (RangeSpecSeq::empty(), "00"), - ( - RangeSpecSeq::all(), - r" - 01 # 1 tuple in total - # first tuple - 00 # span 0 until start - 0100 # 1 element, RangeSpec::all() - ", - ), - ( - RangeSpecSeq::from_ranges([ - ChunkRanges::from(ChunkNum(1)..ChunkNum(3)), - ChunkRanges::from(ChunkNum(7)..ChunkNum(13)), - ]), - r" - 03 # 3 tuples in total - # first tuple - 00 # span 0 until start - 020102 # range 1..3 - # second tuple - 01 # span 1 until next - 020706 # range 7..13 - # third tuple - 01 # span 1 until next - 00 # empty range forever from now - ", - ), - ( - RangeSpecSeq::from_ranges_infinite([ - ChunkRanges::empty(), - ChunkRanges::empty(), - ChunkRanges::empty(), - ChunkRanges::from(ChunkNum(7)..), - ChunkRanges::all(), - ]), - r" - 02 # 2 tuples in total - # first tuple - 03 # span 3 until start (first 3 elements are empty) - 01 07 # range 7.. - # second tuple - 01 # span 1 until next (1 element is 7..) - 01 00 # ChunkRanges::all() forever from now - ", - ), - ]; - for (case, expected_hex) in cases { - let expected = parse_hexdump(expected_hex).unwrap(); - assert_eq_hex!(expected, postcard::to_stdvec(&case).unwrap()); - } - } - - /// Test that the roundtrip from [`Vec`] via [`RangeSpec`] to [`RangeSpecSeq`] and back works. - #[test] - fn range_spec_seq_roundtrip_cases() { - for case in [ - vec![0..1, 0..0], - vec![1..2, 1..2, 1..2], - vec![1..2, 1..2, 2..3, 2..3], - ] { - let case = mk_case(case); - let expected = case.clone(); - let actual = range_spec_seq_roundtrip_impl(&case); - assert_eq!(expected, actual); - } - } - - /// Test that the creation of a [`RangeSpecSeq`] from a sequence of [`ChunkRanges`]s canonicalizes the result. - #[test] - fn range_spec_seq_canonical() { - for (case, expected_count) in [ - (vec![0..1, 0..0], 2), - (vec![1..2, 1..2, 1..2], 2), - (vec![1..2, 1..2, 2..3, 2..3], 3), - ] { - let case = mk_case(case); - let spec = RangeSpecSeq::from_ranges(case); - assert_eq!(spec.0.len(), expected_count); - } - } - - proptest! { - #[test] - fn range_spec_roundtrip(ranges in ranges(0..1000)) { - let spec = RangeSpec::new(&ranges); - let ranges2 = spec.to_chunk_ranges(); - prop_assert_eq!(ranges, ranges2); - } - - #[test] - fn range_spec_seq_roundtrip(ranges in proptest::collection::vec(ranges(0..100), 0..10)) { - let expected = ranges.clone(); - let actual = range_spec_seq_roundtrip_impl(&ranges); - prop_assert_eq!(expected, actual); - } - - #[test] - fn range_spec_seq_bytes_roundtrip(ranges in proptest::collection::vec(ranges(0..100), 0..10)) { - let expected = ranges.clone(); - let actual = range_spec_seq_bytes_roundtrip_impl(&ranges); - prop_assert_eq!(expected, actual); - } - } -} diff --git a/iroh-blobs/src/provider.rs b/iroh-blobs/src/provider.rs deleted file mode 100644 index 1ffc8bff499..00000000000 --- a/iroh-blobs/src/provider.rs +++ /dev/null @@ -1,662 +0,0 @@ -//! The server side API -use std::{fmt::Debug, sync::Arc, time::Duration}; - -use anyhow::{Context, Result}; -use bao_tree::io::{ - fsm::{encode_ranges_validated, Outboard}, - EncodeError, -}; -use futures_lite::future::Boxed as BoxFuture; -use iroh_base::rpc::RpcError; -use iroh_io::{ - stats::{SliceReaderStats, StreamWriterStats, TrackingSliceReader, TrackingStreamWriter}, - AsyncSliceReader, AsyncStreamWriter, TokioStreamWriter, -}; -use iroh_net::endpoint::{self, RecvStream, SendStream}; -use serde::{Deserialize, Serialize}; -use tracing::{debug, debug_span, info, trace, warn}; -use tracing_futures::Instrument; - -use crate::{ - hashseq::parse_hash_seq, - protocol::{GetRequest, RangeSpec, Request}, - store::*, - util::{local_pool::LocalPoolHandle, Tag}, - BlobFormat, Hash, -}; - -/// Events emitted by the provider informing about the current status. -#[derive(Debug, Clone)] -pub enum Event { - /// A new collection or tagged blob has been added - TaggedBlobAdded { - /// The hash of the added data - hash: Hash, - /// The format of the added data - format: BlobFormat, - /// The tag of the added data - tag: Tag, - }, - /// A new client connected to the node. - ClientConnected { - /// An unique connection id. - connection_id: u64, - }, - /// A request was received from a client. - GetRequestReceived { - /// An unique connection id. - connection_id: u64, - /// An identifier uniquely identifying this transfer request. - request_id: u64, - /// The hash for which the client wants to receive data. - hash: Hash, - }, - /// A sequence of hashes has been found and is being transferred. - TransferHashSeqStarted { - /// An unique connection id. - connection_id: u64, - /// An identifier uniquely identifying this transfer request. - request_id: u64, - /// The number of blobs in the sequence. - num_blobs: u64, - }, - /// A chunk of a blob was transferred. - /// - /// These events will be sent with try_send, so you can not assume that you - /// will receive all of them. - TransferProgress { - /// An unique connection id. - connection_id: u64, - /// An identifier uniquely identifying this transfer request. - request_id: u64, - /// The hash for which we are transferring data. - hash: Hash, - /// Offset up to which we have transferred data. - end_offset: u64, - }, - /// A blob in a sequence was transferred. - TransferBlobCompleted { - /// An unique connection id. - connection_id: u64, - /// An identifier uniquely identifying this transfer request. - request_id: u64, - /// The hash of the blob - hash: Hash, - /// The index of the blob in the sequence. - index: u64, - /// The size of the blob transferred. - size: u64, - }, - /// A request was completed and the data was sent to the client. - TransferCompleted { - /// An unique connection id. - connection_id: u64, - /// An identifier uniquely identifying this transfer request. - request_id: u64, - /// statistics about the transfer - stats: Box, - }, - /// A request was aborted because the client disconnected. - TransferAborted { - /// The quic connection id. - connection_id: u64, - /// An identifier uniquely identifying this request. - request_id: u64, - /// statistics about the transfer. This is None if the transfer - /// was aborted before any data was sent. - stats: Option>, - }, -} - -/// The stats for a transfer of a collection or blob. -#[derive(Debug, Clone, Copy, Default)] -pub struct TransferStats { - /// Stats for sending to the client. - pub send: StreamWriterStats, - /// Stats for reading from disk. - pub read: SliceReaderStats, - /// The total duration of the transfer. - pub duration: Duration, -} - -/// Progress updates for the add operation. -#[derive(Debug, Serialize, Deserialize)] -pub enum AddProgress { - /// An item was found with name `name`, from now on referred to via `id` - Found { - /// A new unique id for this entry. - id: u64, - /// The name of the entry. - name: String, - /// The size of the entry in bytes. - size: u64, - }, - /// We got progress ingesting item `id`. - Progress { - /// The unique id of the entry. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done with `id`, and the hash is `hash`. - Done { - /// The unique id of the entry. - id: u64, - /// The hash of the entry. - hash: Hash, - }, - /// We are done with the whole operation. - AllDone { - /// The hash of the created data. - hash: Hash, - /// The format of the added data. - format: BlobFormat, - /// The tag of the added data. - tag: Tag, - }, - /// We got an error and need to abort. - /// - /// This will be the last message in the stream. - Abort(RpcError), -} - -/// Progress updates for the batch add operation. -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchAddPathProgress { - /// An item was found with the given size - Found { - /// The size of the entry in bytes. - size: u64, - }, - /// We got progress ingesting the item. - Progress { - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done, and the hash is `hash`. - Done { - /// The hash of the entry. - hash: Hash, - }, - /// We got an error and need to abort. - /// - /// This will be the last message in the stream. - Abort(RpcError), -} - -/// Read the request from the getter. -/// -/// Will fail if there is an error while reading, if the reader -/// contains more data than the Request, or if no valid request is sent. -/// -/// When successful, the buffer is empty after this function call. -pub async fn read_request(mut reader: RecvStream) -> Result { - let payload = reader - .read_to_end(crate::protocol::MAX_MESSAGE_SIZE) - .await?; - let request: Request = postcard::from_bytes(&payload)?; - Ok(request) -} - -/// Transfers a blob or hash sequence to the client. -/// -/// The difference to [`handle_get`] is that we already have a reader for the -/// root blob and outboard. -/// -/// First, it transfers the root blob. Then, if needed, it sequentially -/// transfers each individual blob data. -/// -/// The transfer fail if there is an error writing to the writer or reading from -/// the database. -/// -/// If a blob from the hash sequence cannot be found in the database, the -/// transfer will return with [`SentStatus::NotFound`]. If the transfer completes -/// successfully, it will return with [`SentStatus::Sent`]. -pub(crate) async fn transfer_hash_seq( - request: GetRequest, - // Store from which to fetch blobs. - db: &D, - // Response writer, containing the quinn stream. - writer: &mut ResponseWriter, - // the collection to transfer - mut outboard: impl Outboard, - mut data: impl AsyncSliceReader, - stats: &mut TransferStats, -) -> Result { - let hash = request.hash; - let events = writer.events.clone(); - let request_id = writer.request_id(); - let connection_id = writer.connection_id(); - - // if the request is just for the root, we don't need to deserialize the collection - let just_root = matches!(request.ranges.as_single(), Some((0, _))); - let mut c = if !just_root { - // parse the hash seq - let (stream, num_blobs) = parse_hash_seq(&mut data).await?; - writer - .events - .send(|| Event::TransferHashSeqStarted { - connection_id: writer.connection_id(), - request_id: writer.request_id(), - num_blobs, - }) - .await; - Some(stream) - } else { - None - }; - - let mk_progress = |end_offset| Event::TransferProgress { - connection_id, - request_id, - hash, - end_offset, - }; - - let mut prev = 0; - for (offset, ranges) in request.ranges.iter_non_empty() { - // create a tracking writer so we can get some stats for writing - let mut tw = writer.tracking_writer(); - if offset == 0 { - debug!("writing ranges '{:?}' of sequence {}", ranges, hash); - // wrap the data reader in a tracking reader so we can get some stats for reading - let mut tracking_reader = TrackingSliceReader::new(&mut data); - let mut sending_reader = - SendingSliceReader::new(&mut tracking_reader, &events, mk_progress); - // send the root - tw.write(outboard.tree().size().to_le_bytes().as_slice()) - .await?; - encode_ranges_validated( - &mut sending_reader, - &mut outboard, - &ranges.to_chunk_ranges(), - &mut tw, - ) - .await?; - stats.read += tracking_reader.stats(); - stats.send += tw.stats(); - debug!( - "finished writing ranges '{:?}' of collection {}", - ranges, hash - ); - } else { - let c = c.as_mut().context("collection parser not available")?; - debug!("wrtiting ranges '{:?}' of child {}", ranges, offset); - // skip to the next blob if there is a gap - if prev < offset - 1 { - c.skip(offset - prev - 1).await?; - } - if let Some(hash) = c.next().await? { - tokio::task::yield_now().await; - let (status, size, blob_read_stats) = - send_blob(db, hash, ranges, &mut tw, events.clone(), mk_progress).await?; - stats.send += tw.stats(); - stats.read += blob_read_stats; - if SentStatus::NotFound == status { - writer.inner.finish()?; - return Ok(status); - } - - writer - .events - .send(|| Event::TransferBlobCompleted { - connection_id: writer.connection_id(), - request_id: writer.request_id(), - hash, - index: offset - 1, - size, - }) - .await; - } else { - // nothing more we can send - break; - } - prev = offset; - } - } - - debug!("done writing"); - Ok(SentStatus::Sent) -} - -struct SendingSliceReader<'a, R, F> { - inner: R, - sender: &'a EventSender, - make_event: F, -} - -impl<'a, R: AsyncSliceReader, F: Fn(u64) -> Event> SendingSliceReader<'a, R, F> { - fn new(inner: R, sender: &'a EventSender, make_event: F) -> Self { - Self { - inner, - sender, - make_event, - } - } -} - -impl<'a, R: AsyncSliceReader, F: Fn(u64) -> Event> AsyncSliceReader - for SendingSliceReader<'a, R, F> -{ - async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result { - let res = self.inner.read_at(offset, len).await; - if let Ok(res) = res.as_ref() { - let end_offset = offset + res.len() as u64; - self.sender.try_send(|| (self.make_event)(end_offset)); - } - res - } - - async fn size(&mut self) -> std::io::Result { - self.inner.size().await - } -} - -/// Trait for sending blob events. -pub trait CustomEventSender: std::fmt::Debug + Sync + Send + 'static { - /// Send an event and wait for it to be sent. - fn send(&self, event: Event) -> BoxFuture<()>; - - /// Try to send an event. - fn try_send(&self, event: Event); -} - -/// A sender for events related to blob transfers. -/// -/// The sender is disabled by default. -#[derive(Debug, Clone, Default)] -pub struct EventSender { - inner: Option>, -} - -impl From for EventSender { - fn from(inner: T) -> Self { - Self { - inner: Some(Arc::new(inner)), - } - } -} - -impl EventSender { - /// Create a new event sender. - pub fn new(inner: Option>) -> Self { - Self { inner } - } - - /// Send an event. - /// - /// If the inner sender is not set, the function to produce the event will - /// not be called. So any cost associated with gathering information for the - /// event will not be incurred. - pub async fn send(&self, event: impl FnOnce() -> Event) { - if let Some(inner) = &self.inner { - let event = event(); - inner.as_ref().send(event).await; - } - } - - /// Try to send an event. - /// - /// This will just drop the event if it can not be sent immediately. So it - /// is only appropriate for events that are not critical, such as - /// self-contained progress updates. - pub fn try_send(&self, event: impl FnOnce() -> Event) { - if let Some(inner) = &self.inner { - let event = event(); - inner.as_ref().try_send(event); - } - } -} - -/// Handle a single connection. -pub async fn handle_connection( - connection: endpoint::Connection, - db: D, - events: EventSender, - rt: LocalPoolHandle, -) { - let remote_addr = connection.remote_address(); - let connection_id = connection.stable_id() as u64; - let span = debug_span!("connection", connection_id, %remote_addr); - async move { - while let Ok((writer, reader)) = connection.accept_bi().await { - // The stream ID index is used to identify this request. Requests only arrive in - // bi-directional RecvStreams initiated by the client, so this uniquely identifies them. - let request_id = reader.id().index(); - let span = debug_span!("stream", stream_id = %request_id); - let writer = ResponseWriter { - connection_id, - events: events.clone(), - inner: writer, - }; - events - .send(|| Event::ClientConnected { connection_id }) - .await; - let db = db.clone(); - rt.spawn_detached(|| { - async move { - if let Err(err) = handle_stream(db, reader, writer).await { - warn!("error: {err:#?}",); - } - } - .instrument(span) - }); - } - } - .instrument(span) - .await -} - -async fn handle_stream(db: D, reader: RecvStream, writer: ResponseWriter) -> Result<()> { - // 1. Decode the request. - debug!("reading request"); - let request = match read_request(reader).await { - Ok(r) => r, - Err(e) => { - writer.notify_transfer_aborted(None).await; - return Err(e); - } - }; - - match request { - Request::Get(request) => handle_get(db, request, writer).await, - } -} - -/// Handle a single get request. -/// -/// Requires the request, a database, and a writer. -pub async fn handle_get( - db: D, - request: GetRequest, - mut writer: ResponseWriter, -) -> Result<()> { - let hash = request.hash; - debug!(%hash, "received request"); - writer - .events - .send(|| Event::GetRequestReceived { - hash, - connection_id: writer.connection_id(), - request_id: writer.request_id(), - }) - .await; - - // 4. Attempt to find hash - match db.get(&hash).await? { - // Collection or blob request - Some(entry) => { - let mut stats = Box::::default(); - let t0 = std::time::Instant::now(); - // 5. Transfer data! - let res = transfer_hash_seq( - request, - &db, - &mut writer, - entry.outboard().await?, - entry.data_reader().await?, - &mut stats, - ) - .await; - stats.duration = t0.elapsed(); - match res { - Ok(SentStatus::Sent) => { - writer.notify_transfer_completed(&hash, stats).await; - } - Ok(SentStatus::NotFound) => { - writer.notify_transfer_aborted(Some(stats)).await; - } - Err(e) => { - writer.notify_transfer_aborted(Some(stats)).await; - return Err(e); - } - } - - debug!("finished response"); - } - None => { - debug!("not found {}", hash); - writer.notify_transfer_aborted(None).await; - writer.inner.finish()?; - } - }; - - Ok(()) -} - -/// A helper struct that combines a quinn::SendStream with auxiliary information -#[derive(Debug)] -pub struct ResponseWriter { - inner: SendStream, - events: EventSender, - connection_id: u64, -} - -impl ResponseWriter { - fn tracking_writer(&mut self) -> TrackingStreamWriter> { - TrackingStreamWriter::new(TokioStreamWriter(&mut self.inner)) - } - - fn connection_id(&self) -> u64 { - self.connection_id - } - - fn request_id(&self) -> u64 { - self.inner.id().index() - } - - fn print_stats(stats: &TransferStats) { - let send = stats.send.total(); - let read = stats.read.total(); - let total_sent_bytes = send.size; - let send_duration = send.stats.duration; - let read_duration = read.stats.duration; - let total_duration = stats.duration; - let other_duration = total_duration - .saturating_sub(send_duration) - .saturating_sub(read_duration); - let avg_send_size = total_sent_bytes.checked_div(send.stats.count).unwrap_or(0); - info!( - "sent {} bytes in {}s", - total_sent_bytes, - total_duration.as_secs_f64() - ); - debug!( - "{}s sending, {}s reading, {}s other", - send_duration.as_secs_f64(), - read_duration.as_secs_f64(), - other_duration.as_secs_f64() - ); - trace!( - "send_count: {} avg_send_size {}", - send.stats.count, - avg_send_size, - ) - } - - async fn notify_transfer_completed(&self, hash: &Hash, stats: Box) { - info!("transfer completed for {}", hash); - Self::print_stats(&stats); - self.events - .send(move || Event::TransferCompleted { - connection_id: self.connection_id(), - request_id: self.request_id(), - stats, - }) - .await; - } - - async fn notify_transfer_aborted(&self, stats: Option>) { - if let Some(stats) = &stats { - Self::print_stats(stats); - }; - self.events - .send(move || Event::TransferAborted { - connection_id: self.connection_id(), - request_id: self.request_id(), - stats, - }) - .await; - } -} - -/// Status of a send operation -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SentStatus { - /// The requested data was sent - Sent, - /// The requested data was not found - NotFound, -} - -/// Send a blob to the client. -pub async fn send_blob( - db: &D, - hash: Hash, - ranges: &RangeSpec, - mut writer: W, - events: EventSender, - mk_progress: impl Fn(u64) -> Event, -) -> Result<(SentStatus, u64, SliceReaderStats)> { - match db.get(&hash).await? { - Some(entry) => { - let outboard = entry.outboard().await?; - let size = outboard.tree().size(); - let mut file_reader = TrackingSliceReader::new(entry.data_reader().await?); - let mut sending_reader = - SendingSliceReader::new(&mut file_reader, &events, mk_progress); - writer.write(size.to_le_bytes().as_slice()).await?; - encode_ranges_validated( - &mut sending_reader, - outboard, - &ranges.to_chunk_ranges(), - writer, - ) - .await - .map_err(|e| encode_error_to_anyhow(e, &hash))?; - - Ok((SentStatus::Sent, size, file_reader.stats())) - } - _ => { - debug!("blob not found {}", hash.to_hex()); - Ok((SentStatus::NotFound, 0, SliceReaderStats::default())) - } - } -} - -fn encode_error_to_anyhow(err: EncodeError, hash: &Hash) -> anyhow::Error { - match err { - EncodeError::LeafHashMismatch(x) => anyhow::Error::from(EncodeError::LeafHashMismatch(x)) - .context(format!("hash {} offset {}", hash.to_hex(), x.to_bytes())), - EncodeError::ParentHashMismatch(n) => { - let r = n.chunk_range(); - anyhow::Error::from(EncodeError::ParentHashMismatch(n)).context(format!( - "hash {} range {}..{}", - hash.to_hex(), - r.start.to_bytes(), - r.end.to_bytes() - )) - } - e => anyhow::Error::from(e).context(format!("hash {}", hash.to_hex())), - } -} diff --git a/iroh-blobs/src/store.rs b/iroh-blobs/src/store.rs deleted file mode 100644 index 3030d55b3f5..00000000000 --- a/iroh-blobs/src/store.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! Implementations of blob stores -use crate::{BlobFormat, Hash, HashAndFormat}; - -#[cfg(feature = "fs-store")] -mod bao_file; -pub mod mem; -mod mutable_mem_storage; -pub mod readonly_mem; - -#[cfg(feature = "fs-store")] -pub mod fs; - -mod traits; -use tracing::warn; -pub use traits::*; - -/// Create a 16 byte unique ID. -fn new_uuid() -> [u8; 16] { - use rand::Rng; - rand::thread_rng().gen::<[u8; 16]>() -} - -/// Create temp file name based on a 16 byte UUID. -fn temp_name() -> String { - format!("{}.temp", hex::encode(new_uuid())) -} - -#[derive(Debug, Default, Clone)] -struct TempCounters { - /// number of raw temp tags for a hash - raw: u64, - /// number of hash seq temp tags for a hash - hash_seq: u64, -} - -impl TempCounters { - fn counter(&mut self, format: BlobFormat) -> &mut u64 { - match format { - BlobFormat::Raw => &mut self.raw, - BlobFormat::HashSeq => &mut self.hash_seq, - } - } - - fn inc(&mut self, format: BlobFormat) { - let counter = self.counter(format); - *counter = counter.checked_add(1).unwrap(); - } - - fn dec(&mut self, format: BlobFormat) { - let counter = self.counter(format); - *counter = counter.saturating_sub(1); - } - - fn is_empty(&self) -> bool { - self.raw == 0 && self.hash_seq == 0 - } -} - -#[derive(Debug, Clone, Default)] -struct TempCounterMap(std::collections::BTreeMap); - -impl TempCounterMap { - fn inc(&mut self, value: &HashAndFormat) { - let HashAndFormat { hash, format } = value; - self.0.entry(*hash).or_default().inc(*format) - } - - fn dec(&mut self, value: &HashAndFormat) { - let HashAndFormat { hash, format } = value; - let Some(counters) = self.0.get_mut(hash) else { - warn!("Decrementing non-existent temp tag"); - return; - }; - counters.dec(*format); - if counters.is_empty() { - self.0.remove(hash); - } - } - - fn contains(&self, hash: &Hash) -> bool { - self.0.contains_key(hash) - } - - fn keys(&self) -> impl Iterator { - let mut res = Vec::new(); - for (k, v) in self.0.iter() { - if v.raw > 0 { - res.push(HashAndFormat::raw(*k)); - } - if v.hash_seq > 0 { - res.push(HashAndFormat::hash_seq(*k)); - } - } - res.into_iter() - } -} diff --git a/iroh-blobs/src/store/bao_file.rs b/iroh-blobs/src/store/bao_file.rs deleted file mode 100644 index 369a3574d74..00000000000 --- a/iroh-blobs/src/store/bao_file.rs +++ /dev/null @@ -1,1043 +0,0 @@ -//! An implementation of a bao file, meaning some data blob with associated -//! outboard. -//! -//! Compared to just a pair of (data, outboard), this implementation also works -//! when both the data and the outboard is incomplete, and not even the size -//! is fully known. -//! -//! There is a full in memory implementation, and an implementation that uses -//! the file system for the data, outboard, and sizes file. There is also a -//! combined implementation that starts in memory and switches to file when -//! the memory limit is reached. -use std::{ - fs::{File, OpenOptions}, - io, - ops::{Deref, DerefMut}, - path::{Path, PathBuf}, - sync::{Arc, RwLock, Weak}, -}; - -use bao_tree::{ - io::{ - fsm::BaoContentItem, - outboard::PreOrderOutboard, - sync::{ReadAt, WriteAt}, - }, - BaoTree, -}; -use bytes::{Bytes, BytesMut}; -use derive_more::Debug; -use iroh_base::hash::Hash; -use iroh_io::AsyncSliceReader; - -use super::mutable_mem_storage::{MutableMemStorage, SizeInfo}; -use crate::{ - store::BaoBatchWriter, - util::{get_limited_slice, MemOrFile, SparseMemFile}, - IROH_BLOCK_SIZE, -}; - -/// Data files are stored in 3 files. The data file, the outboard file, -/// and a sizes file. The sizes file contains the size that the remote side told us -/// when writing each data block. -/// -/// For complete data files, the sizes file is not needed, since you can just -/// use the size of the data file. -/// -/// For files below the chunk size, the outboard file is not needed, since -/// there is only one leaf, and the outboard file is empty. -struct DataPaths { - /// The data file. Size is determined by the chunk with the highest offset - /// that has been written. - /// - /// Gaps will be filled with zeros. - data: PathBuf, - /// The outboard file. This is *without* the size header, since that is not - /// known for partial files. - /// - /// The size of the outboard file is therefore a multiple of a hash pair - /// (64 bytes). - /// - /// The naming convention is to use obao for pre order traversal and oboa - /// for post order traversal. The log2 of the chunk group size is appended, - /// so for the default chunk group size in iroh of 4, the file extension - /// is .obao4. - outboard: PathBuf, - /// The sizes file. This is a file with 8 byte sizes for each chunk group. - /// The naming convention is to prepend the log2 of the chunk group size, - /// so for the default chunk group size in iroh of 4, the file extension - /// is .sizes4. - /// - /// The traversal order is not relevant for the sizes file, since it is - /// about the data chunks, not the hash pairs. - sizes: PathBuf, -} - -/// Storage for complete blobs. There is no longer any uncertainty about the -/// size, so we don't need a sizes file. -/// -/// Writing is not possible but also not needed, since the file is complete. -/// This covers all combinations of data and outboard being in memory or on -/// disk. -/// -/// For the memory variant, it does reading in a zero copy way, since storage -/// is already a `Bytes`. -#[derive(Default, derive_more::Debug)] -pub struct CompleteStorage { - /// data part, which can be in memory or on disk. - #[debug("{:?}", data.as_ref().map_mem(|x| x.len()))] - pub data: MemOrFile, - /// outboard part, which can be in memory or on disk. - #[debug("{:?}", outboard.as_ref().map_mem(|x| x.len()))] - pub outboard: MemOrFile, -} - -impl CompleteStorage { - /// Read from the data file at the given offset, until end of file or max bytes. - pub fn read_data_at(&self, offset: u64, len: usize) -> Bytes { - match &self.data { - MemOrFile::Mem(mem) => get_limited_slice(mem, offset, len), - MemOrFile::File((file, _size)) => read_to_end(file, offset, len).unwrap(), - } - } - - /// Read from the outboard file at the given offset, until end of file or max bytes. - pub fn read_outboard_at(&self, offset: u64, len: usize) -> Bytes { - match &self.outboard { - MemOrFile::Mem(mem) => get_limited_slice(mem, offset, len), - MemOrFile::File((file, _size)) => read_to_end(file, offset, len).unwrap(), - } - } - - /// The size of the data file. - pub fn data_size(&self) -> u64 { - match &self.data { - MemOrFile::Mem(mem) => mem.len() as u64, - MemOrFile::File((_file, size)) => *size, - } - } - - /// The size of the outboard file. - pub fn outboard_size(&self) -> u64 { - match &self.outboard { - MemOrFile::Mem(mem) => mem.len() as u64, - MemOrFile::File((_file, size)) => *size, - } - } -} - -/// Create a file for reading and writing, but *without* truncating the existing -/// file. -fn create_read_write(path: impl AsRef) -> io::Result { - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(path) -} - -/// Read from the given file at the given offset, until end of file or max bytes. -fn read_to_end(file: impl ReadAt, offset: u64, max: usize) -> io::Result { - let mut res = BytesMut::new(); - let mut buf = [0u8; 4096]; - let mut remaining = max; - let mut offset = offset; - while remaining > 0 { - let end = buf.len().min(remaining); - let read = file.read_at(offset, &mut buf[..end])?; - if read == 0 { - // eof - break; - } - res.extend_from_slice(&buf[..read]); - offset += read as u64; - remaining -= read; - } - Ok(res.freeze()) -} - -fn max_offset(batch: &[BaoContentItem]) -> u64 { - batch - .iter() - .filter_map(|item| match item { - BaoContentItem::Leaf(leaf) => { - let len = leaf.data.len().try_into().unwrap(); - let end = leaf - .offset - .checked_add(len) - .expect("u64 overflow for leaf end"); - Some(end) - } - _ => None, - }) - .max() - .unwrap_or(0) -} - -/// A file storage for an incomplete bao file. -#[derive(Debug)] -pub struct FileStorage { - data: std::fs::File, - outboard: std::fs::File, - sizes: std::fs::File, -} - -impl FileStorage { - /// Split into data, outboard and sizes files. - pub fn into_parts(self) -> (File, File, File) { - (self.data, self.outboard, self.sizes) - } - - fn current_size(&self) -> io::Result { - let len = self.sizes.metadata()?.len(); - if len < 8 { - Ok(0) - } else { - // todo: use the last full u64 in case the sizes file is not a multiple of 8 - // bytes. Not sure how that would happen, but we should handle it. - let mut buf = [0u8; 8]; - self.sizes.read_exact_at(len - 8, &mut buf)?; - Ok(u64::from_le_bytes(buf)) - } - } - - fn write_batch(&mut self, size: u64, batch: &[BaoContentItem]) -> io::Result<()> { - let tree = BaoTree::new(size, IROH_BLOCK_SIZE); - for item in batch { - match item { - BaoContentItem::Parent(parent) => { - if let Some(offset) = tree.pre_order_offset(parent.node) { - let o0 = offset * 64; - self.outboard - .write_all_at(o0, parent.pair.0.as_bytes().as_slice())?; - self.outboard - .write_all_at(o0 + 32, parent.pair.1.as_bytes().as_slice())?; - } - } - BaoContentItem::Leaf(leaf) => { - let o0 = leaf.offset; - // divide by chunk size, multiply by 8 - let index = (leaf.offset >> (tree.block_size().chunk_log() + 10)) << 3; - tracing::trace!( - "write_batch f={:?} o={} l={}", - self.data, - o0, - leaf.data.len() - ); - self.data.write_all_at(o0, leaf.data.as_ref())?; - let size = tree.size(); - self.sizes.write_all_at(index, &size.to_le_bytes())?; - } - } - } - Ok(()) - } - - fn read_data_at(&self, offset: u64, len: usize) -> io::Result { - read_to_end(&self.data, offset, len) - } - - fn read_outboard_at(&self, offset: u64, len: usize) -> io::Result { - read_to_end(&self.outboard, offset, len) - } -} - -/// The storage for a bao file. This can be either in memory or on disk. -#[derive(Debug)] -pub(crate) enum BaoFileStorage { - /// The entry is incomplete and in memory. - /// - /// Since it is incomplete, it must be writeable. - /// - /// This is used mostly for tiny entries, <= 16 KiB. But in principle it - /// can be used for larger sizes. - /// - /// Incomplete mem entries are *not* persisted at all. So if the store - /// crashes they will be gone. - IncompleteMem(MutableMemStorage), - /// The entry is incomplete and on disk. - IncompleteFile(FileStorage), - /// The entry is complete. Outboard and data can come from different sources - /// (memory or file). - /// - /// Writing to this is a no-op, since it is already complete. - Complete(CompleteStorage), -} - -impl Default for BaoFileStorage { - fn default() -> Self { - BaoFileStorage::Complete(Default::default()) - } -} - -impl BaoFileStorage { - /// Take the storage out, leaving an empty storage in its place. - /// - /// Be careful to put something back in its place, or you will lose data. - #[cfg(feature = "fs-store")] - pub fn take(&mut self) -> Self { - std::mem::take(self) - } - - /// Create a new mutable mem storage. - pub fn incomplete_mem() -> Self { - Self::IncompleteMem(Default::default()) - } - - /// Call sync_all on all the files. - fn sync_all(&self) -> io::Result<()> { - match self { - Self::Complete(_) => Ok(()), - Self::IncompleteMem(_) => Ok(()), - Self::IncompleteFile(file) => { - file.data.sync_all()?; - file.outboard.sync_all()?; - file.sizes.sync_all()?; - Ok(()) - } - } - } - - /// True if the storage is in memory. - pub fn is_mem(&self) -> bool { - match self { - Self::IncompleteMem(_) => true, - Self::IncompleteFile(_) => false, - Self::Complete(c) => c.data.is_mem() && c.outboard.is_mem(), - } - } -} - -/// A weak reference to a bao file handle. -#[derive(Debug, Clone)] -pub struct BaoFileHandleWeak(Weak); - -impl BaoFileHandleWeak { - /// Upgrade to a strong reference if possible. - pub fn upgrade(&self) -> Option { - self.0.upgrade().map(BaoFileHandle) - } - - /// True if the handle is still live (has strong references) - pub fn is_live(&self) -> bool { - self.0.strong_count() > 0 - } -} - -/// The inner part of a bao file handle. -#[derive(Debug)] -pub struct BaoFileHandleInner { - pub(crate) storage: RwLock, - config: Arc, - hash: Hash, -} - -/// A cheaply cloneable handle to a bao file, including the hash and the configuration. -#[derive(Debug, Clone, derive_more::Deref)] -pub struct BaoFileHandle(Arc); - -pub(crate) type CreateCb = Arc io::Result<()> + Send + Sync>; - -/// Configuration for the deferred batch writer. It will start writing to memory, -/// and then switch to a file when the memory limit is reached. -#[derive(derive_more::Debug, Clone)] -pub struct BaoFileConfig { - /// Directory to store files in. Only used when memory limit is reached. - dir: Arc, - /// Maximum data size (inclusive) before switching to file mode. - max_mem: usize, - /// Callback to call when we switch to file mode. - /// - /// Todo: make this async. - #[debug("{:?}", on_file_create.as_ref().map(|_| ()))] - on_file_create: Option, -} - -impl BaoFileConfig { - /// Create a new deferred batch writer configuration. - pub fn new(dir: Arc, max_mem: usize, on_file_create: Option) -> Self { - Self { - dir, - max_mem, - on_file_create, - } - } - - /// Get the paths for a hash. - fn paths(&self, hash: &Hash) -> DataPaths { - DataPaths { - data: self.dir.join(format!("{}.data", hash.to_hex())), - outboard: self.dir.join(format!("{}.obao4", hash.to_hex())), - sizes: self.dir.join(format!("{}.sizes4", hash.to_hex())), - } - } -} - -/// A reader for a bao file, reading just the data. -#[derive(Debug)] -pub struct DataReader(Option); - -async fn with_storage(opt: &mut Option, no_io: P, f: F) -> io::Result -where - P: Fn(&BaoFileStorage) -> bool + Send + 'static, - F: FnOnce(&BaoFileStorage) -> io::Result + Send + 'static, - T: Send + 'static, -{ - let handle = opt - .take() - .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "deferred batch busy"))?; - // if we can get the lock immediately, and we are in memory mode, we can - // avoid spawning a task. - if let Ok(storage) = handle.storage.try_read() { - if no_io(&storage) { - let res = f(&storage); - // clone because for some reason even when we drop storage, the - // borrow checker still thinks handle is borrowed. - *opt = Some(handle.clone()); - return res; - } - }; - // otherwise, we have to spawn a task. - let (handle, res) = tokio::task::spawn_blocking(move || { - let storage = handle.storage.read().unwrap(); - let res = f(storage.deref()); - drop(storage); - (handle, res) - }) - .await - .expect("spawn_blocking failed"); - *opt = Some(handle); - res -} - -impl AsyncSliceReader for DataReader { - async fn read_at(&mut self, offset: u64, len: usize) -> io::Result { - with_storage( - &mut self.0, - BaoFileStorage::is_mem, - move |storage| match storage { - BaoFileStorage::Complete(mem) => Ok(mem.read_data_at(offset, len)), - BaoFileStorage::IncompleteMem(mem) => Ok(mem.read_data_at(offset, len)), - BaoFileStorage::IncompleteFile(file) => file.read_data_at(offset, len), - }, - ) - .await - } - - async fn size(&mut self) -> io::Result { - with_storage( - &mut self.0, - BaoFileStorage::is_mem, - move |storage| match storage { - BaoFileStorage::Complete(mem) => Ok(mem.data_size()), - BaoFileStorage::IncompleteMem(mem) => Ok(mem.data.len() as u64), - BaoFileStorage::IncompleteFile(file) => file.data.metadata().map(|m| m.len()), - }, - ) - .await - } -} - -/// A reader for the outboard part of a bao file. -#[derive(Debug)] -pub struct OutboardReader(Option); - -impl AsyncSliceReader for OutboardReader { - async fn read_at(&mut self, offset: u64, len: usize) -> io::Result { - with_storage( - &mut self.0, - BaoFileStorage::is_mem, - move |storage| match storage { - BaoFileStorage::Complete(mem) => Ok(mem.read_outboard_at(offset, len)), - BaoFileStorage::IncompleteMem(mem) => Ok(mem.read_outboard_at(offset, len)), - BaoFileStorage::IncompleteFile(file) => file.read_outboard_at(offset, len), - }, - ) - .await - } - - async fn size(&mut self) -> io::Result { - with_storage( - &mut self.0, - BaoFileStorage::is_mem, - move |storage| match storage { - BaoFileStorage::Complete(mem) => Ok(mem.outboard_size()), - BaoFileStorage::IncompleteMem(mem) => Ok(mem.outboard.len() as u64), - BaoFileStorage::IncompleteFile(file) => file.outboard.metadata().map(|m| m.len()), - }, - ) - .await - } -} - -enum HandleChange { - None, - MemToFile, - // later: size verified -} - -impl BaoFileHandle { - /// Create a new bao file handle. - /// - /// This will create a new file handle with an empty memory storage. - /// Since there are very likely to be many of these, we use an arc rwlock - pub fn incomplete_mem(config: Arc, hash: Hash) -> Self { - let storage = BaoFileStorage::incomplete_mem(); - Self(Arc::new(BaoFileHandleInner { - storage: RwLock::new(storage), - config, - hash, - })) - } - - /// Create a new bao file handle with a partial file. - pub fn incomplete_file(config: Arc, hash: Hash) -> io::Result { - let paths = config.paths(&hash); - let storage = BaoFileStorage::IncompleteFile(FileStorage { - data: create_read_write(&paths.data)?, - outboard: create_read_write(&paths.outboard)?, - sizes: create_read_write(&paths.sizes)?, - }); - Ok(Self(Arc::new(BaoFileHandleInner { - storage: RwLock::new(storage), - config, - hash, - }))) - } - - /// Create a new complete bao file handle. - pub fn new_complete( - config: Arc, - hash: Hash, - data: MemOrFile, - outboard: MemOrFile, - ) -> Self { - let storage = BaoFileStorage::Complete(CompleteStorage { data, outboard }); - Self(Arc::new(BaoFileHandleInner { - storage: RwLock::new(storage), - config, - hash, - })) - } - - /// Transform the storage in place. If the transform fails, the storage will - /// be an immutable empty storage. - #[cfg(feature = "fs-store")] - pub(crate) fn transform( - &self, - f: impl FnOnce(BaoFileStorage) -> io::Result, - ) -> io::Result<()> { - let mut lock = self.storage.write().unwrap(); - let storage = lock.take(); - *lock = f(storage)?; - Ok(()) - } - - /// True if the file is complete. - pub fn is_complete(&self) -> bool { - matches!( - self.storage.read().unwrap().deref(), - BaoFileStorage::Complete(_) - ) - } - - /// An AsyncSliceReader for the data file. - /// - /// Caution: this is a reader for the unvalidated data file. Reading this - /// can produce data that does not match the hash. - pub fn data_reader(&self) -> DataReader { - DataReader(Some(self.clone())) - } - - /// An AsyncSliceReader for the outboard file. - /// - /// The outboard file is used to validate the data file. It is not guaranteed - /// to be complete. - pub fn outboard_reader(&self) -> OutboardReader { - OutboardReader(Some(self.clone())) - } - - /// The most precise known total size of the data file. - pub fn current_size(&self) -> io::Result { - match self.storage.read().unwrap().deref() { - BaoFileStorage::Complete(mem) => Ok(mem.data_size()), - BaoFileStorage::IncompleteMem(mem) => Ok(mem.current_size()), - BaoFileStorage::IncompleteFile(file) => file.current_size(), - } - } - - /// The outboard for the file. - pub fn outboard(&self) -> io::Result> { - let root = self.hash.into(); - let tree = BaoTree::new(self.current_size()?, IROH_BLOCK_SIZE); - let outboard = self.outboard_reader(); - Ok(PreOrderOutboard { - root, - tree, - data: outboard, - }) - } - - /// The hash of the file. - pub fn hash(&self) -> Hash { - self.hash - } - - /// Create a new writer from the handle. - pub fn writer(&self) -> BaoFileWriter { - BaoFileWriter(Some(self.clone())) - } - - /// This is the synchronous impl for writing a batch. - fn write_batch(&self, size: u64, batch: &[BaoContentItem]) -> io::Result { - let mut storage = self.storage.write().unwrap(); - match storage.deref_mut() { - BaoFileStorage::IncompleteMem(mem) => { - // check if we need to switch to file mode, otherwise write to memory - if max_offset(batch) <= self.config.max_mem as u64 { - mem.write_batch(size, batch)?; - Ok(HandleChange::None) - } else { - // create the paths. This allocates 3 pathbufs, so we do it - // only when we need to. - let paths = self.config.paths(&self.hash); - // *first* switch to file mode, *then* write the batch. - // - // otherwise we might allocate a lot of memory if we get - // a write at the end of a very large file. - let mut file_batch = mem.persist(paths)?; - file_batch.write_batch(size, batch)?; - *storage = BaoFileStorage::IncompleteFile(file_batch); - Ok(HandleChange::MemToFile) - } - } - BaoFileStorage::IncompleteFile(file) => { - // already in file mode, just write the batch - file.write_batch(size, batch)?; - Ok(HandleChange::None) - } - BaoFileStorage::Complete(_) => { - // we are complete, so just ignore the write - // unless there is a bug, this would just write the exact same data - Ok(HandleChange::None) - } - } - } - - /// Downgrade to a weak reference. - pub fn downgrade(&self) -> BaoFileHandleWeak { - BaoFileHandleWeak(Arc::downgrade(&self.0)) - } -} - -impl SizeInfo { - /// Persist into a file where each chunk has its own slot. - pub fn persist(&self, mut target: impl WriteAt) -> io::Result<()> { - let size_offset = (self.offset >> IROH_BLOCK_SIZE.chunk_log()) << 3; - target.write_all_at(size_offset, self.size.to_le_bytes().as_slice())?; - Ok(()) - } - - /// Convert to a vec in slot format. - pub fn to_vec(&self) -> Vec { - let mut res = Vec::new(); - self.persist(&mut res).expect("io error writing to vec"); - res - } -} - -impl MutableMemStorage { - /// Persist the batch to disk, creating a FileBatch. - fn persist(&self, paths: DataPaths) -> io::Result { - let mut data = create_read_write(&paths.data)?; - let mut outboard = create_read_write(&paths.outboard)?; - let mut sizes = create_read_write(&paths.sizes)?; - self.data.persist(&mut data)?; - self.outboard.persist(&mut outboard)?; - self.sizes.persist(&mut sizes)?; - data.sync_all()?; - outboard.sync_all()?; - sizes.sync_all()?; - Ok(FileStorage { - data, - outboard, - sizes, - }) - } - - /// Get the parts data, outboard and sizes - pub fn into_parts(self) -> (SparseMemFile, SparseMemFile, SizeInfo) { - (self.data, self.outboard, self.sizes) - } -} - -/// This is finally the thing for which we can implement BaoPairMut. -/// -/// It is a BaoFileHandle wrapped in an Option, so that we can take it out -/// in the future. -#[derive(Debug)] -pub struct BaoFileWriter(Option); - -impl BaoBatchWriter for BaoFileWriter { - async fn write_batch(&mut self, size: u64, batch: Vec) -> std::io::Result<()> { - let Some(handle) = self.0.take() else { - return Err(io::Error::new(io::ErrorKind::Other, "deferred batch busy")); - }; - let (handle, change) = tokio::task::spawn_blocking(move || { - let change = handle.write_batch(size, &batch); - (handle, change) - }) - .await - .expect("spawn_blocking failed"); - match change? { - HandleChange::None => {} - HandleChange::MemToFile => { - if let Some(cb) = handle.config.on_file_create.as_ref() { - cb(&handle.hash)?; - } - } - } - self.0 = Some(handle); - Ok(()) - } - - async fn sync(&mut self) -> io::Result<()> { - let Some(handle) = self.0.take() else { - return Err(io::Error::new(io::ErrorKind::Other, "deferred batch busy")); - }; - let (handle, res) = tokio::task::spawn_blocking(move || { - let res = handle.storage.write().unwrap().sync_all(); - (handle, res) - }) - .await - .expect("spawn_blocking failed"); - self.0 = Some(handle); - res - } -} - -#[cfg(test)] -pub mod test_support { - use std::{future::Future, io::Cursor, ops::Range}; - - use bao_tree::{ - io::{ - fsm::{ResponseDecoder, ResponseDecoderNext}, - outboard::PostOrderMemOutboard, - round_up_to_chunks, - sync::encode_ranges_validated, - }, - BlockSize, ChunkRanges, - }; - use futures_lite::{Stream, StreamExt}; - use iroh_io::AsyncStreamReader; - use rand::RngCore; - use range_collections::RangeSet2; - - use super::*; - use crate::util::limited_range; - - pub const IROH_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4); - - /// Decode a response into a batch file writer. - pub async fn decode_response_into_batch( - root: Hash, - block_size: BlockSize, - ranges: ChunkRanges, - mut encoded: R, - mut target: W, - ) -> io::Result<()> - where - R: AsyncStreamReader, - W: BaoBatchWriter, - { - let size = encoded.read::<8>().await?; - let size = u64::from_le_bytes(size); - let mut reading = - ResponseDecoder::new(root.into(), ranges, BaoTree::new(size, block_size), encoded); - let mut stack = Vec::new(); - loop { - let item = match reading.next().await { - ResponseDecoderNext::Done(_reader) => break, - ResponseDecoderNext::More((next, item)) => { - reading = next; - item? - } - }; - match item { - BaoContentItem::Parent(_) => { - stack.push(item); - } - BaoContentItem::Leaf(_) => { - // write a batch every time we see a leaf - // the last item will be a leaf. - stack.push(item); - target.write_batch(size, std::mem::take(&mut stack)).await?; - } - } - } - assert!(stack.is_empty(), "last item should be a leaf"); - Ok(()) - } - - pub fn random_test_data(size: usize) -> Vec { - let mut rand = rand::thread_rng(); - let mut res = vec![0u8; size]; - rand.fill_bytes(&mut res); - res - } - - /// Take some data and encode it - pub fn simulate_remote(data: &[u8]) -> (Hash, Cursor) { - let outboard = bao_tree::io::outboard::PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE); - let size = data.len() as u64; - let mut encoded = size.to_le_bytes().to_vec(); - bao_tree::io::sync::encode_ranges_validated( - data, - &outboard, - &ChunkRanges::all(), - &mut encoded, - ) - .unwrap(); - let hash = outboard.root; - (hash.into(), Cursor::new(encoded.into())) - } - - pub fn to_ranges(ranges: &[Range]) -> RangeSet2 { - let mut range_set = RangeSet2::empty(); - for range in ranges.as_ref().iter().cloned() { - range_set |= RangeSet2::from(range); - } - range_set - } - - /// Simulate the send side, when asked to send bao encoded data for the given ranges. - pub fn make_wire_data( - data: &[u8], - ranges: impl AsRef<[Range]>, - ) -> (Hash, ChunkRanges, Vec) { - // compute a range set from the given ranges - let range_set = to_ranges(ranges.as_ref()); - // round up to chunks - let chunk_ranges = round_up_to_chunks(&range_set); - // compute the outboard - let outboard = PostOrderMemOutboard::create(data, IROH_BLOCK_SIZE).flip(); - let size = data.len() as u64; - let mut encoded = size.to_le_bytes().to_vec(); - encode_ranges_validated(data, &outboard, &chunk_ranges, &mut encoded).unwrap(); - (outboard.root.into(), chunk_ranges, encoded) - } - - pub async fn validate(handle: &BaoFileHandle, original: &[u8], ranges: &[Range]) { - let mut r = handle.data_reader(); - for range in ranges { - let start = range.start; - let len = (range.end - range.start).try_into().unwrap(); - let data = &original[limited_range(start, len, original.len())]; - let read = r.read_at(start, len).await.unwrap(); - assert_eq!(data.len(), read.as_ref().len()); - assert_eq!(data, read.as_ref()); - } - } - - /// Helper to simulate a slow request. - pub fn trickle( - data: &[u8], - mtu: usize, - delay: std::time::Duration, - ) -> impl Stream { - let parts = data - .chunks(mtu) - .map(Bytes::copy_from_slice) - .collect::>(); - futures_lite::stream::iter(parts).then(move |part| async move { - tokio::time::sleep(delay).await; - part - }) - } - - pub async fn local(f: F) -> F::Output - where - F: Future, - { - tokio::task::LocalSet::new().run_until(f).await - } -} - -#[cfg(test)] -mod tests { - use std::io::Write; - - use bao_tree::{blake3, ChunkNum, ChunkRanges}; - use futures_lite::StreamExt; - use iroh_io::TokioStreamReader; - use tests::test_support::{ - decode_response_into_batch, local, make_wire_data, random_test_data, trickle, validate, - }; - use tokio::task::JoinSet; - - use super::*; - use crate::util::local_pool::LocalPool; - - #[tokio::test] - async fn partial_downloads() { - local(async move { - let n = 1024 * 64u64; - let test_data = random_test_data(n as usize); - let temp_dir = tempfile::tempdir().unwrap(); - let hash = blake3::hash(&test_data); - let handle = BaoFileHandle::incomplete_mem( - Arc::new(BaoFileConfig::new( - Arc::new(temp_dir.as_ref().to_owned()), - 1024 * 16, - None, - )), - hash.into(), - ); - let mut tasks = JoinSet::new(); - for i in 1..3 { - let file = handle.writer(); - let range = (i * (n / 4))..((i + 1) * (n / 4)); - println!("range: {:?}", range); - let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &[range]); - let trickle = trickle(&wire_data, 1200, std::time::Duration::from_millis(10)) - .map(io::Result::Ok) - .boxed(); - let trickle = TokioStreamReader::new(tokio_util::io::StreamReader::new(trickle)); - let _task = tasks.spawn_local(async move { - decode_response_into_batch(hash, IROH_BLOCK_SIZE, chunk_ranges, trickle, file) - .await - }); - } - while let Some(res) = tasks.join_next().await { - res.unwrap().unwrap(); - } - println!( - "len {:?} {:?}", - handle, - handle.data_reader().size().await.unwrap() - ); - #[allow(clippy::single_range_in_vec_init)] - let ranges = [1024 * 16..1024 * 48]; - validate(&handle, &test_data, &ranges).await; - - // let ranges = - // let full_chunks = bao_tree::io::full_chunk_groups(); - let mut encoded = Vec::new(); - let ob = handle.outboard().unwrap(); - encoded - .write_all(ob.tree.size().to_le_bytes().as_slice()) - .unwrap(); - bao_tree::io::fsm::encode_ranges_validated( - handle.data_reader(), - ob, - &ChunkRanges::from(ChunkNum(16)..ChunkNum(48)), - encoded, - ) - .await - .unwrap(); - }) - .await; - } - - #[tokio::test] - async fn concurrent_downloads() { - let n = 1024 * 32u64; - let test_data = random_test_data(n as usize); - let temp_dir = tempfile::tempdir().unwrap(); - let hash = blake3::hash(&test_data); - let handle = BaoFileHandle::incomplete_mem( - Arc::new(BaoFileConfig::new( - Arc::new(temp_dir.as_ref().to_owned()), - 1024 * 16, - None, - )), - hash.into(), - ); - let local = LocalPool::default(); - let mut tasks = Vec::new(); - for i in 0..4 { - let file = handle.writer(); - let range = (i * (n / 4))..((i + 1) * (n / 4)); - println!("range: {:?}", range); - let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &[range]); - let trickle = trickle(&wire_data, 1200, std::time::Duration::from_millis(10)) - .map(io::Result::Ok) - .boxed(); - let trickle = TokioStreamReader::new(tokio_util::io::StreamReader::new(trickle)); - let task = local.spawn(move || async move { - decode_response_into_batch(hash, IROH_BLOCK_SIZE, chunk_ranges, trickle, file).await - }); - tasks.push(task); - } - for task in tasks { - task.await.unwrap().unwrap(); - } - println!( - "len {:?} {:?}", - handle, - handle.data_reader().size().await.unwrap() - ); - #[allow(clippy::single_range_in_vec_init)] - let ranges = [0..n]; - validate(&handle, &test_data, &ranges).await; - - let mut encoded = Vec::new(); - let ob = handle.outboard().unwrap(); - encoded - .write_all(ob.tree.size().to_le_bytes().as_slice()) - .unwrap(); - bao_tree::io::fsm::encode_ranges_validated( - handle.data_reader(), - ob, - &ChunkRanges::all(), - encoded, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn stay_in_mem() { - let test_data = random_test_data(1024 * 17); - #[allow(clippy::single_range_in_vec_init)] - let ranges = [0..test_data.len().try_into().unwrap()]; - let (hash, chunk_ranges, wire_data) = make_wire_data(&test_data, &ranges); - println!("file len is {:?}", chunk_ranges); - let temp_dir = tempfile::tempdir().unwrap(); - let handle = BaoFileHandle::incomplete_mem( - Arc::new(BaoFileConfig::new( - Arc::new(temp_dir.as_ref().to_owned()), - 1024 * 16, - None, - )), - hash, - ); - decode_response_into_batch( - hash, - IROH_BLOCK_SIZE, - chunk_ranges, - wire_data.as_slice(), - handle.writer(), - ) - .await - .unwrap(); - validate(&handle, &test_data, &ranges).await; - - let mut encoded = Vec::new(); - let ob = handle.outboard().unwrap(); - encoded - .write_all(ob.tree.size().to_le_bytes().as_slice()) - .unwrap(); - bao_tree::io::fsm::encode_ranges_validated( - handle.data_reader(), - ob, - &ChunkRanges::all(), - encoded, - ) - .await - .unwrap(); - println!("{:?}", handle); - } -} diff --git a/iroh-blobs/src/store/fs.rs b/iroh-blobs/src/store/fs.rs deleted file mode 100644 index f0d666f1217..00000000000 --- a/iroh-blobs/src/store/fs.rs +++ /dev/null @@ -1,2612 +0,0 @@ -//! redb backed storage -//! -//! Data can get into the store in two ways: -//! -//! 1. import from local data -//! 2. sync from a remote -//! -//! These two cases are very different. In the first case, we have the data -//! completely and don't know the hash yet. We compute the outboard and hash, -//! and only then move/reference the data into the store. -//! -//! The entry for the hash comes into existence already complete. -//! -//! In the second case, we know the hash, but don't have the data yet. We create -//! a partial entry, and then request the data from the remote. This is the more -//! complex case. -//! -//! Partial entries always start as pure in memory entries without a database -//! entry. Only once we receive enough data, we convert them into a persistent -//! partial entry. This is necessary because we can't trust the size given -//! by the remote side before receiving data. It is also an optimization, -//! because for small blobs it is not worth it to create a partial entry. -//! -//! A persistent partial entry is always stored as three files in the file -//! system: The data file, the outboard file, and a sizes file that contains -//! the most up to date information about the size of the data. -//! -//! The redb database entry for a persistent partial entry does not contain -//! any information about the size of the data until the size is exactly known. -//! -//! Updating this information on each write would be too costly. -//! -//! Marking a partial entry as complete is done from the outside. At this point -//! the size is taken as validated. Depending on the size we decide whether to -//! store data and outboard inline or to keep storing it in external files. -//! -//! Data can get out of the store in two ways: -//! -//! 1. the data and outboard of both partial and complete entries can be read at any time and -//! shared over the network. Only data that is complete will be shared, everything else will -//! lead to validation errors. -//! -//! 2. entries can be exported to the file system. This currently only works for complete entries. -//! -//! Tables: -//! -//! The blobs table contains a mapping from hash to rough entry state. -//! The inline_data table contains the actual data for complete entries. -//! The inline_outboard table contains the actual outboard for complete entries. -//! The tags table contains a mapping from tag to hash. -//! -//! Design: -//! -//! The redb store is accessed in a single threaded way by an actor that runs -//! on its own std thread. Communication with this actor is via a flume channel, -//! with oneshot channels for the return values if needed. -//! -//! Errors: -//! -//! ActorError is an enum containing errors that can happen inside message -//! handlers of the actor. This includes various redb related errors and io -//! errors when reading or writing non-inlined data or outboard files. -//! -//! OuterError is an enum containing all the actor errors and in addition -//! errors when communicating with the actor. -use std::{ - collections::{BTreeMap, BTreeSet}, - future::Future, - io, - path::{Path, PathBuf}, - sync::{Arc, RwLock}, - time::{Duration, SystemTime}, -}; - -use bao_tree::io::{ - fsm::Outboard, - sync::{ReadAt, Size}, -}; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use genawaiter::rc::{Co, Gen}; -use iroh_base::hash::{BlobFormat, Hash, HashAndFormat}; -use iroh_io::AsyncSliceReader; -use redb::{AccessGuard, DatabaseError, ReadableTable, StorageError}; -use serde::{Deserialize, Serialize}; -use smallvec::SmallVec; -use tokio::io::AsyncWriteExt; -use tracing::trace_span; - -mod migrate_redb_v1_v2; -mod tables; -#[doc(hidden)] -pub mod test_support; -#[cfg(test)] -mod tests; -mod util; -mod validate; - -use tables::{ReadOnlyTables, ReadableTables, Tables}; - -use self::{tables::DeleteSet, test_support::EntryData, util::PeekableFlumeReceiver}; -use super::{ - bao_file::{BaoFileConfig, BaoFileHandle, BaoFileHandleWeak, CreateCb}, - temp_name, BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, EntryStatus, ExportMode, - ExportProgressCb, ImportMode, ImportProgress, Map, ReadableStore, TempCounterMap, -}; -use crate::{ - store::{ - bao_file::{BaoFileStorage, CompleteStorage}, - fs::{ - tables::BaoFilePart, - util::{overwrite_and_sync, read_and_remove}, - }, - GcMarkEvent, GcSweepEvent, - }, - util::{ - compute_outboard, - progress::{ - BoxedProgressSender, IdGenerator, IgnoreProgressSender, ProgressSendError, - ProgressSender, - }, - raw_outboard_size, MemOrFile, TagCounter, TagDrop, - }, - Tag, TempTag, -}; - -/// Location of the data. -/// -/// Data can be inlined in the database, a file conceptually owned by the store, -/// or a number of external files conceptually owned by the user. -/// -/// Only complete data can be inlined. -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) enum DataLocation { - /// Data is in the inline_data table. - Inline(I), - /// Data is in the canonical location in the data directory. - Owned(E), - /// Data is in several external locations. This should be a non-empty list. - External(Vec, E), -} - -impl DataLocation { - fn union(self, that: DataLocation) -> ActorResult { - Ok(match (self, that) { - ( - DataLocation::External(mut paths, a_size), - DataLocation::External(b_paths, b_size), - ) => { - if a_size != b_size { - return Err(ActorError::Inconsistent(format!( - "complete size mismatch {} {}", - a_size, b_size - ))); - } - paths.extend(b_paths); - paths.sort(); - paths.dedup(); - DataLocation::External(paths, a_size) - } - (_, b @ DataLocation::Owned(_)) => { - // owned needs to win, since it has an associated file. Choosing - // external would orphan the file. - b - } - (a @ DataLocation::Owned(_), _) => { - // owned needs to win, since it has an associated file. Choosing - // external would orphan the file. - a - } - (_, b @ DataLocation::Inline(_)) => { - // inline needs to win, since it has associated data. Choosing - // external would orphan the file. - b - } - (a @ DataLocation::Inline(_), _) => { - // inline needs to win, since it has associated data. Choosing - // external would orphan the file. - a - } - }) - } -} - -impl DataLocation { - fn discard_inline_data(self) -> DataLocation<(), E> { - match self { - DataLocation::Inline(_) => DataLocation::Inline(()), - DataLocation::Owned(x) => DataLocation::Owned(x), - DataLocation::External(paths, x) => DataLocation::External(paths, x), - } - } -} - -/// Location of the outboard. -/// -/// Outboard can be inlined in the database or a file conceptually owned by the store. -/// Outboards are implementation specific to the store and as such are always owned. -/// -/// Only complete outboards can be inlined. -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) enum OutboardLocation { - /// Outboard is in the inline_outboard table. - Inline(I), - /// Outboard is in the canonical location in the data directory. - Owned, - /// Outboard is not needed - NotNeeded, -} - -impl OutboardLocation { - fn discard_extra_data(self) -> OutboardLocation<()> { - match self { - Self::Inline(_) => OutboardLocation::Inline(()), - Self::Owned => OutboardLocation::Owned, - Self::NotNeeded => OutboardLocation::NotNeeded, - } - } -} - -/// The information about an entry that we keep in the entry table for quick access. -/// -/// The exact info to store here is TBD, so usually you should use the accessor methods. -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) enum EntryState { - /// For a complete entry we always know the size. It does not make much sense - /// to write to a complete entry, so they are much easier to share. - Complete { - /// Location of the data. - data_location: DataLocation, - /// Location of the outboard. - outboard_location: OutboardLocation, - }, - /// Partial entries are entries for which we know the hash, but don't have - /// all the data. They are created when syncing from somewhere else by hash. - /// - /// As such they are always owned. There is also no inline storage for them. - /// Non short lived partial entries always live in the file system, and for - /// short lived ones we never create a database entry in the first place. - Partial { - /// Once we get the last chunk of a partial entry, we have validated - /// the size of the entry despite it still being incomplete. - /// - /// E.g. a giant file where we just requested the last chunk. - size: Option, - }, -} - -impl Default for EntryState { - fn default() -> Self { - Self::Partial { size: None } - } -} - -impl EntryState { - fn union(self, that: Self) -> ActorResult { - match (self, that) { - ( - Self::Complete { - data_location, - outboard_location, - }, - Self::Complete { - data_location: b_data_location, - .. - }, - ) => Ok(Self::Complete { - // combine external paths if needed - data_location: data_location.union(b_data_location)?, - outboard_location, - }), - (a @ Self::Complete { .. }, Self::Partial { .. }) => - // complete wins over partial - { - Ok(a) - } - (Self::Partial { .. }, b @ Self::Complete { .. }) => - // complete wins over partial - { - Ok(b) - } - (Self::Partial { size: a_size }, Self::Partial { size: b_size }) => - // keep known size from either entry - { - let size = match (a_size, b_size) { - (Some(a_size), Some(b_size)) => { - // validated sizes are different. this means that at - // least one validation was wrong, which would be a bug - // in bao-tree. - if a_size != b_size { - return Err(ActorError::Inconsistent(format!( - "validated size mismatch {} {}", - a_size, b_size - ))); - } - Some(a_size) - } - (Some(a_size), None) => Some(a_size), - (None, Some(b_size)) => Some(b_size), - (None, None) => None, - }; - Ok(Self::Partial { size }) - } - } - } -} - -impl redb::Value for EntryState { - type SelfType<'a> = EntryState; - - type AsBytes<'a> = SmallVec<[u8; 128]>; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - postcard::from_bytes(data).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - postcard::to_extend(value, SmallVec::new()).unwrap() - } - - fn type_name() -> redb::TypeName { - redb::TypeName::new("EntryState") - } -} - -/// Options for inlining small complete data or outboards. -#[derive(Debug, Clone)] -pub struct InlineOptions { - /// Maximum data size to inline. - pub max_data_inlined: u64, - /// Maximum outboard size to inline. - pub max_outboard_inlined: u64, -} - -impl InlineOptions { - /// Do not inline anything, ever. - pub const NO_INLINE: Self = Self { - max_data_inlined: 0, - max_outboard_inlined: 0, - }; - /// Always inline everything - pub const ALWAYS_INLINE: Self = Self { - max_data_inlined: u64::MAX, - max_outboard_inlined: u64::MAX, - }; -} - -impl Default for InlineOptions { - fn default() -> Self { - Self { - max_data_inlined: 1024 * 16, - max_outboard_inlined: 1024 * 16, - } - } -} - -/// Options for directories used by the file store. -#[derive(Debug, Clone)] -pub struct PathOptions { - /// Path to the directory where data and outboard files are stored. - pub data_path: PathBuf, - /// Path to the directory where temp files are stored. - /// This *must* be on the same device as `data_path`, since we need to - /// atomically move temp files into place. - pub temp_path: PathBuf, -} - -impl PathOptions { - fn new(root: &Path) -> Self { - Self { - data_path: root.join("data"), - temp_path: root.join("temp"), - } - } - - fn owned_data_path(&self, hash: &Hash) -> PathBuf { - self.data_path.join(format!("{}.data", hash.to_hex())) - } - - fn owned_outboard_path(&self, hash: &Hash) -> PathBuf { - self.data_path.join(format!("{}.obao4", hash.to_hex())) - } - - fn owned_sizes_path(&self, hash: &Hash) -> PathBuf { - self.data_path.join(format!("{}.sizes4", hash.to_hex())) - } - - fn temp_file_name(&self) -> PathBuf { - self.temp_path.join(temp_name()) - } -} - -/// Options for transaction batching. -#[derive(Debug, Clone)] -pub struct BatchOptions { - /// Maximum number of actor messages to batch before creating a new read transaction. - pub max_read_batch: usize, - /// Maximum duration to wait before committing a read transaction. - pub max_read_duration: Duration, - /// Maximum number of actor messages to batch before committing write transaction. - pub max_write_batch: usize, - /// Maximum duration to wait before committing a write transaction. - pub max_write_duration: Duration, -} - -impl Default for BatchOptions { - fn default() -> Self { - Self { - max_read_batch: 10000, - max_read_duration: Duration::from_secs(1), - max_write_batch: 1000, - max_write_duration: Duration::from_millis(500), - } - } -} - -/// Options for the file store. -#[derive(Debug, Clone)] -pub struct Options { - /// Path options. - pub path: PathOptions, - /// Inline storage options. - pub inline: InlineOptions, - /// Transaction batching options. - pub batch: BatchOptions, -} - -#[derive(derive_more::Debug)] -pub(crate) enum ImportSource { - TempFile(PathBuf), - External(PathBuf), - Memory(#[debug(skip)] Bytes), -} - -impl ImportSource { - fn content(&self) -> MemOrFile<&[u8], &Path> { - match self { - Self::TempFile(path) => MemOrFile::File(path.as_path()), - Self::External(path) => MemOrFile::File(path.as_path()), - Self::Memory(data) => MemOrFile::Mem(data.as_ref()), - } - } - - fn len(&self) -> io::Result { - match self { - Self::TempFile(path) => std::fs::metadata(path).map(|m| m.len()), - Self::External(path) => std::fs::metadata(path).map(|m| m.len()), - Self::Memory(data) => Ok(data.len() as u64), - } - } -} - -/// Use BaoFileHandle as the entry type for the map. -pub type Entry = BaoFileHandle; - -impl super::MapEntry for Entry { - fn hash(&self) -> Hash { - self.hash() - } - - fn size(&self) -> BaoBlobSize { - let size = self.current_size().unwrap(); - tracing::trace!("redb::Entry::size() = {}", size); - BaoBlobSize::new(size, self.is_complete()) - } - - fn is_complete(&self) -> bool { - self.is_complete() - } - - async fn outboard(&self) -> io::Result { - self.outboard() - } - - async fn data_reader(&self) -> io::Result { - Ok(self.data_reader()) - } -} - -impl super::MapEntryMut for Entry { - async fn batch_writer(&self) -> io::Result { - Ok(self.writer()) - } -} - -#[derive(derive_more::Debug)] -pub(crate) struct Import { - /// The hash and format of the data to import - content_id: HashAndFormat, - /// The source of the data to import, can be a temp file, external file, or memory - source: ImportSource, - /// Data size - data_size: u64, - /// Outboard without length prefix - #[debug("{:?}", outboard.as_ref().map(|x| x.len()))] - outboard: Option>, -} - -#[derive(derive_more::Debug)] -pub(crate) struct Export { - /// A temp tag to keep the entry alive while exporting. This also - /// contains the hash to be exported. - temp_tag: TempTag, - /// The target path for the export. - target: PathBuf, - /// The export mode to use. - mode: ExportMode, - /// The progress callback to use. - #[debug(skip)] - progress: ExportProgressCb, -} - -#[derive(derive_more::Debug)] -pub(crate) enum ActorMessage { - // Query method: get a file handle for a hash, if it exists. - // This will produce a file handle even for entries that are not yet in redb at all. - Get { - hash: Hash, - tx: oneshot::Sender>>, - }, - /// Query method: get the rough entry status for a hash. Just complete, partial or not found. - EntryStatus { - hash: Hash, - tx: oneshot::Sender>, - }, - #[cfg(test)] - /// Query method: get the full entry state for a hash, both in memory and in redb. - /// This is everything we got about the entry, including the actual inline outboard and data. - EntryState { - hash: Hash, - tx: oneshot::Sender>, - }, - /// Query method: get the full entry state for a hash. - GetFullEntryState { - hash: Hash, - tx: oneshot::Sender>>, - }, - /// Modification method: set the full entry state for a hash. - SetFullEntryState { - hash: Hash, - entry: Option, - tx: oneshot::Sender>, - }, - /// Modification method: get or create a file handle for a hash. - /// - /// If the entry exists in redb, either partial or complete, the corresponding - /// data will be returned. If it does not yet exist, a new partial file handle - /// will be created, but not yet written to redb. - GetOrCreate { - hash: Hash, - tx: oneshot::Sender>, - }, - /// Modification method: inline size was exceeded for a partial entry. - /// If the entry is complete, this is a no-op. If the entry is partial and in - /// memory, it will be written to a file and created in redb. - OnMemSizeExceeded { hash: Hash }, - /// Modification method: marks a partial entry as complete. - /// Calling this on a complete entry is a no-op. - OnComplete { handle: BaoFileHandle }, - /// Modification method: import data into a redb store - /// - /// At this point the size, hash and outboard must already be known. - Import { - cmd: Import, - tx: oneshot::Sender>, - }, - /// Modification method: export data from a redb store - /// - /// In most cases this will not modify the store. Only when using - /// [`ExportMode::TryReference`] and the entry is large enough to not be - /// inlined. - Export { - cmd: Export, - tx: oneshot::Sender>, - }, - /// Update inline options - UpdateInlineOptions { - /// The new inline options - inline_options: InlineOptions, - /// Whether to reapply the new options to existing entries - reapply: bool, - tx: oneshot::Sender<()>, - }, - /// Bulk query method: get entries from the blobs table - Blobs { - #[debug(skip)] - filter: FilterPredicate, - #[allow(clippy::type_complexity)] - tx: oneshot::Sender< - ActorResult>>, - >, - }, - /// Bulk query method: get the entire tags table - Tags { - #[debug(skip)] - filter: FilterPredicate, - #[allow(clippy::type_complexity)] - tx: oneshot::Sender< - ActorResult>>, - >, - }, - /// Modification method: set a tag to a value, or remove it. - SetTag { - tag: Tag, - value: Option, - tx: oneshot::Sender>, - }, - /// Modification method: create a new unique tag and set it to a value. - CreateTag { - hash: HashAndFormat, - tx: oneshot::Sender>, - }, - /// Modification method: unconditional delete the data for a number of hashes - Delete { - hashes: Vec, - tx: oneshot::Sender>, - }, - /// Modification method: delete the data for a number of hashes, only if not protected - GcDelete { - hashes: Vec, - tx: oneshot::Sender>, - }, - /// Sync the entire database to disk. - /// - /// This just makes sure that there is no write transaction open. - Sync { tx: oneshot::Sender<()> }, - /// Internal method: dump the entire database to stdout. - Dump, - /// Internal method: validate the entire database. - /// - /// Note that this will block the actor until it is done, so don't use it - /// on a node under load. - Fsck { - repair: bool, - progress: BoxedProgressSender, - tx: oneshot::Sender>, - }, - /// Internal method: notify the actor that a new gc epoch has started. - /// - /// This will be called periodically and can be used to do misc cleanups. - GcStart { tx: oneshot::Sender<()> }, - /// Internal method: shutdown the actor. - /// - /// Can have an optional oneshot sender to signal when the actor has shut down. - Shutdown { tx: Option> }, -} - -impl ActorMessage { - fn category(&self) -> MessageCategory { - match self { - Self::Get { .. } - | Self::GetOrCreate { .. } - | Self::EntryStatus { .. } - | Self::Blobs { .. } - | Self::Tags { .. } - | Self::GcStart { .. } - | Self::GetFullEntryState { .. } - | Self::Dump => MessageCategory::ReadOnly, - Self::Import { .. } - | Self::Export { .. } - | Self::OnMemSizeExceeded { .. } - | Self::OnComplete { .. } - | Self::SetTag { .. } - | Self::CreateTag { .. } - | Self::SetFullEntryState { .. } - | Self::Delete { .. } - | Self::GcDelete { .. } => MessageCategory::ReadWrite, - Self::UpdateInlineOptions { .. } - | Self::Sync { .. } - | Self::Shutdown { .. } - | Self::Fsck { .. } => MessageCategory::TopLevel, - #[cfg(test)] - Self::EntryState { .. } => MessageCategory::ReadOnly, - } - } -} - -enum MessageCategory { - ReadOnly, - ReadWrite, - TopLevel, -} - -/// Predicate for filtering entries in a redb table. -pub(crate) type FilterPredicate = - Box, AccessGuard) -> Option<(K, V)> + Send + Sync>; - -/// Storage that is using a redb database for small files and files for -/// large files. -#[derive(Debug, Clone)] -pub struct Store(Arc); - -impl Store { - /// Load or create a new store. - pub async fn load(root: impl AsRef) -> io::Result { - let path = root.as_ref(); - let db_path = path.join("blobs.db"); - let options = Options { - path: PathOptions::new(path), - inline: Default::default(), - batch: Default::default(), - }; - Self::new(db_path, options).await - } - - /// Create a new store with custom options. - pub async fn new(path: PathBuf, options: Options) -> io::Result { - // spawn_blocking because StoreInner::new creates directories - let rt = tokio::runtime::Handle::try_current() - .map_err(|_| io::Error::new(io::ErrorKind::Other, "no tokio runtime"))?; - let inner = - tokio::task::spawn_blocking(move || StoreInner::new_sync(path, options, rt)).await??; - Ok(Self(Arc::new(inner))) - } - - /// Update the inline options. - /// - /// When reapply is true, the new options will be applied to all existing - /// entries. - pub async fn update_inline_options( - &self, - inline_options: InlineOptions, - reapply: bool, - ) -> io::Result<()> { - Ok(self - .0 - .update_inline_options(inline_options, reapply) - .await?) - } - - /// Dump the entire content of the database to stdout. - pub async fn dump(&self) -> io::Result<()> { - Ok(self.0.dump().await?) - } -} - -#[derive(Debug)] -struct StoreInner { - tx: async_channel::Sender, - temp: Arc>, - handle: Option>, - path_options: Arc, -} - -impl TagDrop for RwLock { - fn on_drop(&self, content: &HashAndFormat) { - self.write().unwrap().dec(content); - } -} - -impl TagCounter for RwLock { - fn on_create(&self, content: &HashAndFormat) { - self.write().unwrap().inc(content); - } -} - -impl StoreInner { - fn new_sync(path: PathBuf, options: Options, rt: tokio::runtime::Handle) -> io::Result { - tracing::trace!( - "creating data directory: {}", - options.path.data_path.display() - ); - std::fs::create_dir_all(&options.path.data_path)?; - tracing::trace!( - "creating temp directory: {}", - options.path.temp_path.display() - ); - std::fs::create_dir_all(&options.path.temp_path)?; - tracing::trace!( - "creating parent directory for db file{}", - path.parent().unwrap().display() - ); - std::fs::create_dir_all(path.parent().unwrap())?; - let temp: Arc> = Default::default(); - let (actor, tx) = Actor::new(&path, options.clone(), temp.clone(), rt.clone())?; - let handle = std::thread::Builder::new() - .name("redb-actor".to_string()) - .spawn(move || { - rt.block_on(async move { - if let Err(cause) = actor.run_batched().await { - tracing::error!("redb actor failed: {}", cause); - } - }); - }) - .expect("failed to spawn thread"); - Ok(Self { - tx, - temp, - handle: Some(handle), - path_options: Arc::new(options.path), - }) - } - - pub async fn get(&self, hash: Hash) -> OuterResult> { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::Get { hash, tx }).await?; - Ok(rx.await??) - } - - async fn get_or_create(&self, hash: Hash) -> OuterResult { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::GetOrCreate { hash, tx }).await?; - Ok(rx.await??) - } - - async fn blobs(&self) -> OuterResult>> { - let (tx, rx) = oneshot::channel(); - let filter: FilterPredicate = Box::new(|_i, k, v| { - let v = v.value(); - if let EntryState::Complete { .. } = &v { - Some((k.value(), v)) - } else { - None - } - }); - self.tx.send(ActorMessage::Blobs { filter, tx }).await?; - let blobs = rx.await?; - let res = blobs? - .into_iter() - .map(|r| { - r.map(|(hash, _)| hash) - .map_err(|e| ActorError::from(e).into()) - }) - .collect::>(); - Ok(res) - } - - async fn partial_blobs(&self) -> OuterResult>> { - let (tx, rx) = oneshot::channel(); - let filter: FilterPredicate = Box::new(|_i, k, v| { - let v = v.value(); - if let EntryState::Partial { .. } = &v { - Some((k.value(), v)) - } else { - None - } - }); - self.tx.send(ActorMessage::Blobs { filter, tx }).await?; - let blobs = rx.await?; - let res = blobs? - .into_iter() - .map(|r| { - r.map(|(hash, _)| hash) - .map_err(|e| ActorError::from(e).into()) - }) - .collect::>(); - Ok(res) - } - - async fn tags(&self) -> OuterResult>> { - let (tx, rx) = oneshot::channel(); - let filter: FilterPredicate = - Box::new(|_i, k, v| Some((k.value(), v.value()))); - self.tx.send(ActorMessage::Tags { filter, tx }).await?; - let tags = rx.await?; - // transform the internal error type into io::Error - let tags = tags? - .into_iter() - .map(|r| r.map_err(|e| ActorError::from(e).into())) - .collect(); - Ok(tags) - } - - async fn set_tag(&self, tag: Tag, value: Option) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::SetTag { tag, value, tx }) - .await?; - Ok(rx.await??) - } - - async fn create_tag(&self, hash: HashAndFormat) -> OuterResult { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::CreateTag { hash, tx }).await?; - Ok(rx.await??) - } - - async fn delete(&self, hashes: Vec) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::Delete { hashes, tx }).await?; - Ok(rx.await??) - } - - async fn gc_delete(&self, hashes: Vec) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::GcDelete { hashes, tx }).await?; - Ok(rx.await??) - } - - async fn gc_start(&self) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::GcStart { tx }).await?; - Ok(rx.await?) - } - - async fn entry_status(&self, hash: &Hash) -> OuterResult { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::EntryStatus { hash: *hash, tx }) - .await?; - Ok(rx.await??) - } - - fn entry_status_sync(&self, hash: &Hash) -> OuterResult { - let (tx, rx) = oneshot::channel(); - self.tx - .send_blocking(ActorMessage::EntryStatus { hash: *hash, tx })?; - Ok(rx.recv()??) - } - - async fn complete(&self, entry: Entry) -> OuterResult<()> { - self.tx - .send(ActorMessage::OnComplete { handle: entry }) - .await?; - Ok(()) - } - - async fn export( - &self, - hash: Hash, - target: PathBuf, - mode: ExportMode, - progress: ExportProgressCb, - ) -> OuterResult<()> { - tracing::debug!( - "exporting {} to {} using mode {:?}", - hash.to_hex(), - target.display(), - mode - ); - if !target.is_absolute() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "target path must be absolute", - ) - .into()); - } - let parent = target.parent().ok_or_else(|| { - OuterError::from(io::Error::new( - io::ErrorKind::InvalidInput, - "target path has no parent directory", - )) - })?; - std::fs::create_dir_all(parent)?; - let temp_tag = self.temp.temp_tag(HashAndFormat::raw(hash)); - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::Export { - cmd: Export { - temp_tag, - target, - mode, - progress, - }, - tx, - }) - .await?; - Ok(rx.await??) - } - - async fn consistency_check( - &self, - repair: bool, - progress: BoxedProgressSender, - ) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::Fsck { - repair, - progress, - tx, - }) - .await?; - Ok(rx.await??) - } - - async fn update_inline_options( - &self, - inline_options: InlineOptions, - reapply: bool, - ) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::UpdateInlineOptions { - inline_options, - reapply, - tx, - }) - .await?; - Ok(rx.await?) - } - - async fn dump(&self) -> OuterResult<()> { - self.tx.send(ActorMessage::Dump).await?; - Ok(()) - } - - async fn sync(&self) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::Sync { tx }).await?; - Ok(rx.await?) - } - - fn import_file_sync( - &self, - path: PathBuf, - mode: ImportMode, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> OuterResult<(TempTag, u64)> { - if !path.is_absolute() { - return Err( - io::Error::new(io::ErrorKind::InvalidInput, "path must be absolute").into(), - ); - } - if !path.is_file() && !path.is_symlink() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path is not a file or symlink", - ) - .into()); - } - let id = progress.new_id(); - progress.blocking_send(ImportProgress::Found { - id, - name: path.to_string_lossy().to_string(), - })?; - let file = match mode { - ImportMode::TryReference => ImportSource::External(path), - ImportMode::Copy => { - if std::fs::metadata(&path)?.len() < 16 * 1024 { - // we don't know if the data will be inlined since we don't - // have the inline options here. But still for such a small file - // it does not seem worth it do to the temp file ceremony. - let data = std::fs::read(&path)?; - ImportSource::Memory(data.into()) - } else { - let temp_path = self.temp_file_name(); - // copy the data, since it is not stable - progress.try_send(ImportProgress::CopyProgress { id, offset: 0 })?; - if reflink_copy::reflink_or_copy(&path, &temp_path)?.is_none() { - tracing::debug!("reflinked {} to {}", path.display(), temp_path.display()); - } else { - tracing::debug!("copied {} to {}", path.display(), temp_path.display()); - } - // copy progress for size will be called in finalize_import_sync - ImportSource::TempFile(temp_path) - } - } - }; - let (tag, size) = self.finalize_import_sync(file, format, id, progress)?; - Ok((tag, size)) - } - - fn import_bytes_sync(&self, data: Bytes, format: BlobFormat) -> OuterResult { - let id = 0; - let file = ImportSource::Memory(data); - let progress = IgnoreProgressSender::default(); - let (tag, _size) = self.finalize_import_sync(file, format, id, progress)?; - Ok(tag) - } - - fn finalize_import_sync( - &self, - file: ImportSource, - format: BlobFormat, - id: u64, - progress: impl ProgressSender + IdGenerator, - ) -> OuterResult<(TempTag, u64)> { - let data_size = file.len()?; - tracing::debug!("finalize_import_sync {:?} {}", file, data_size); - progress.blocking_send(ImportProgress::Size { - id, - size: data_size, - })?; - let progress2 = progress.clone(); - let (hash, outboard) = match file.content() { - MemOrFile::File(path) => { - let span = trace_span!("outboard.compute", path = %path.display()); - let _guard = span.enter(); - let file = std::fs::File::open(path)?; - compute_outboard(file, data_size, move |offset| { - Ok(progress2.try_send(ImportProgress::OutboardProgress { id, offset })?) - })? - } - MemOrFile::Mem(bytes) => { - // todo: progress? usually this is will be small enough that progress might not be needed. - compute_outboard(bytes, data_size, |_| Ok(()))? - } - }; - progress.blocking_send(ImportProgress::OutboardDone { id, hash })?; - // from here on, everything related to the hash is protected by the temp tag - let tag = self.temp.temp_tag(HashAndFormat { hash, format }); - let hash = *tag.hash(); - // blocking send for the import - let (tx, rx) = oneshot::channel(); - self.tx.send_blocking(ActorMessage::Import { - cmd: Import { - content_id: HashAndFormat { hash, format }, - source: file, - outboard, - data_size, - }, - tx, - })?; - Ok(rx.recv()??) - } - - fn temp_file_name(&self) -> PathBuf { - self.path_options.temp_file_name() - } - - async fn shutdown(&self) { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::Shutdown { tx: Some(tx) }) - .await - .ok(); - rx.await.ok(); - } -} - -impl Drop for StoreInner { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - self.tx - .send_blocking(ActorMessage::Shutdown { tx: None }) - .ok(); - handle.join().ok(); - } - } -} - -struct ActorState { - handles: BTreeMap, - protected: BTreeSet, - temp: Arc>, - msgs_rx: async_channel::Receiver, - create_options: Arc, - options: Options, - rt: tokio::runtime::Handle, -} - -/// The actor for the redb store. -/// -/// It is split into the database and the rest of the state to allow for split -/// borrows in the message handlers. -struct Actor { - db: redb::Database, - state: ActorState, -} - -/// Error type for message handler functions of the redb actor. -/// -/// What can go wrong are various things with redb, as well as io errors related -/// to files other than redb. -#[derive(Debug, thiserror::Error)] -pub(crate) enum ActorError { - #[error("table error: {0}")] - Table(#[from] redb::TableError), - #[error("database error: {0}")] - Database(#[from] redb::DatabaseError), - #[error("transaction error: {0}")] - Transaction(#[from] redb::TransactionError), - #[error("commit error: {0}")] - Commit(#[from] redb::CommitError), - #[error("storage error: {0}")] - Storage(#[from] redb::StorageError), - #[error("io error: {0}")] - Io(#[from] io::Error), - #[error("inconsistent database state: {0}")] - Inconsistent(String), - #[error("error during database migration: {0}")] - Migration(#[source] anyhow::Error), -} - -impl From for io::Error { - fn from(e: ActorError) -> Self { - match e { - ActorError::Io(e) => e, - e => io::Error::new(io::ErrorKind::Other, e), - } - } -} - -/// Result type for handler functions of the redb actor. -/// -/// See [`ActorError`] for what can go wrong. -pub(crate) type ActorResult = std::result::Result; - -/// Error type for calling the redb actor from the store. -/// -/// What can go wrong is all the things in [`ActorError`] and in addition -/// sending and receiving messages. -#[derive(Debug, thiserror::Error)] -pub(crate) enum OuterError { - #[error("inner error: {0}")] - Inner(#[from] ActorError), - #[error("send error")] - Send, - #[error("progress send error: {0}")] - ProgressSend(#[from] ProgressSendError), - #[error("recv error: {0}")] - Recv(#[from] oneshot::RecvError), - #[error("recv error: {0}")] - AsyncChannelRecv(#[from] async_channel::RecvError), - #[error("join error: {0}")] - JoinTask(#[from] tokio::task::JoinError), -} - -impl From> for OuterError { - fn from(_e: async_channel::SendError) -> Self { - OuterError::Send - } -} - -/// Result type for calling the redb actor from the store. -/// -/// See [`OuterError`] for what can go wrong. -pub(crate) type OuterResult = std::result::Result; - -impl From for OuterError { - fn from(e: io::Error) -> Self { - OuterError::Inner(ActorError::Io(e)) - } -} - -impl From for io::Error { - fn from(e: OuterError) -> Self { - match e { - OuterError::Inner(ActorError::Io(e)) => e, - e => io::Error::new(io::ErrorKind::Other, e), - } - } -} - -impl super::Map for Store { - type Entry = Entry; - - async fn get(&self, hash: &Hash) -> io::Result> { - Ok(self.0.get(*hash).await?.map(From::from)) - } -} - -impl super::MapMut for Store { - type EntryMut = Entry; - - async fn get_or_create(&self, hash: Hash, _size: u64) -> io::Result { - Ok(self.0.get_or_create(hash).await?) - } - - async fn entry_status(&self, hash: &Hash) -> io::Result { - Ok(self.0.entry_status(hash).await?) - } - - async fn get_mut(&self, hash: &Hash) -> io::Result> { - self.get(hash).await - } - - async fn insert_complete(&self, entry: Self::EntryMut) -> io::Result<()> { - Ok(self.0.complete(entry).await?) - } - - fn entry_status_sync(&self, hash: &Hash) -> io::Result { - Ok(self.0.entry_status_sync(hash)?) - } -} - -impl super::ReadableStore for Store { - async fn blobs(&self) -> io::Result> { - Ok(Box::new(self.0.blobs().await?.into_iter())) - } - - async fn partial_blobs(&self) -> io::Result> { - Ok(Box::new(self.0.partial_blobs().await?.into_iter())) - } - - async fn tags(&self) -> io::Result> { - Ok(Box::new(self.0.tags().await?.into_iter())) - } - - fn temp_tags(&self) -> Box + Send + Sync + 'static> { - Box::new(self.0.temp.read().unwrap().keys()) - } - - async fn consistency_check( - &self, - repair: bool, - tx: BoxedProgressSender, - ) -> io::Result<()> { - self.0.consistency_check(repair, tx.clone()).await?; - Ok(()) - } - - async fn export( - &self, - hash: Hash, - target: PathBuf, - mode: ExportMode, - progress: ExportProgressCb, - ) -> io::Result<()> { - Ok(self.0.export(hash, target, mode, progress).await?) - } -} - -impl super::Store for Store { - async fn import_file( - &self, - path: PathBuf, - mode: ImportMode, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(crate::TempTag, u64)> { - let this = self.0.clone(); - Ok( - tokio::task::spawn_blocking(move || { - this.import_file_sync(path, mode, format, progress) - }) - .await??, - ) - } - - async fn import_bytes( - &self, - data: bytes::Bytes, - format: iroh_base::hash::BlobFormat, - ) -> io::Result { - let this = self.0.clone(); - Ok(tokio::task::spawn_blocking(move || this.import_bytes_sync(data, format)).await??) - } - - async fn import_stream( - &self, - mut data: impl Stream> + Unpin + Send + 'static, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(TempTag, u64)> { - let this = self.clone(); - let id = progress.new_id(); - // write to a temp file - let temp_data_path = this.0.temp_file_name(); - let name = temp_data_path - .file_name() - .expect("just created") - .to_string_lossy() - .to_string(); - progress.send(ImportProgress::Found { id, name }).await?; - let mut writer = tokio::fs::File::create(&temp_data_path).await?; - let mut offset = 0; - while let Some(chunk) = data.next().await { - let chunk = chunk?; - writer.write_all(&chunk).await?; - offset += chunk.len() as u64; - progress.try_send(ImportProgress::CopyProgress { id, offset })?; - } - writer.flush().await?; - drop(writer); - let file = ImportSource::TempFile(temp_data_path); - Ok(tokio::task::spawn_blocking(move || { - this.0.finalize_import_sync(file, format, id, progress) - }) - .await??) - } - - async fn set_tag(&self, name: Tag, hash: Option) -> io::Result<()> { - Ok(self.0.set_tag(name, hash).await?) - } - - async fn create_tag(&self, hash: HashAndFormat) -> io::Result { - Ok(self.0.create_tag(hash).await?) - } - - async fn delete(&self, hashes: Vec) -> io::Result<()> { - Ok(self.0.delete(hashes).await?) - } - - async fn gc_run(&self, config: super::GcConfig, protected_cb: G) - where - G: Fn() -> Gut, - Gut: Future> + Send, - { - tracing::info!("Starting GC task with interval {:?}", config.period); - let mut live = BTreeSet::new(); - 'outer: loop { - if let Err(cause) = self.0.gc_start().await { - tracing::debug!( - "unable to notify the db of GC start: {cause}. Shutting down GC loop." - ); - break; - } - // do delay before the two phases of GC - tokio::time::sleep(config.period).await; - tracing::debug!("Starting GC"); - live.clear(); - - let p = protected_cb().await; - live.extend(p); - - tracing::debug!("Starting GC mark phase"); - let live_ref = &mut live; - let mut stream = Gen::new(|co| async move { - if let Err(e) = super::gc_mark_task(self, live_ref, &co).await { - co.yield_(GcMarkEvent::Error(e)).await; - } - }); - while let Some(item) = stream.next().await { - match item { - GcMarkEvent::CustomDebug(text) => { - tracing::debug!("{}", text); - } - GcMarkEvent::CustomWarning(text, _) => { - tracing::warn!("{}", text); - } - GcMarkEvent::Error(err) => { - tracing::error!("Fatal error during GC mark {}", err); - continue 'outer; - } - } - } - drop(stream); - - tracing::debug!("Starting GC sweep phase"); - let live_ref = &live; - let mut stream = Gen::new(|co| async move { - if let Err(e) = gc_sweep_task(self, live_ref, &co).await { - co.yield_(GcSweepEvent::Error(e)).await; - } - }); - while let Some(item) = stream.next().await { - match item { - GcSweepEvent::CustomDebug(text) => { - tracing::debug!("{}", text); - } - GcSweepEvent::CustomWarning(text, _) => { - tracing::warn!("{}", text); - } - GcSweepEvent::Error(err) => { - tracing::error!("Fatal error during GC mark {}", err); - continue 'outer; - } - } - } - if let Some(ref cb) = config.done_callback { - cb(); - } - } - } - - fn temp_tag(&self, value: HashAndFormat) -> TempTag { - self.0.temp.temp_tag(value) - } - - async fn sync(&self) -> io::Result<()> { - Ok(self.0.sync().await?) - } - - async fn shutdown(&self) { - self.0.shutdown().await; - } -} - -pub(super) async fn gc_sweep_task<'a>( - store: &'a Store, - live: &BTreeSet, - co: &Co, -) -> anyhow::Result<()> { - let blobs = store.blobs().await?.chain(store.partial_blobs().await?); - let mut count = 0; - let mut batch = Vec::new(); - for hash in blobs { - let hash = hash?; - if !live.contains(&hash) { - batch.push(hash); - count += 1; - } - if batch.len() >= 100 { - store.0.gc_delete(batch.clone()).await?; - batch.clear(); - } - } - if !batch.is_empty() { - store.0.gc_delete(batch).await?; - } - co.yield_(GcSweepEvent::CustomDebug(format!( - "deleted {} blobs", - count - ))) - .await; - Ok(()) -} - -impl Actor { - fn new( - path: &Path, - options: Options, - temp: Arc>, - rt: tokio::runtime::Handle, - ) -> ActorResult<(Self, async_channel::Sender)> { - let db = match redb::Database::create(path) { - Ok(db) => db, - Err(DatabaseError::UpgradeRequired(1)) => { - migrate_redb_v1_v2::run(path).map_err(ActorError::Migration)? - } - Err(err) => return Err(err.into()), - }; - - let txn = db.begin_write()?; - // create tables and drop them just to create them. - let mut t = Default::default(); - let tables = Tables::new(&txn, &mut t)?; - drop(tables); - txn.commit()?; - // make the channel relatively large. there are some messages that don't - // require a response, it's fine if they pile up a bit. - let (tx, rx) = async_channel::bounded(1024); - let tx2 = tx.clone(); - let on_file_create: CreateCb = Arc::new(move |hash| { - // todo: make the callback allow async - tx2.send_blocking(ActorMessage::OnMemSizeExceeded { hash: *hash }) - .ok(); - Ok(()) - }); - let create_options = BaoFileConfig::new( - Arc::new(options.path.data_path.clone()), - 16 * 1024, - Some(on_file_create), - ); - Ok(( - Self { - db, - state: ActorState { - temp, - handles: BTreeMap::new(), - protected: BTreeSet::new(), - msgs_rx: rx, - options, - create_options: Arc::new(create_options), - rt, - }, - }, - tx, - )) - } - - async fn run_batched(mut self) -> ActorResult<()> { - let mut msgs = PeekableFlumeReceiver::new(self.state.msgs_rx.clone()); - while let Some(msg) = msgs.recv().await { - if let ActorMessage::Shutdown { tx } = msg { - // Make sure the database is dropped before we send the reply. - drop(self); - if let Some(tx) = tx { - tx.send(()).ok(); - } - break; - } - match msg.category() { - MessageCategory::TopLevel => { - self.state.handle_toplevel(&self.db, msg)?; - } - MessageCategory::ReadOnly => { - msgs.push_back(msg).expect("just recv'd"); - tracing::debug!("starting read transaction"); - let txn = self.db.begin_read()?; - let tables = ReadOnlyTables::new(&txn)?; - let count = self.state.options.batch.max_read_batch; - let timeout = tokio::time::sleep(self.state.options.batch.max_read_duration); - tokio::pin!(timeout); - for _ in 0..count { - tokio::select! { - msg = msgs.recv() => { - if let Some(msg) = msg { - if let Err(msg) = self.state.handle_readonly(&tables, msg)? { - msgs.push_back(msg).expect("just recv'd"); - break; - } - } else { - break; - } - } - _ = &mut timeout => { - tracing::debug!("read transaction timed out"); - break; - } - } - } - tracing::debug!("done with read transaction"); - } - MessageCategory::ReadWrite => { - msgs.push_back(msg).expect("just recv'd"); - tracing::debug!("starting write transaction"); - let txn = self.db.begin_write()?; - let mut delete_after_commit = Default::default(); - let mut tables = Tables::new(&txn, &mut delete_after_commit)?; - let count = self.state.options.batch.max_write_batch; - let timeout = tokio::time::sleep(self.state.options.batch.max_write_duration); - tokio::pin!(timeout); - for _ in 0..count { - tokio::select! { - msg = msgs.recv() => { - if let Some(msg) = msg { - if let Err(msg) = self.state.handle_readwrite(&mut tables, msg)? { - msgs.push_back(msg).expect("just recv'd"); - break; - } - } else { - break; - } - } - _ = &mut timeout => { - tracing::debug!("write transaction timed out"); - break; - } - } - } - drop(tables); - txn.commit()?; - delete_after_commit.apply_and_clear(&self.state.options.path); - tracing::debug!("write transaction committed"); - } - } - } - tracing::debug!("redb actor done"); - Ok(()) - } -} - -impl ActorState { - fn entry_status( - &mut self, - tables: &impl ReadableTables, - hash: Hash, - ) -> ActorResult { - let status = match tables.blobs().get(hash)? { - Some(guard) => match guard.value() { - EntryState::Complete { .. } => EntryStatus::Complete, - EntryState::Partial { .. } => EntryStatus::Partial, - }, - None => EntryStatus::NotFound, - }; - Ok(status) - } - - fn get( - &mut self, - tables: &impl ReadableTables, - hash: Hash, - ) -> ActorResult> { - if let Some(handle) = self.handles.get(&hash).and_then(|weak| weak.upgrade()) { - return Ok(Some(handle)); - } - let Some(entry) = tables.blobs().get(hash)? else { - return Ok(None); - }; - // todo: if complete, load inline data and/or outboard into memory if needed, - // and return a complete entry. - let entry = entry.value(); - let config = self.create_options.clone(); - let handle = match entry { - EntryState::Complete { - data_location, - outboard_location, - } => { - let data = load_data(tables, &self.options.path, data_location, &hash)?; - let outboard = load_outboard( - tables, - &self.options.path, - outboard_location, - data.size(), - &hash, - )?; - BaoFileHandle::new_complete(config, hash, data, outboard) - } - EntryState::Partial { .. } => BaoFileHandle::incomplete_file(config, hash)?, - }; - self.handles.insert(hash, handle.downgrade()); - Ok(Some(handle)) - } - - fn export( - &mut self, - tables: &mut Tables, - cmd: Export, - tx: oneshot::Sender>, - ) -> ActorResult<()> { - let Export { - temp_tag, - target, - mode, - progress, - } = cmd; - let guard = tables - .blobs - .get(temp_tag.hash())? - .ok_or_else(|| ActorError::Inconsistent("entry not found".to_owned()))?; - let entry = guard.value(); - match entry { - EntryState::Complete { - data_location, - outboard_location, - } => match data_location { - DataLocation::Inline(()) => { - // ignore export mode, just copy. For inline data we can not reference anyway. - let data = tables.inline_data.get(temp_tag.hash())?.ok_or_else(|| { - ActorError::Inconsistent("inline data not found".to_owned()) - })?; - tracing::trace!("exporting inline data to {}", target.display()); - tx.send(std::fs::write(&target, data.value()).map_err(|e| e.into())) - .ok(); - } - DataLocation::Owned(size) => { - let path = self.options.path.owned_data_path(temp_tag.hash()); - match mode { - ExportMode::Copy => { - // copy in an external thread - self.rt.spawn_blocking(move || { - tx.send(export_file_copy(temp_tag, path, size, target, progress)) - .ok(); - }); - } - ExportMode::TryReference => match std::fs::rename(&path, &target) { - Ok(()) => { - let entry = EntryState::Complete { - data_location: DataLocation::External(vec![target], size), - outboard_location, - }; - drop(guard); - tables.blobs.insert(temp_tag.hash(), entry)?; - drop(temp_tag); - tx.send(Ok(())).ok(); - } - Err(e) => { - const ERR_CROSS: i32 = 18; - if e.raw_os_error() == Some(ERR_CROSS) { - // Cross device renaming failed, copy instead - match std::fs::copy(&path, &target) { - Ok(_) => { - let entry = EntryState::Complete { - data_location: DataLocation::External( - vec![target], - size, - ), - outboard_location, - }; - - drop(guard); - tables.blobs.insert(temp_tag.hash(), entry)?; - tables - .delete_after_commit - .insert(*temp_tag.hash(), [BaoFilePart::Data]); - drop(temp_tag); - - tx.send(Ok(())).ok(); - } - Err(e) => { - drop(temp_tag); - tx.send(Err(e.into())).ok(); - } - } - } else { - drop(temp_tag); - tx.send(Err(e.into())).ok(); - } - } - }, - } - } - DataLocation::External(paths, size) => { - let path = paths - .first() - .ok_or_else(|| { - ActorError::Inconsistent("external path missing".to_owned()) - })? - .to_owned(); - // we can not reference external files, so we just copy them. But this does not have to happen in the actor. - if path == target { - // export to the same path, nothing to do - tx.send(Ok(())).ok(); - } else { - // copy in an external thread - self.rt.spawn_blocking(move || { - tx.send(export_file_copy(temp_tag, path, size, target, progress)) - .ok(); - }); - } - } - }, - EntryState::Partial { .. } => { - return Err(io::Error::new(io::ErrorKind::Unsupported, "partial entry").into()); - } - } - Ok(()) - } - - fn import(&mut self, tables: &mut Tables, cmd: Import) -> ActorResult<(TempTag, u64)> { - let Import { - content_id, - source: file, - outboard, - data_size, - } = cmd; - let outboard_size = outboard.as_ref().map(|x| x.len() as u64).unwrap_or(0); - let inline_data = data_size <= self.options.inline.max_data_inlined; - let inline_outboard = - outboard_size <= self.options.inline.max_outboard_inlined && outboard_size != 0; - // from here on, everything related to the hash is protected by the temp tag - let tag = self.temp.temp_tag(content_id); - let hash = *tag.hash(); - self.protected.insert(hash); - // move the data file into place, or create a reference to it - let data_location = match file { - ImportSource::External(external_path) => { - tracing::debug!("stored external reference {}", external_path.display()); - if inline_data { - tracing::debug!( - "reading external data to inline it: {}", - external_path.display() - ); - let data = Bytes::from(std::fs::read(&external_path)?); - DataLocation::Inline(data) - } else { - DataLocation::External(vec![external_path], data_size) - } - } - ImportSource::TempFile(temp_data_path) => { - if inline_data { - tracing::debug!( - "reading and deleting temp file to inline it: {}", - temp_data_path.display() - ); - let data = Bytes::from(read_and_remove(&temp_data_path)?); - DataLocation::Inline(data) - } else { - let data_path = self.options.path.owned_data_path(&hash); - std::fs::rename(&temp_data_path, &data_path)?; - tracing::debug!("created file {}", data_path.display()); - DataLocation::Owned(data_size) - } - } - ImportSource::Memory(data) => { - if inline_data { - DataLocation::Inline(data) - } else { - let data_path = self.options.path.owned_data_path(&hash); - overwrite_and_sync(&data_path, &data)?; - tracing::debug!("created file {}", data_path.display()); - DataLocation::Owned(data_size) - } - } - }; - let outboard_location = if let Some(outboard) = outboard { - if inline_outboard { - OutboardLocation::Inline(Bytes::from(outboard)) - } else { - let outboard_path = self.options.path.owned_outboard_path(&hash); - // todo: this blocks the actor when writing a large outboard - overwrite_and_sync(&outboard_path, &outboard)?; - OutboardLocation::Owned - } - } else { - OutboardLocation::NotNeeded - }; - if let DataLocation::Inline(data) = &data_location { - tables.inline_data.insert(hash, data.as_ref())?; - } - if let OutboardLocation::Inline(outboard) = &outboard_location { - tables.inline_outboard.insert(hash, outboard.as_ref())?; - } - if let DataLocation::Owned(_) = &data_location { - tables.delete_after_commit.remove(hash, [BaoFilePart::Data]); - } - if let OutboardLocation::Owned = &outboard_location { - tables - .delete_after_commit - .remove(hash, [BaoFilePart::Outboard]); - } - let entry = tables.blobs.get(hash)?; - let entry = entry.map(|x| x.value()).unwrap_or_default(); - let data_location = data_location.discard_inline_data(); - let outboard_location = outboard_location.discard_extra_data(); - let entry = entry.union(EntryState::Complete { - data_location, - outboard_location, - })?; - tables.blobs.insert(hash, entry)?; - Ok((tag, data_size)) - } - - fn get_or_create( - &mut self, - tables: &impl ReadableTables, - hash: Hash, - ) -> ActorResult { - self.protected.insert(hash); - if let Some(handle) = self.handles.get(&hash).and_then(|x| x.upgrade()) { - return Ok(handle); - } - let entry = tables.blobs().get(hash)?; - let handle = if let Some(entry) = entry { - let entry = entry.value(); - match entry { - EntryState::Complete { - data_location, - outboard_location, - .. - } => { - let data = load_data(tables, &self.options.path, data_location, &hash)?; - let outboard = load_outboard( - tables, - &self.options.path, - outboard_location, - data.size(), - &hash, - )?; - tracing::debug!("creating complete entry for {}", hash.to_hex()); - BaoFileHandle::new_complete(self.create_options.clone(), hash, data, outboard) - } - EntryState::Partial { .. } => { - tracing::debug!("creating partial entry for {}", hash.to_hex()); - BaoFileHandle::incomplete_file(self.create_options.clone(), hash)? - } - } - } else { - BaoFileHandle::incomplete_mem(self.create_options.clone(), hash) - }; - self.handles.insert(hash, handle.downgrade()); - Ok(handle) - } - - /// Read the entire blobs table. Callers can then sift through the results to find what they need - fn blobs( - &mut self, - tables: &impl ReadableTables, - filter: FilterPredicate, - ) -> ActorResult>> { - let mut res = Vec::new(); - let mut index = 0u64; - #[allow(clippy::explicit_counter_loop)] - for item in tables.blobs().iter()? { - match item { - Ok((k, v)) => { - if let Some(item) = filter(index, k, v) { - res.push(Ok(item)); - } - } - Err(e) => { - res.push(Err(e)); - } - } - index += 1; - } - Ok(res) - } - - /// Read the entire tags table. Callers can then sift through the results to find what they need - fn tags( - &mut self, - tables: &impl ReadableTables, - filter: FilterPredicate, - ) -> ActorResult>> { - let mut res = Vec::new(); - let mut index = 0u64; - #[allow(clippy::explicit_counter_loop)] - for item in tables.tags().iter()? { - match item { - Ok((k, v)) => { - if let Some(item) = filter(index, k, v) { - res.push(Ok(item)); - } - } - Err(e) => { - res.push(Err(e)); - } - } - index += 1; - } - Ok(res) - } - - fn create_tag(&mut self, tables: &mut Tables, content: HashAndFormat) -> ActorResult { - let tag = { - let tag = Tag::auto(SystemTime::now(), |x| { - matches!(tables.tags.get(Tag(Bytes::copy_from_slice(x))), Ok(Some(_))) - }); - tables.tags.insert(tag.clone(), content)?; - tag - }; - Ok(tag) - } - - fn set_tag( - &self, - tables: &mut Tables, - tag: Tag, - value: Option, - ) -> ActorResult<()> { - match value { - Some(value) => { - tables.tags.insert(tag, value)?; - } - None => { - tables.tags.remove(tag)?; - } - } - Ok(()) - } - - fn on_mem_size_exceeded(&mut self, tables: &mut Tables, hash: Hash) -> ActorResult<()> { - let entry = tables - .blobs - .get(hash)? - .map(|x| x.value()) - .unwrap_or_default(); - let entry = entry.union(EntryState::Partial { size: None })?; - tables.blobs.insert(hash, entry)?; - // protect all three parts of the entry - tables.delete_after_commit.remove( - hash, - [BaoFilePart::Data, BaoFilePart::Outboard, BaoFilePart::Sizes], - ); - Ok(()) - } - - fn update_inline_options( - &mut self, - db: &redb::Database, - options: InlineOptions, - reapply: bool, - ) -> ActorResult<()> { - self.options.inline = options; - if reapply { - let mut delete_after_commit = Default::default(); - let tx = db.begin_write()?; - { - let mut tables = Tables::new(&tx, &mut delete_after_commit)?; - let hashes = tables - .blobs - .iter()? - .map(|x| x.map(|(k, _)| k.value())) - .collect::, _>>()?; - for hash in hashes { - let guard = tables - .blobs - .get(hash)? - .ok_or_else(|| ActorError::Inconsistent("hash not found".to_owned()))?; - let entry = guard.value(); - if let EntryState::Complete { - data_location, - outboard_location, - } = entry - { - let (data_location, data_size, data_location_changed) = match data_location - { - DataLocation::Owned(size) => { - // inline - if size <= self.options.inline.max_data_inlined { - let path = self.options.path.owned_data_path(&hash); - let data = std::fs::read(&path)?; - tables.delete_after_commit.insert(hash, [BaoFilePart::Data]); - tables.inline_data.insert(hash, data.as_slice())?; - (DataLocation::Inline(()), size, true) - } else { - (DataLocation::Owned(size), size, false) - } - } - DataLocation::Inline(()) => { - let guard = tables.inline_data.get(hash)?.ok_or_else(|| { - ActorError::Inconsistent("inline data missing".to_owned()) - })?; - let data = guard.value(); - let size = data.len() as u64; - if size > self.options.inline.max_data_inlined { - let path = self.options.path.owned_data_path(&hash); - std::fs::write(&path, data)?; - drop(guard); - tables.inline_data.remove(hash)?; - (DataLocation::Owned(size), size, true) - } else { - (DataLocation::Inline(()), size, false) - } - } - DataLocation::External(paths, size) => { - (DataLocation::External(paths, size), size, false) - } - }; - let outboard_size = raw_outboard_size(data_size); - let (outboard_location, outboard_location_changed) = match outboard_location - { - OutboardLocation::Owned - if outboard_size <= self.options.inline.max_outboard_inlined => - { - let path = self.options.path.owned_outboard_path(&hash); - let outboard = std::fs::read(&path)?; - tables - .delete_after_commit - .insert(hash, [BaoFilePart::Outboard]); - tables.inline_outboard.insert(hash, outboard.as_slice())?; - (OutboardLocation::Inline(()), true) - } - OutboardLocation::Inline(()) - if outboard_size > self.options.inline.max_outboard_inlined => - { - let guard = tables.inline_outboard.get(hash)?.ok_or_else(|| { - ActorError::Inconsistent("inline outboard missing".to_owned()) - })?; - let outboard = guard.value(); - let path = self.options.path.owned_outboard_path(&hash); - std::fs::write(&path, outboard)?; - drop(guard); - tables.inline_outboard.remove(hash)?; - (OutboardLocation::Owned, true) - } - x => (x, false), - }; - drop(guard); - if data_location_changed || outboard_location_changed { - tables.blobs.insert( - hash, - EntryState::Complete { - data_location, - outboard_location, - }, - )?; - } - } - } - } - tx.commit()?; - delete_after_commit.apply_and_clear(&self.options.path); - } - Ok(()) - } - - fn delete(&mut self, tables: &mut Tables, hashes: Vec, force: bool) -> ActorResult<()> { - for hash in hashes { - if self.temp.as_ref().read().unwrap().contains(&hash) { - continue; - } - if !force && self.protected.contains(&hash) { - tracing::debug!("protected hash, continuing {}", &hash.to_hex()[..8]); - continue; - } - - tracing::debug!("deleting {}", &hash.to_hex()[..8]); - - self.handles.remove(&hash); - if let Some(entry) = tables.blobs.remove(hash)? { - match entry.value() { - EntryState::Complete { - data_location, - outboard_location, - } => { - match data_location { - DataLocation::Inline(_) => { - tables.inline_data.remove(hash)?; - } - DataLocation::Owned(_) => { - // mark the data for deletion - tables.delete_after_commit.insert(hash, [BaoFilePart::Data]); - } - DataLocation::External(_, _) => {} - } - match outboard_location { - OutboardLocation::Inline(_) => { - tables.inline_outboard.remove(hash)?; - } - OutboardLocation::Owned => { - // mark the outboard for deletion - tables - .delete_after_commit - .insert(hash, [BaoFilePart::Outboard]); - } - OutboardLocation::NotNeeded => {} - } - } - EntryState::Partial { .. } => { - // mark all parts for deletion - tables.delete_after_commit.insert( - hash, - [BaoFilePart::Outboard, BaoFilePart::Data, BaoFilePart::Sizes], - ); - } - } - } - } - Ok(()) - } - - fn on_complete(&mut self, tables: &mut Tables, entry: BaoFileHandle) -> ActorResult<()> { - let hash = entry.hash(); - let mut info = None; - tracing::trace!("on_complete({})", hash.to_hex()); - entry.transform(|state| { - tracing::trace!("on_complete transform {:?}", state); - let entry = match complete_storage( - state, - &hash, - &self.options.path, - &self.options.inline, - tables.delete_after_commit, - )? { - Ok(entry) => { - // store the info so we can insert it into the db later - info = Some(( - entry.data_size(), - entry.data.mem().cloned(), - entry.outboard_size(), - entry.outboard.mem().cloned(), - )); - entry - } - Err(entry) => { - // the entry was already complete, nothing to do - entry - } - }; - Ok(BaoFileStorage::Complete(entry)) - })?; - if let Some((data_size, data, outboard_size, outboard)) = info { - let data_location = if data.is_some() { - DataLocation::Inline(()) - } else { - DataLocation::Owned(data_size) - }; - let outboard_location = if outboard_size == 0 { - OutboardLocation::NotNeeded - } else if outboard.is_some() { - OutboardLocation::Inline(()) - } else { - OutboardLocation::Owned - }; - { - tracing::debug!( - "inserting complete entry for {}, {} bytes", - hash.to_hex(), - data_size, - ); - let entry = tables - .blobs() - .get(hash)? - .map(|x| x.value()) - .unwrap_or_default(); - let entry = entry.union(EntryState::Complete { - data_location, - outboard_location, - })?; - tables.blobs.insert(hash, entry)?; - if let Some(data) = data { - tables.inline_data.insert(hash, data.as_ref())?; - } - if let Some(outboard) = outboard { - tables.inline_outboard.insert(hash, outboard.as_ref())?; - } - } - } - Ok(()) - } - - fn handle_toplevel(&mut self, db: &redb::Database, msg: ActorMessage) -> ActorResult<()> { - match msg { - ActorMessage::UpdateInlineOptions { - inline_options, - reapply, - tx, - } => { - let res = self.update_inline_options(db, inline_options, reapply); - tx.send(res?).ok(); - } - ActorMessage::Fsck { - repair, - progress, - tx, - } => { - let res = self.consistency_check(db, repair, progress); - tx.send(res).ok(); - } - ActorMessage::Sync { tx } => { - tx.send(()).ok(); - } - x => { - return Err(ActorError::Inconsistent(format!( - "unexpected message for handle_toplevel: {:?}", - x - ))) - } - } - Ok(()) - } - - fn handle_readonly( - &mut self, - tables: &impl ReadableTables, - msg: ActorMessage, - ) -> ActorResult> { - match msg { - ActorMessage::Get { hash, tx } => { - let res = self.get(tables, hash); - tx.send(res).ok(); - } - ActorMessage::GetOrCreate { hash, tx } => { - let res = self.get_or_create(tables, hash); - tx.send(res).ok(); - } - ActorMessage::EntryStatus { hash, tx } => { - let res = self.entry_status(tables, hash); - tx.send(res).ok(); - } - ActorMessage::Blobs { filter, tx } => { - let res = self.blobs(tables, filter); - tx.send(res).ok(); - } - ActorMessage::Tags { filter, tx } => { - let res = self.tags(tables, filter); - tx.send(res).ok(); - } - ActorMessage::GcStart { tx } => { - self.protected.clear(); - self.handles.retain(|_, weak| weak.is_live()); - tx.send(()).ok(); - } - ActorMessage::Dump => { - dump(tables).ok(); - } - #[cfg(test)] - ActorMessage::EntryState { hash, tx } => { - tx.send(self.entry_state(tables, hash)).ok(); - } - ActorMessage::GetFullEntryState { hash, tx } => { - let res = self.get_full_entry_state(tables, hash); - tx.send(res).ok(); - } - x => return Ok(Err(x)), - } - Ok(Ok(())) - } - - fn handle_readwrite( - &mut self, - tables: &mut Tables, - msg: ActorMessage, - ) -> ActorResult> { - match msg { - ActorMessage::Import { cmd, tx } => { - let res = self.import(tables, cmd); - tx.send(res).ok(); - } - ActorMessage::SetTag { tag, value, tx } => { - let res = self.set_tag(tables, tag, value); - tx.send(res).ok(); - } - ActorMessage::CreateTag { hash, tx } => { - let res = self.create_tag(tables, hash); - tx.send(res).ok(); - } - ActorMessage::Delete { hashes, tx } => { - let res = self.delete(tables, hashes, true); - tx.send(res).ok(); - } - ActorMessage::GcDelete { hashes, tx } => { - let res = self.delete(tables, hashes, false); - tx.send(res).ok(); - } - ActorMessage::OnComplete { handle } => { - let res = self.on_complete(tables, handle); - res.ok(); - } - ActorMessage::Export { cmd, tx } => { - self.export(tables, cmd, tx)?; - } - ActorMessage::OnMemSizeExceeded { hash } => { - let res = self.on_mem_size_exceeded(tables, hash); - res.ok(); - } - ActorMessage::Dump => { - let res = dump(tables); - res.ok(); - } - ActorMessage::SetFullEntryState { hash, entry, tx } => { - let res = self.set_full_entry_state(tables, hash, entry); - tx.send(res).ok(); - } - msg => { - // try to handle it as readonly - if let Err(msg) = self.handle_readonly(tables, msg)? { - return Ok(Err(msg)); - } - } - } - Ok(Ok(())) - } -} - -/// Export a file by copying out its content to a new location -fn export_file_copy( - temp_tag: TempTag, - path: PathBuf, - size: u64, - target: PathBuf, - progress: ExportProgressCb, -) -> ActorResult<()> { - progress(0)?; - // todo: fine grained copy progress - reflink_copy::reflink_or_copy(path, target)?; - progress(size)?; - drop(temp_tag); - Ok(()) -} - -fn dump(tables: &impl ReadableTables) -> ActorResult<()> { - for e in tables.blobs().iter()? { - let (k, v) = e?; - let k = k.value(); - let v = v.value(); - println!("blobs: {} -> {:?}", k.to_hex(), v); - } - for e in tables.tags().iter()? { - let (k, v) = e?; - let k = k.value(); - let v = v.value(); - println!("tags: {} -> {:?}", k, v); - } - for e in tables.inline_data().iter()? { - let (k, v) = e?; - let k = k.value(); - let v = v.value(); - println!("inline_data: {} -> {:?}", k.to_hex(), v.len()); - } - for e in tables.inline_outboard().iter()? { - let (k, v) = e?; - let k = k.value(); - let v = v.value(); - println!("inline_outboard: {} -> {:?}", k.to_hex(), v.len()); - } - Ok(()) -} - -fn load_data( - tables: &impl ReadableTables, - options: &PathOptions, - location: DataLocation<(), u64>, - hash: &Hash, -) -> ActorResult> { - Ok(match location { - DataLocation::Inline(()) => { - let Some(data) = tables.inline_data().get(hash)? else { - return Err(ActorError::Inconsistent(format!( - "inconsistent database state: {} should have inline data but does not", - hash.to_hex() - ))); - }; - MemOrFile::Mem(Bytes::copy_from_slice(data.value())) - } - DataLocation::Owned(data_size) => { - let path = options.owned_data_path(hash); - let Ok(file) = std::fs::File::open(&path) else { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("file not found: {}", path.display()), - ) - .into()); - }; - MemOrFile::File((file, data_size)) - } - DataLocation::External(paths, data_size) => { - if paths.is_empty() { - return Err(ActorError::Inconsistent( - "external data location must not be empty".into(), - )); - } - let path = &paths[0]; - let Ok(file) = std::fs::File::open(path) else { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("external file not found: {}", path.display()), - ) - .into()); - }; - MemOrFile::File((file, data_size)) - } - }) -} - -fn load_outboard( - tables: &impl ReadableTables, - options: &PathOptions, - location: OutboardLocation, - size: u64, - hash: &Hash, -) -> ActorResult> { - Ok(match location { - OutboardLocation::NotNeeded => MemOrFile::Mem(Bytes::new()), - OutboardLocation::Inline(_) => { - let Some(outboard) = tables.inline_outboard().get(hash)? else { - return Err(ActorError::Inconsistent(format!( - "inconsistent database state: {} should have inline outboard but does not", - hash.to_hex() - ))); - }; - MemOrFile::Mem(Bytes::copy_from_slice(outboard.value())) - } - OutboardLocation::Owned => { - let outboard_size = raw_outboard_size(size); - let path = options.owned_outboard_path(hash); - let Ok(file) = std::fs::File::open(&path) else { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("file not found: {} size={}", path.display(), outboard_size), - ) - .into()); - }; - MemOrFile::File((file, outboard_size)) - } - }) -} - -/// Take a possibly incomplete storage and turn it into complete -fn complete_storage( - storage: BaoFileStorage, - hash: &Hash, - path_options: &PathOptions, - inline_options: &InlineOptions, - delete_after_commit: &mut DeleteSet, -) -> ActorResult> { - let (data, outboard, _sizes) = match storage { - BaoFileStorage::Complete(c) => return Ok(Err(c)), - BaoFileStorage::IncompleteMem(storage) => { - let (data, outboard, sizes) = storage.into_parts(); - ( - MemOrFile::Mem(Bytes::from(data.into_parts().0)), - MemOrFile::Mem(Bytes::from(outboard.into_parts().0)), - MemOrFile::Mem(Bytes::from(sizes.to_vec())), - ) - } - BaoFileStorage::IncompleteFile(storage) => { - let (data, outboard, sizes) = storage.into_parts(); - ( - MemOrFile::File(data), - MemOrFile::File(outboard), - MemOrFile::File(sizes), - ) - } - }; - let data_size = data.size()?.unwrap(); - let outboard_size = outboard.size()?.unwrap(); - // todo: perform more sanity checks if in debug mode - debug_assert!(raw_outboard_size(data_size) == outboard_size); - // inline data if needed, or write to file if needed - let data = if data_size <= inline_options.max_data_inlined { - match data { - MemOrFile::File(data) => { - let mut buf = vec![0; data_size as usize]; - data.read_at(0, &mut buf)?; - // mark data for deletion after commit - delete_after_commit.insert(*hash, [BaoFilePart::Data]); - MemOrFile::Mem(Bytes::from(buf)) - } - MemOrFile::Mem(data) => MemOrFile::Mem(data), - } - } else { - // protect the data from previous deletions - delete_after_commit.remove(*hash, [BaoFilePart::Data]); - match data { - MemOrFile::Mem(data) => { - let path = path_options.owned_data_path(hash); - let file = overwrite_and_sync(&path, &data)?; - MemOrFile::File((file, data_size)) - } - MemOrFile::File(data) => MemOrFile::File((data, data_size)), - } - }; - // inline outboard if needed, or write to file if needed - let outboard = if outboard_size == 0 { - Default::default() - } else if outboard_size <= inline_options.max_outboard_inlined { - match outboard { - MemOrFile::File(outboard) => { - let mut buf = vec![0; outboard_size as usize]; - outboard.read_at(0, &mut buf)?; - drop(outboard); - // mark outboard for deletion after commit - delete_after_commit.insert(*hash, [BaoFilePart::Outboard]); - MemOrFile::Mem(Bytes::from(buf)) - } - MemOrFile::Mem(outboard) => MemOrFile::Mem(outboard), - } - } else { - // protect the outboard from previous deletions - delete_after_commit.remove(*hash, [BaoFilePart::Outboard]); - match outboard { - MemOrFile::Mem(outboard) => { - let path = path_options.owned_outboard_path(hash); - let file = overwrite_and_sync(&path, &outboard)?; - MemOrFile::File((file, outboard_size)) - } - MemOrFile::File(outboard) => MemOrFile::File((outboard, outboard_size)), - } - }; - // mark sizes for deletion after commit in any case - a complete entry - // does not need sizes. - delete_after_commit.insert(*hash, [BaoFilePart::Sizes]); - Ok(Ok(CompleteStorage { data, outboard })) -} diff --git a/iroh-blobs/src/store/fs/migrate_redb_v1_v2.rs b/iroh-blobs/src/store/fs/migrate_redb_v1_v2.rs deleted file mode 100644 index d9ff3b07af8..00000000000 --- a/iroh-blobs/src/store/fs/migrate_redb_v1_v2.rs +++ /dev/null @@ -1,323 +0,0 @@ -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use redb_v1::ReadableTable; -use tempfile::NamedTempFile; -use tracing::info; - -pub fn run(source: impl AsRef) -> Result { - let source = source.as_ref(); - let dir = source.parent().expect("database is not in root"); - // create the new database in a tempfile in the same directory as the old db - let target = NamedTempFile::with_prefix_in("blobs.db.migrate", dir)?; - let target = target.into_temp_path(); - info!("migrate {} to {}", source.display(), target.display()); - let old_db = redb_v1::Database::open(source)?; - let new_db = redb::Database::create(&target)?; - - let rtx = old_db.begin_read()?; - let wtx = new_db.begin_write()?; - - { - let old_blobs = rtx.open_table(old::BLOBS_TABLE)?; - let mut new_blobs = wtx.open_table(new::BLOBS_TABLE)?; - let len = old_blobs.len()?; - info!("migrate blobs table ({len} rows)"); - for (i, entry) in old_blobs.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_blobs.insert(key, value)?; - } - info!("migrate blobs table done"); - let old_tags = rtx.open_table(old::TAGS_TABLE)?; - let mut new_tags = wtx.open_table(new::TAGS_TABLE)?; - let len = old_tags.len()?; - info!("migrate tags table ({len} rows)"); - for (i, entry) in old_tags.iter()?.enumerate() { - let (key, value) = entry?; - let key = key.value(); - let value: crate::HashAndFormat = value.value().into(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_tags.insert(key, value)?; - } - info!("migrate tags table done"); - let old_inline_data = rtx.open_table(old::INLINE_DATA_TABLE)?; - let mut new_inline_data = wtx.open_table(new::INLINE_DATA_TABLE)?; - let len = old_inline_data.len()?; - info!("migrate inline data table ({len} rows)"); - for (i, entry) in old_inline_data.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_inline_data.insert(key, value)?; - } - info!("migrate inline data table done"); - let old_inline_outboard = rtx.open_table(old::INLINE_OUTBOARD_TABLE)?; - let mut new_inline_outboard = wtx.open_table(new::INLINE_OUTBOARD_TABLE)?; - let len = old_inline_outboard.len()?; - info!("migrate inline outboard table ({len} rows)"); - for (i, entry) in old_inline_outboard.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_inline_outboard.insert(key, value)?; - } - info!("migrate inline outboard table done"); - } - - wtx.commit()?; - drop(rtx); - drop(old_db); - drop(new_db); - - let backup_path: PathBuf = { - let mut p = source.to_owned().into_os_string(); - p.push(".backup-redb-v1"); - p.into() - }; - info!("rename {} to {}", source.display(), backup_path.display()); - std::fs::rename(source, &backup_path)?; - info!("rename {} to {}", target.display(), source.display()); - target.persist_noclobber(source)?; - info!("opening migrated database from {}", source.display()); - let db = redb::Database::open(source)?; - Ok(db) -} - -mod new { - pub(super) use super::super::tables::*; -} - -mod old { - use bytes::Bytes; - use iroh_base::hash::BlobFormat; - use postcard::experimental::max_size::MaxSize; - use redb_v1::{RedbKey, RedbValue, TableDefinition, TypeName}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use smallvec::SmallVec; - - use super::super::EntryState; - use crate::util::Tag; - - pub const BLOBS_TABLE: TableDefinition = TableDefinition::new("blobs-0"); - - pub const TAGS_TABLE: TableDefinition = TableDefinition::new("tags-0"); - - pub const INLINE_DATA_TABLE: TableDefinition = - TableDefinition::new("inline-data-0"); - - pub const INLINE_OUTBOARD_TABLE: TableDefinition = - TableDefinition::new("inline-outboard-0"); - - impl redb_v1::RedbValue for EntryState { - type SelfType<'a> = EntryState; - - type AsBytes<'a> = SmallVec<[u8; 128]>; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - postcard::from_bytes(data).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - postcard::to_extend(value, SmallVec::new()).unwrap() - } - - fn type_name() -> TypeName { - TypeName::new("EntryState") - } - } - - impl RedbValue for HashAndFormat { - type SelfType<'a> = Self; - - type AsBytes<'a> = [u8; Self::POSTCARD_MAX_SIZE]; - - fn fixed_width() -> Option { - Some(Self::POSTCARD_MAX_SIZE) - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - let t: &'a [u8; Self::POSTCARD_MAX_SIZE] = data.try_into().unwrap(); - postcard::from_bytes(t.as_slice()).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - let mut res = [0u8; 33]; - postcard::to_slice(&value, &mut res).unwrap(); - res - } - - fn type_name() -> TypeName { - TypeName::new("iroh_base::HashAndFormat") - } - } - - impl RedbValue for Tag { - type SelfType<'a> = Self; - - type AsBytes<'a> = bytes::Bytes; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - Self(Bytes::copy_from_slice(data)) - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - value.0.clone() - } - - fn type_name() -> TypeName { - TypeName::new("Tag") - } - } - - impl RedbKey for Tag { - fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { - data1.cmp(data2) - } - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] - pub struct Hash([u8; 32]); - - impl Serialize for Hash { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.0.serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for Hash { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let data: [u8; 32] = Deserialize::deserialize(deserializer)?; - Ok(Self(data)) - } - } - - impl MaxSize for Hash { - const POSTCARD_MAX_SIZE: usize = 32; - } - - impl From for crate::Hash { - fn from(value: Hash) -> Self { - value.0.into() - } - } - - impl RedbValue for Hash { - type SelfType<'a> = Self; - - type AsBytes<'a> = &'a [u8; 32]; - - fn fixed_width() -> Option { - Some(32) - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - let contents: &'a [u8; 32] = data.try_into().unwrap(); - Hash(*contents) - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - &value.0 - } - - fn type_name() -> TypeName { - TypeName::new("iroh_base::Hash") - } - } - - impl RedbKey for Hash { - fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { - data1.cmp(data2) - } - } - - /// A hash and format pair - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, MaxSize)] - pub struct HashAndFormat { - /// The hash - pub hash: Hash, - /// The format - pub format: BlobFormat, - } - - impl From for crate::HashAndFormat { - fn from(value: HashAndFormat) -> Self { - crate::HashAndFormat { - hash: value.hash.into(), - format: value.format, - } - } - } - impl Serialize for HashAndFormat { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - (self.hash, self.format).serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for HashAndFormat { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let (hash, format) = <(Hash, BlobFormat)>::deserialize(deserializer)?; - Ok(Self { hash, format }) - } - } -} diff --git a/iroh-blobs/src/store/fs/tables.rs b/iroh-blobs/src/store/fs/tables.rs deleted file mode 100644 index afbf8b66f8a..00000000000 --- a/iroh-blobs/src/store/fs/tables.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Table definitions and accessors for the redb database. -use std::collections::BTreeSet; - -use iroh_base::hash::{Hash, HashAndFormat}; -use redb::{ReadableTable, TableDefinition, TableError}; - -use super::{EntryState, PathOptions}; -use crate::util::Tag; - -pub(super) const BLOBS_TABLE: TableDefinition = TableDefinition::new("blobs-0"); - -pub(super) const TAGS_TABLE: TableDefinition = TableDefinition::new("tags-0"); - -pub(super) const INLINE_DATA_TABLE: TableDefinition = - TableDefinition::new("inline-data-0"); - -pub(super) const INLINE_OUTBOARD_TABLE: TableDefinition = - TableDefinition::new("inline-outboard-0"); - -/// A trait similar to [`redb::ReadableTable`] but for all tables that make up -/// the blob store. This can be used in places where either a readonly or -/// mutable table is needed. -pub(super) trait ReadableTables { - fn blobs(&self) -> &impl ReadableTable; - fn tags(&self) -> &impl ReadableTable; - fn inline_data(&self) -> &impl ReadableTable; - fn inline_outboard(&self) -> &impl ReadableTable; -} - -/// A struct similar to [`redb::Table`] but for all tables that make up the -/// blob store. -pub(super) struct Tables<'a> { - pub blobs: redb::Table<'a, Hash, EntryState>, - pub tags: redb::Table<'a, Tag, HashAndFormat>, - pub inline_data: redb::Table<'a, Hash, &'static [u8]>, - pub inline_outboard: redb::Table<'a, Hash, &'static [u8]>, - pub delete_after_commit: &'a mut DeleteSet, -} - -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -pub(super) enum BaoFilePart { - Outboard, - Data, - Sizes, -} - -impl<'txn> Tables<'txn> { - pub fn new( - tx: &'txn redb::WriteTransaction, - delete_after_commit: &'txn mut DeleteSet, - ) -> std::result::Result { - Ok(Self { - blobs: tx.open_table(BLOBS_TABLE)?, - tags: tx.open_table(TAGS_TABLE)?, - inline_data: tx.open_table(INLINE_DATA_TABLE)?, - inline_outboard: tx.open_table(INLINE_OUTBOARD_TABLE)?, - delete_after_commit, - }) - } -} - -impl ReadableTables for Tables<'_> { - fn blobs(&self) -> &impl ReadableTable { - &self.blobs - } - fn tags(&self) -> &impl ReadableTable { - &self.tags - } - fn inline_data(&self) -> &impl ReadableTable { - &self.inline_data - } - fn inline_outboard(&self) -> &impl ReadableTable { - &self.inline_outboard - } -} - -/// A struct similar to [`redb::ReadOnlyTable`] but for all tables that make up -/// the blob store. -pub(super) struct ReadOnlyTables { - pub blobs: redb::ReadOnlyTable, - pub tags: redb::ReadOnlyTable, - pub inline_data: redb::ReadOnlyTable, - pub inline_outboard: redb::ReadOnlyTable, -} - -impl<'txn> ReadOnlyTables { - pub fn new(tx: &'txn redb::ReadTransaction) -> std::result::Result { - Ok(Self { - blobs: tx.open_table(BLOBS_TABLE)?, - tags: tx.open_table(TAGS_TABLE)?, - inline_data: tx.open_table(INLINE_DATA_TABLE)?, - inline_outboard: tx.open_table(INLINE_OUTBOARD_TABLE)?, - }) - } -} - -impl ReadableTables for ReadOnlyTables { - fn blobs(&self) -> &impl ReadableTable { - &self.blobs - } - fn tags(&self) -> &impl ReadableTable { - &self.tags - } - fn inline_data(&self) -> &impl ReadableTable { - &self.inline_data - } - fn inline_outboard(&self) -> &impl ReadableTable { - &self.inline_outboard - } -} - -/// Helper to keep track of files to delete after a transaction is committed. -#[derive(Debug, Default)] -pub(super) struct DeleteSet(BTreeSet<(Hash, BaoFilePart)>); - -impl DeleteSet { - /// Mark a file as to be deleted after the transaction is committed. - pub fn insert(&mut self, hash: Hash, parts: impl IntoIterator) { - for part in parts { - self.0.insert((hash, part)); - } - } - - /// Mark a file as to be kept after the transaction is committed. - /// - /// This will cancel any previous delete for the same file in the same transaction. - pub fn remove(&mut self, hash: Hash, parts: impl IntoIterator) { - for part in parts { - self.0.remove(&(hash, part)); - } - } - - /// Get the inner set of files to delete. - pub fn into_inner(self) -> BTreeSet<(Hash, BaoFilePart)> { - self.0 - } - - /// Apply the delete set and clear it. - /// - /// This will delete all files marked for deletion and then clear the set. - /// Errors will just be logged. - pub fn apply_and_clear(&mut self, options: &PathOptions) { - for (hash, to_delete) in &self.0 { - tracing::debug!("deleting {:?} for {hash}", to_delete); - let path = match to_delete { - BaoFilePart::Data => options.owned_data_path(hash), - BaoFilePart::Outboard => options.owned_outboard_path(hash), - BaoFilePart::Sizes => options.owned_sizes_path(hash), - }; - if let Err(cause) = std::fs::remove_file(&path) { - // Ignore NotFound errors, if the file is already gone that's fine. - if cause.kind() != std::io::ErrorKind::NotFound { - tracing::warn!( - "failed to delete {:?} {}: {}", - to_delete, - path.display(), - cause - ); - } - } - } - self.0.clear(); - } -} diff --git a/iroh-blobs/src/store/fs/test_support.rs b/iroh-blobs/src/store/fs/test_support.rs deleted file mode 100644 index 9cc62bb869e..00000000000 --- a/iroh-blobs/src/store/fs/test_support.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! DB functions to support testing -//! -//! For some tests we need to modify the state of the store in ways that are not -//! possible through the public API. This module provides functions to do that. -use std::{ - io, - path::{Path, PathBuf}, -}; - -use redb::ReadableTable; - -use super::{ - tables::{ReadableTables, Tables}, - ActorError, ActorMessage, ActorResult, ActorState, DataLocation, EntryState, FilterPredicate, - OutboardLocation, OuterResult, Store, StoreInner, -}; -use crate::{ - store::{mutable_mem_storage::SizeInfo, DbIter}, - util::raw_outboard_size, - Hash, -}; - -/// The full state of an entry, including the data. -#[derive(derive_more::Debug)] -pub enum EntryData { - /// Complete - Complete { - /// Data - #[debug("data")] - data: Vec, - /// Outboard - #[debug("outboard")] - outboard: Vec, - }, - /// Partial - Partial { - /// Data - #[debug("data")] - data: Vec, - /// Outboard - #[debug("outboard")] - outboard: Vec, - /// Sizes - #[debug("sizes")] - sizes: Vec, - }, -} - -impl Store { - /// Get the complete state of an entry, both in memory and in redb. - #[cfg(test)] - pub(crate) async fn entry_state(&self, hash: Hash) -> io::Result { - Ok(self.0.entry_state(hash).await?) - } - - async fn all_blobs(&self) -> io::Result> { - Ok(Box::new(self.0.all_blobs().await?.into_iter())) - } - - /// Transform all entries in the store. This is for testing and can be used to get the store - /// in a wrong state. - pub async fn transform_entries( - &self, - transform: impl Fn(Hash, EntryData) -> Option + Send + Sync, - ) -> io::Result<()> { - let blobs = self.all_blobs().await?; - for blob in blobs { - let hash = blob?; - let entry = self.get_full_entry_state(hash).await?; - if let Some(entry) = entry { - let entry1 = transform(hash, entry); - self.set_full_entry_state(hash, entry1).await?; - } - } - Ok(()) - } - - /// Set the full entry state for a hash. This is for testing and can be used to get the store - /// in a wrong state. - pub(crate) async fn set_full_entry_state( - &self, - hash: Hash, - entry: Option, - ) -> io::Result<()> { - Ok(self.0.set_full_entry_state(hash, entry).await?) - } - - /// Set the full entry state for a hash. This is for testing and can be used to get the store - /// in a wrong state. - pub(crate) async fn get_full_entry_state(&self, hash: Hash) -> io::Result> { - Ok(self.0.get_full_entry_state(hash).await?) - } - - /// Owned data path - pub fn owned_data_path(&self, hash: &Hash) -> PathBuf { - self.0.path_options.owned_data_path(hash) - } - - /// Owned outboard path - pub fn owned_outboard_path(&self, hash: &Hash) -> PathBuf { - self.0.path_options.owned_outboard_path(hash) - } -} - -impl StoreInner { - #[cfg(test)] - async fn entry_state(&self, hash: Hash) -> OuterResult { - let (tx, rx) = oneshot::channel(); - self.tx.send(ActorMessage::EntryState { hash, tx }).await?; - Ok(rx.await??) - } - - async fn set_full_entry_state(&self, hash: Hash, entry: Option) -> OuterResult<()> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::SetFullEntryState { hash, entry, tx }) - .await?; - Ok(rx.await??) - } - - async fn get_full_entry_state(&self, hash: Hash) -> OuterResult> { - let (tx, rx) = oneshot::channel(); - self.tx - .send(ActorMessage::GetFullEntryState { hash, tx }) - .await?; - Ok(rx.await??) - } - - async fn all_blobs(&self) -> OuterResult>> { - let (tx, rx) = oneshot::channel(); - let filter: FilterPredicate = - Box::new(|_i, k, v| Some((k.value(), v.value()))); - self.tx.send(ActorMessage::Blobs { filter, tx }).await?; - let blobs = rx.await?; - let res = blobs? - .into_iter() - .map(|r| { - r.map(|(hash, _)| hash) - .map_err(|e| ActorError::from(e).into()) - }) - .collect::>(); - Ok(res) - } -} - -#[cfg(test)] -#[derive(Debug)] -pub(crate) struct EntryStateResponse { - pub mem: Option, - pub db: Option>>, -} - -impl ActorState { - pub(super) fn get_full_entry_state( - &mut self, - tables: &impl ReadableTables, - hash: Hash, - ) -> ActorResult> { - let data_path = self.options.path.owned_data_path(&hash); - let outboard_path = self.options.path.owned_outboard_path(&hash); - let sizes_path = self.options.path.owned_sizes_path(&hash); - let entry = match tables.blobs().get(hash)? { - Some(guard) => match guard.value() { - EntryState::Complete { - data_location, - outboard_location, - } => { - let data = match data_location { - DataLocation::External(paths, size) => { - let path = paths.first().ok_or_else(|| { - ActorError::Inconsistent("external data missing".to_owned()) - })?; - let res = std::fs::read(path)?; - if res.len() != size as usize { - return Err(ActorError::Inconsistent( - "external data size mismatch".to_owned(), - )); - } - res - } - DataLocation::Owned(size) => { - let res = std::fs::read(data_path)?; - if res.len() != size as usize { - return Err(ActorError::Inconsistent( - "owned data size mismatch".to_owned(), - )); - } - res - } - DataLocation::Inline(_) => { - let data = tables.inline_data().get(hash)?.ok_or_else(|| { - ActorError::Inconsistent("inline data missing".to_owned()) - })?; - data.value().to_vec() - } - }; - let expected_outboard_size = raw_outboard_size(data.len() as u64); - let outboard = match outboard_location { - OutboardLocation::Owned => std::fs::read(outboard_path)?, - OutboardLocation::Inline(_) => tables - .inline_outboard() - .get(hash)? - .ok_or_else(|| { - ActorError::Inconsistent("inline outboard missing".to_owned()) - })? - .value() - .to_vec(), - OutboardLocation::NotNeeded => Vec::new(), - }; - if outboard.len() != expected_outboard_size as usize { - return Err(ActorError::Inconsistent( - "outboard size mismatch".to_owned(), - )); - } - Some(EntryData::Complete { data, outboard }) - } - EntryState::Partial { .. } => { - let data = std::fs::read(data_path)?; - let outboard = std::fs::read(outboard_path)?; - let sizes = std::fs::read(sizes_path)?; - Some(EntryData::Partial { - data, - outboard, - sizes, - }) - } - }, - None => None, - }; - Ok(entry) - } - - pub(super) fn set_full_entry_state( - &mut self, - tables: &mut Tables, - hash: Hash, - entry: Option, - ) -> ActorResult<()> { - let data_path = self.options.path.owned_data_path(&hash); - let outboard_path = self.options.path.owned_outboard_path(&hash); - let sizes_path = self.options.path.owned_sizes_path(&hash); - // tabula rasa - std::fs::remove_file(&outboard_path).ok(); - std::fs::remove_file(&data_path).ok(); - std::fs::remove_file(&sizes_path).ok(); - tables.inline_data.remove(&hash)?; - tables.inline_outboard.remove(&hash)?; - let Some(entry) = entry else { - tables.blobs.remove(&hash)?; - return Ok(()); - }; - // write the new data and determine the new state - let entry = match entry { - EntryData::Complete { data, outboard } => { - let data_size = data.len() as u64; - let data_location = if data_size > self.options.inline.max_data_inlined { - std::fs::write(data_path, &data)?; - DataLocation::Owned(data_size) - } else { - tables.inline_data.insert(hash, data.as_slice())?; - DataLocation::Inline(()) - }; - let outboard_size = outboard.len() as u64; - let outboard_location = if outboard_size > self.options.inline.max_outboard_inlined - { - std::fs::write(outboard_path, &outboard)?; - OutboardLocation::Owned - } else if outboard_size > 0 { - tables.inline_outboard.insert(hash, outboard.as_slice())?; - OutboardLocation::Inline(()) - } else { - OutboardLocation::NotNeeded - }; - EntryState::Complete { - data_location, - outboard_location, - } - } - EntryData::Partial { - data, - outboard, - sizes, - } => { - std::fs::write(data_path, data)?; - std::fs::write(outboard_path, outboard)?; - std::fs::write(sizes_path, sizes)?; - EntryState::Partial { size: None } - } - }; - // finally, write the state - tables.blobs.insert(hash, entry)?; - Ok(()) - } - - #[cfg(test)] - pub(super) fn entry_state( - &mut self, - tables: &impl ReadableTables, - hash: Hash, - ) -> ActorResult { - let mem = self.handles.get(&hash).and_then(|weak| weak.upgrade()); - let db = match tables.blobs().get(hash)? { - Some(entry) => Some({ - match entry.value() { - EntryState::Complete { - data_location, - outboard_location, - } => { - let data_location = match data_location { - DataLocation::Inline(()) => { - let data = tables.inline_data().get(hash)?.ok_or_else(|| { - ActorError::Inconsistent("inline data missing".to_owned()) - })?; - DataLocation::Inline(data.value().to_vec()) - } - DataLocation::Owned(x) => DataLocation::Owned(x), - DataLocation::External(p, s) => DataLocation::External(p, s), - }; - let outboard_location = match outboard_location { - OutboardLocation::Inline(()) => { - let outboard = - tables.inline_outboard().get(hash)?.ok_or_else(|| { - ActorError::Inconsistent( - "inline outboard missing".to_owned(), - ) - })?; - OutboardLocation::Inline(outboard.value().to_vec()) - } - OutboardLocation::Owned => OutboardLocation::Owned, - OutboardLocation::NotNeeded => OutboardLocation::NotNeeded, - }; - EntryState::Complete { - data_location, - outboard_location, - } - } - EntryState::Partial { size } => EntryState::Partial { size }, - } - }), - None => None, - }; - Ok(EntryStateResponse { mem, db }) - } -} - -/// What do to with a file pair when making partial files -#[derive(Debug)] -pub enum MakePartialResult { - /// leave the file as is - Retain, - /// remove it entirely - Remove, - /// truncate the data file to the given size - Truncate(u64), -} - -/// Open a database and make it partial. -pub fn make_partial( - path: &Path, - f: impl Fn(Hash, u64) -> MakePartialResult + Send + Sync, -) -> io::Result<()> { - tracing::info!("starting runtime for make_partial"); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - rt.block_on(async move { - let blobs_path = path.join("blobs"); - let store = Store::load(blobs_path).await?; - store - .transform_entries(|hash, entry| match &entry { - EntryData::Complete { data, outboard } => { - let res = f(hash, data.len() as u64); - tracing::info!("make_partial: {} {:?}", hash, res); - match res { - MakePartialResult::Retain => Some(entry), - MakePartialResult::Remove => None, - MakePartialResult::Truncate(size) => { - let current_size = data.len() as u64; - if size < current_size { - let size = size as usize; - let sizes = SizeInfo::complete(current_size).to_vec(); - Some(EntryData::Partial { - data: data[..size].to_vec(), - outboard: outboard.to_vec(), - sizes, - }) - } else { - Some(entry) - } - } - } - } - EntryData::Partial { .. } => Some(entry), - }) - .await?; - std::io::Result::Ok(()) - })?; - drop(rt); - tracing::info!("done with make_partial"); - Ok(()) -} diff --git a/iroh-blobs/src/store/fs/tests.rs b/iroh-blobs/src/store/fs/tests.rs deleted file mode 100644 index 85540eb8912..00000000000 --- a/iroh-blobs/src/store/fs/tests.rs +++ /dev/null @@ -1,811 +0,0 @@ -use std::io::Cursor; - -use bao_tree::ChunkRanges; -use iroh_io::AsyncSliceReaderExt; - -use crate::{ - store::{ - bao_file::test_support::{ - decode_response_into_batch, make_wire_data, random_test_data, simulate_remote, validate, - }, - Map as _, MapEntryMut, MapMut, ReadableStore, Store as _, - }, - util::raw_outboard, - IROH_BLOCK_SIZE, -}; - -macro_rules! assert_matches { - ($expression:expr, $pattern:pat) => { - match $expression { - $pattern => (), - _ => panic!("assertion failed: `(expr matches pattern)` \ - expression: `{:?}`, pattern: `{}`", $expression, stringify!($pattern)), - } - }; - ($expression:expr, $pattern:pat, $($arg:tt)+) => { - match $expression { - $pattern => (), - _ => panic!("{}: expression: `{:?}`, pattern: `{}`", format_args!($($arg)+), $expression, stringify!($pattern)), - } - }; - } - -use super::*; - -/// Helper to simulate a slow request. -pub fn to_stream( - data: &[u8], - mtu: usize, - delay: std::time::Duration, -) -> impl Stream> + 'static { - let parts = data - .chunks(mtu) - .map(Bytes::copy_from_slice) - .collect::>(); - futures_lite::stream::iter(parts) - .then(move |part| async move { - tokio::time::sleep(delay).await; - io::Result::Ok(part) - }) - .boxed() -} - -async fn create_test_db() -> (tempfile::TempDir, Store) { - let _ = tracing_subscriber::fmt::try_init(); - let testdir = tempfile::tempdir().unwrap(); - let db_path = testdir.path().join("db.redb"); - let options = Options { - path: PathOptions::new(testdir.path()), - batch: Default::default(), - inline: Default::default(), - }; - let db = Store::new(db_path, options).await.unwrap(); - (testdir, db) -} - -/// small file that does not have outboard at all -const SMALL_SIZE: u64 = 1024; -/// medium file that has inline outboard but file data -const MID_SIZE: u64 = 1024 * 32; -/// large file that has file outboard and file data -const LARGE_SIZE: u64 = 1024 * 1024 * 10; - -#[tokio::test] -async fn get_cases() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - { - let small = Bytes::from(random_test_data(SMALL_SIZE as usize)); - let (_outboard, hash) = raw_outboard(&small); - let res = db.get(&hash).await.unwrap(); - assert_matches!(res, None); - let tt = db - .import_bytes(small.clone(), BlobFormat::Raw) - .await - .unwrap(); - let res = db.get(&hash).await.unwrap(); - let entry = res.expect("entry not found"); - let actual = entry.data_reader().read_to_end().await.unwrap(); - assert_eq!(actual, small); - drop(tt); - } - { - let mid = Bytes::from(random_test_data(MID_SIZE as usize)); - let (_outboard, hash) = raw_outboard(&mid); - let res = db.get(&hash).await.unwrap(); - assert_matches!(res, None); - let tt = db.import_bytes(mid.clone(), BlobFormat::Raw).await.unwrap(); - let res = db.get(&hash).await.unwrap(); - let entry = res.expect("entry not found"); - let actual = entry.data_reader().read_to_end().await.unwrap(); - assert_eq!(actual, mid); - drop(tt); - } - { - let large = Bytes::from(random_test_data(LARGE_SIZE as usize)); - let (_outboard, hash) = raw_outboard(&large); - let res = db.get(&hash).await.unwrap(); - assert_matches!(res, None); - let tt = db - .import_bytes(large.clone(), BlobFormat::Raw) - .await - .unwrap(); - let res = db.get(&hash).await.unwrap(); - let entry = res.expect("entry not found"); - let actual = entry.data_reader().read_to_end().await.unwrap(); - assert_eq!(actual, large); - drop(tt); - } - { - let mid = random_test_data(MID_SIZE as usize); - let path = tempdir.path().join("mid.data"); - std::fs::write(&path, &mid).unwrap(); - let (_outboard, hash) = raw_outboard(&mid); - let res = db.get(&hash).await.unwrap(); - assert_matches!(res, None); - let tt = db - .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np()) - .await - .unwrap(); - let res = db.get(&hash).await.unwrap(); - let entry = res.expect("entry not found"); - let actual = entry.data_reader().read_to_end().await.unwrap(); - assert_eq!(actual, mid); - drop(tt); - } -} - -#[tokio::test] -async fn get_or_create_cases() { - let (_tempdir, db) = create_test_db().await; - { - const SIZE: u64 = SMALL_SIZE; - let data = random_test_data(SIZE as usize); - let (hash, reader) = simulate_remote(&data); - let entry = db.get_or_create(hash, 0).await.unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_eq!(state.db, None); - assert_matches!(state.mem, Some(_)); - } - let writer = entry.batch_writer().await.unwrap(); - decode_response_into_batch(hash, IROH_BLOCK_SIZE, ChunkRanges::all(), reader, writer) - .await - .unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_matches!(state.mem, Some(_)); - assert_matches!(state.db, None); - } - db.insert_complete(entry.clone()).await.unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_matches!(state.mem, Some(_)); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Inline(_), - .. - }) - ); - } - drop(entry); - // sync so we know the msg sent on drop is processed - db.sync().await.unwrap(); - let state = db.entry_state(hash).await.unwrap(); - assert_matches!(state.mem, None); - } - { - const SIZE: u64 = MID_SIZE; - let data = random_test_data(SIZE as usize); - let (hash, reader) = simulate_remote(&data); - let entry = db.get_or_create(hash, 0).await.unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_eq!(state.db, None); - assert_matches!(state.mem, Some(_)); - } - let writer = entry.batch_writer().await.unwrap(); - decode_response_into_batch(hash, IROH_BLOCK_SIZE, ChunkRanges::all(), reader, writer) - .await - .unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_matches!(state.mem, Some(_)); - assert_matches!(state.db, Some(EntryState::Partial { .. })); - } - db.insert_complete(entry.clone()).await.unwrap(); - { - let state = db.entry_state(hash).await.unwrap(); - assert_matches!(state.mem, Some(_)); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - .. - }) - ); - } - drop(entry); - // // sync so we know the msg sent on drop is processed - // db.sync().await.unwrap(); - // let state = db.entry_state(hash).await.unwrap(); - // assert_matches!(state.mem, None); - } -} - -/// Import mem cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file) -#[tokio::test] -async fn import_mem_cases() { - let (_tempdir, db) = create_test_db().await; - { - const SIZE: u64 = SMALL_SIZE; - let small = random_test_data(SIZE as usize); - let (outboard, hash) = raw_outboard(&small); - let tt = db - .import_bytes(small.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }; - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert!(outboard.is_empty()); - } - { - const SIZE: u64 = MID_SIZE; - let mid = Bytes::from(random_test_data(SIZE as usize)); - let (outboard, hash) = raw_outboard(&mid); - let tt = db.import_bytes(mid.clone(), BlobFormat::Raw).await.unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Inline(outboard), - }; - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap()); - } - { - const SIZE: u64 = LARGE_SIZE; - let large = Bytes::from(random_test_data(SIZE as usize)); - let (outboard, hash) = raw_outboard(&large); - let tt = db - .import_bytes(large.clone(), BlobFormat::Raw) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Owned, - }; - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap()); - assert_eq!( - outboard, - tokio::fs::read(db.owned_outboard_path(&hash)) - .await - .unwrap() - ); - } -} - -/// Import mem cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file) -#[tokio::test] -async fn import_stream_cases() { - let np = IgnoreProgressSender::::default; - let (_tempdir, db) = create_test_db().await; - { - const SIZE: u64 = SMALL_SIZE; - let small = random_test_data(SIZE as usize); - let (outboard, hash) = raw_outboard(&small); - let (tt, size) = db - .import_stream( - to_stream(&small, 100, Duration::from_millis(1)), - BlobFormat::Raw, - np(), - ) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert!(outboard.is_empty()); - } - { - const SIZE: u64 = MID_SIZE; - let mid = Bytes::from(random_test_data(SIZE as usize)); - let (outboard, hash) = raw_outboard(&mid); - let (tt, size) = db - .import_stream( - to_stream(&mid, 1000, Duration::from_millis(1)), - BlobFormat::Raw, - np(), - ) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Inline(outboard), - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap()); - } - { - const SIZE: u64 = LARGE_SIZE; - let large = Bytes::from(random_test_data(SIZE as usize)); - let (outboard, hash) = raw_outboard(&large); - let (tt, size) = db - .import_stream( - to_stream(&large, 100000, Duration::from_millis(1)), - BlobFormat::Raw, - np(), - ) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Owned, - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap()); - assert_eq!( - outboard, - tokio::fs::read(db.owned_outboard_path(&hash)) - .await - .unwrap() - ); - } -} - -/// Import file cases, small (data inline, outboard none), mid (data file, outboard inline), large (data file, outboard file) -#[tokio::test] -async fn import_file_cases() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - { - const SIZE: u64 = SMALL_SIZE; - let small = random_test_data(SIZE as usize); - let path = tempdir.path().join("small.data"); - std::fs::write(&path, &small).unwrap(); - let (outboard, hash) = raw_outboard(&small); - let (tt, size) = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert!(outboard.is_empty()); - } - { - const SIZE: u64 = MID_SIZE; - let mid = Bytes::from(random_test_data(SIZE as usize)); - let path = tempdir.path().join("mid.data"); - std::fs::write(&path, &mid).unwrap(); - let (outboard, hash) = raw_outboard(&mid); - let (tt, size) = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Inline(outboard), - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(mid, std::fs::read(db.owned_data_path(&hash)).unwrap()); - } - { - const SIZE: u64 = LARGE_SIZE; - let large = Bytes::from(random_test_data(SIZE as usize)); - let path = tempdir.path().join("mid.data"); - std::fs::write(&path, &large).unwrap(); - let (outboard, hash) = raw_outboard(&large); - let (tt, size) = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Owned(SIZE), - outboard_location: OutboardLocation::Owned, - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(large, std::fs::read(db.owned_data_path(&hash)).unwrap()); - assert_eq!( - outboard, - tokio::fs::read(db.owned_outboard_path(&hash)) - .await - .unwrap() - ); - } -} - -#[tokio::test] -async fn import_file_reference_cases() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - { - const SIZE: u64 = SMALL_SIZE; - let small = random_test_data(SIZE as usize); - let path = tempdir.path().join("small.data"); - std::fs::write(&path, &small).unwrap(); - let (outboard, hash) = raw_outboard(&small); - let (tt, size) = db - .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np()) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert!(outboard.is_empty()); - } - { - const SIZE: u64 = MID_SIZE; - let mid = random_test_data(SIZE as usize); - let path = tempdir.path().join("mid.data"); - std::fs::write(&path, &mid).unwrap(); - let (outboard, hash) = raw_outboard(&mid); - let (tt, size) = db - .import_file( - path.clone(), - ImportMode::TryReference, - BlobFormat::Raw, - np(), - ) - .await - .unwrap(); - let actual = db.entry_state(*tt.hash()).await.unwrap(); - let expected = EntryState::Complete { - data_location: DataLocation::External(vec![path.clone()], SIZE), - outboard_location: OutboardLocation::Inline(outboard), - }; - assert_eq!(size, SIZE); - assert_eq!(tt.hash(), &hash); - assert_eq!(actual.db, Some(expected)); - assert_eq!(mid, std::fs::read(path).unwrap()); - assert!(!db.owned_data_path(&hash).exists()); - } -} - -#[tokio::test] -async fn import_file_error_cases() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - // relative path is not allowed - { - let path = PathBuf::from("relativepath.data"); - let cause = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap_err(); - assert_eq!(cause.kind(), io::ErrorKind::InvalidInput); - } - // file does not exist - { - let path = tempdir.path().join("pathdoesnotexist.data"); - let cause = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap_err(); - assert_eq!(cause.kind(), io::ErrorKind::InvalidInput); - } - // file is a directory - { - let path = tempdir.path().join("pathisdir.data"); - std::fs::create_dir_all(&path).unwrap(); - let cause = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap_err(); - assert_eq!(cause.kind(), io::ErrorKind::InvalidInput); - } -} - -#[tokio::test] -async fn import_file_tempdir_is_file() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - // temp dir is readonly, this is a bit mean since we mess with the internals of the store - { - let temp_dir = db.0.temp_file_name().parent().unwrap().to_owned(); - std::fs::remove_dir_all(&temp_dir).unwrap(); - std::fs::write(temp_dir, []).unwrap(); - // std::fs::set_permissions(temp_dir, std::os::unix::fs::PermissionsExt::from_mode(0o0)) - // .unwrap(); - let path = tempdir.path().join("mid.data"); - let data = random_test_data(MID_SIZE as usize); - std::fs::write(&path, &data).unwrap(); - let _cause = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap_err(); - // cause is NotADirectory, but unstable - // assert_eq!(cause.kind(), io::ErrorKind::NotADirectory); - } - drop(tempdir); -} - -#[tokio::test] -async fn import_file_datadir_is_file() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - // temp dir is readonly, this is a bit mean since we mess with the internals of the store - { - let data_dir = db.0.path_options.data_path.to_owned(); - std::fs::remove_dir_all(&data_dir).unwrap(); - std::fs::write(data_dir, []).unwrap(); - let path = tempdir.path().join("mid.data"); - let data = random_test_data(MID_SIZE as usize); - std::fs::write(&path, &data).unwrap(); - let _cause = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap_err(); - // cause is NotADirectory, but unstable - // assert_eq!(cause.kind(), io::ErrorKind::NotADirectory); - } - drop(tempdir); -} - -/// tests that owned wins over external in both cases -#[tokio::test] -async fn import_file_overwrite() { - let np = IgnoreProgressSender::::default; - let (tempdir, db) = create_test_db().await; - // overwrite external with owned - { - let path = tempdir.path().join("mid.data"); - let data = random_test_data(MID_SIZE as usize); - let (_outboard, hash) = raw_outboard(&data); - std::fs::write(&path, &data).unwrap(); - let (tt1, size1) = db - .import_file(path.clone(), ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap(); - assert_eq!(size1, MID_SIZE); - assert_eq!(tt1.hash(), &hash); - let state = db.entry_state(hash).await.unwrap(); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(_), - .. - }) - ); - let (tt2, size2) = db - .import_file(path, ImportMode::TryReference, BlobFormat::Raw, np()) - .await - .unwrap(); - assert_eq!(size2, MID_SIZE); - assert_eq!(tt2.hash(), &hash); - let state = db.entry_state(hash).await.unwrap(); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(_), - .. - }) - ); - } - { - let path = tempdir.path().join("mid2.data"); - let data = random_test_data(MID_SIZE as usize); - let (_outboard, hash) = raw_outboard(&data); - std::fs::write(&path, &data).unwrap(); - let (tt1, size1) = db - .import_file( - path.clone(), - ImportMode::TryReference, - BlobFormat::Raw, - np(), - ) - .await - .unwrap(); - let state = db.entry_state(hash).await.unwrap(); - assert_eq!(size1, MID_SIZE); - assert_eq!(tt1.hash(), &hash); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::External(_, _), - .. - }) - ); - let (tt2, size2) = db - .import_file(path, ImportMode::Copy, BlobFormat::Raw, np()) - .await - .unwrap(); - let state = db.entry_state(hash).await.unwrap(); - assert_eq!(size2, MID_SIZE); - assert_eq!(tt2.hash(), &hash); - assert_matches!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(_), - .. - }) - ); - } -} - -/// tests that export works in copy mode -#[tokio::test] -async fn export_copy_cases() { - let np = || Box::new(|_: u64| io::Result::Ok(())); - let (tempdir, db) = create_test_db().await; - let small = random_test_data(SMALL_SIZE as usize); - let mid = random_test_data(MID_SIZE as usize); - let large = random_test_data(LARGE_SIZE as usize); - let small_tt = db - .import_bytes(small.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let mid_tt = db - .import_bytes(mid.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let large_tt = db - .import_bytes(large.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let small_path = tempdir.path().join("small.data"); - let mid_path = tempdir.path().join("mid.data"); - let large_path = tempdir.path().join("large.data"); - db.export(*small_tt.hash(), small_path.clone(), ExportMode::Copy, np()) - .await - .unwrap(); - assert_eq!(small.to_vec(), std::fs::read(&small_path).unwrap()); - db.export(*mid_tt.hash(), mid_path.clone(), ExportMode::Copy, np()) - .await - .unwrap(); - assert_eq!(mid.to_vec(), std::fs::read(&mid_path).unwrap()); - db.export(*large_tt.hash(), large_path.clone(), ExportMode::Copy, np()) - .await - .unwrap(); - assert_eq!(large.to_vec(), std::fs::read(&large_path).unwrap()); - let state = db.entry_state(*small_tt.hash()).await.unwrap(); - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }) - ); - let state = db.entry_state(*mid_tt.hash()).await.unwrap(); - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(MID_SIZE), - outboard_location: OutboardLocation::Inline(raw_outboard(&mid).0), - }) - ); - let state = db.entry_state(*large_tt.hash()).await.unwrap(); - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Owned(LARGE_SIZE), - outboard_location: OutboardLocation::Owned, - }) - ); -} - -/// tests that export works in reference mode -#[tokio::test] -async fn export_reference_cases() { - let np = || Box::new(|_: u64| io::Result::Ok(())); - let (tempdir, db) = create_test_db().await; - let small = random_test_data(SMALL_SIZE as usize); - let mid = random_test_data(MID_SIZE as usize); - let large = random_test_data(LARGE_SIZE as usize); - let small_tt = db - .import_bytes(small.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let mid_tt = db - .import_bytes(mid.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let large_tt = db - .import_bytes(large.clone().into(), BlobFormat::Raw) - .await - .unwrap(); - let small_path = tempdir.path().join("small.data"); - let mid_path = tempdir.path().join("mid.data"); - let large_path = tempdir.path().join("large.data"); - db.export( - *small_tt.hash(), - small_path.clone(), - ExportMode::TryReference, - np(), - ) - .await - .unwrap(); - assert_eq!(small.to_vec(), std::fs::read(&small_path).unwrap()); - db.export( - *mid_tt.hash(), - mid_path.clone(), - ExportMode::TryReference, - np(), - ) - .await - .unwrap(); - assert_eq!(mid.to_vec(), std::fs::read(&mid_path).unwrap()); - db.export( - *large_tt.hash(), - large_path.clone(), - ExportMode::TryReference, - np(), - ) - .await - .unwrap(); - assert_eq!(large.to_vec(), std::fs::read(&large_path).unwrap()); - let state = db.entry_state(*small_tt.hash()).await.unwrap(); - // small entries will never use external references - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::Inline(small), - outboard_location: OutboardLocation::NotNeeded, - }) - ); - // mid entries should now use external references - let state = db.entry_state(*mid_tt.hash()).await.unwrap(); - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::External(vec![mid_path], MID_SIZE), - outboard_location: OutboardLocation::Inline(raw_outboard(&mid).0), - }) - ); - // large entries should now use external references - let state = db.entry_state(*large_tt.hash()).await.unwrap(); - assert_eq!( - state.db, - Some(EntryState::Complete { - data_location: DataLocation::External(vec![large_path], LARGE_SIZE), - outboard_location: OutboardLocation::Owned, - }) - ); -} - -#[tokio::test] -async fn actor_store_smoke() { - let testdir = tempfile::tempdir().unwrap(); - let db_path = testdir.path().join("test.redb"); - let options = Options { - path: PathOptions::new(testdir.path()), - batch: Default::default(), - inline: Default::default(), - }; - let db = Store::new(db_path, options).await.unwrap(); - db.dump().await.unwrap(); - let data = random_test_data(1024 * 1024); - #[allow(clippy::single_range_in_vec_init)] - let ranges = [0..data.len() as u64]; - let (hash, chunk_ranges, wire_data) = make_wire_data(&data, &ranges); - let handle = db.get_or_create(hash, 0).await.unwrap(); - decode_response_into_batch( - hash, - IROH_BLOCK_SIZE, - chunk_ranges.clone(), - Cursor::new(wire_data.as_slice()), - handle.batch_writer().await.unwrap(), - ) - .await - .unwrap(); - validate(&handle, &data, &ranges).await; - db.insert_complete(handle).await.unwrap(); - db.sync().await.unwrap(); - db.dump().await.unwrap(); -} diff --git a/iroh-blobs/src/store/fs/util.rs b/iroh-blobs/src/store/fs/util.rs deleted file mode 100644 index b747d9d7b5a..00000000000 --- a/iroh-blobs/src/store/fs/util.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::{ - fs::OpenOptions, - io::{self, Write}, - path::Path, -}; - -/// overwrite a file with the given data. -/// -/// This is almost like `std::fs::write`, but it does not truncate the file. -/// -/// So if you overwrite a file with less data than it had before, the file will -/// still have the same size as before. -/// -/// Also, if you overwrite a file with the same data as it had before, the -/// file will be unchanged even if the overwrite operation is interrupted. -pub fn overwrite_and_sync(path: &Path, data: &[u8]) -> io::Result { - tracing::trace!( - "overwriting file {} with {} bytes", - path.display(), - data.len() - ); - // std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - // tracing::error!("{}", path.parent().unwrap().display()); - // tracing::error!("{}", path.parent().unwrap().metadata().unwrap().is_dir()); - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(path)?; - file.write_all(data)?; - // todo: figure out if it is safe to not sync here - file.sync_all()?; - Ok(file) -} - -/// Read a file into memory and then delete it. -pub fn read_and_remove(path: &Path) -> io::Result> { - let data = std::fs::read(path)?; - // todo: should we fail here or just log a warning? - // remove could fail e.g. on windows if the file is still open - std::fs::remove_file(path)?; - Ok(data) -} - -/// A wrapper for a flume receiver that allows peeking at the next message. -#[derive(Debug)] -pub(super) struct PeekableFlumeReceiver { - msg: Option, - recv: async_channel::Receiver, -} - -#[allow(dead_code)] -impl PeekableFlumeReceiver { - pub fn new(recv: async_channel::Receiver) -> Self { - Self { msg: None, recv } - } - - /// Receive the next message. - /// - /// Will block if there are no messages. - /// Returns None only if there are no more messages (sender is dropped). - pub async fn recv(&mut self) -> Option { - if let Some(msg) = self.msg.take() { - return Some(msg); - } - self.recv.recv().await.ok() - } - - /// Push back a message. This will only work if there is room for it. - /// Otherwise, it will fail and return the message. - pub fn push_back(&mut self, msg: T) -> std::result::Result<(), T> { - if self.msg.is_none() { - self.msg = Some(msg); - Ok(()) - } else { - Err(msg) - } - } -} diff --git a/iroh-blobs/src/store/fs/validate.rs b/iroh-blobs/src/store/fs/validate.rs deleted file mode 100644 index ae1870471db..00000000000 --- a/iroh-blobs/src/store/fs/validate.rs +++ /dev/null @@ -1,492 +0,0 @@ -//! Validation of the store's contents. -use std::collections::BTreeSet; - -use redb::ReadableTable; - -use super::{ - raw_outboard_size, tables::Tables, ActorResult, ActorState, DataLocation, EntryState, Hash, - OutboardLocation, -}; -use crate::{ - store::{fs::tables::BaoFilePart, ConsistencyCheckProgress, ReportLevel}, - util::progress::BoxedProgressSender, -}; - -impl ActorState { - //! This performs a full consistency check. Eventually it will also validate - //! file content again, but that part is not yet implemented. - //! - //! Currently the following checks are performed for complete entries: - //! - //! Check that the data in the entries table is consistent with the data in - //! the inline_data and inline_outboard tables. - //! - //! For every entry where data_location is inline, the inline_data table - //! must contain the data. For every entry where - //! data_location is not inline, the inline_data table must not contain data. - //! Instead, the data must exist as a file in the data directory or be - //! referenced to one or many external files. - //! - //! For every entry where outboard_location is inline, the inline_outboard - //! table must contain the outboard. For every entry where outboard_location - //! is not inline, the inline_outboard table must not contain data, and the - //! outboard must exist as a file in the data directory. Outboards are never - //! external. - //! - //! In addition to these consistency checks, it is checked that the size of - //! the outboard is consistent with the size of the data. - //! - //! For partial entries, it is checked that the data and outboard files - //! exist. - //! - //! In addition to the consistency checks, it is checked that there are no - //! orphaned or unexpected files in the data directory. Also, all entries of - //! all tables are dumped at trace level. This is helpful for debugging and - //! also ensures that the data can be read. - //! - //! Note that during validation, a set of all hashes will be kept in memory. - //! So to validate exceedingly large stores, the validation process will - //! consume a lot of memory. - //! - //! In addition, validation is a blocking operation that will make the store - //! unresponsive for the duration of the validation. - pub(super) fn consistency_check( - &mut self, - db: &redb::Database, - repair: bool, - progress: BoxedProgressSender, - ) -> ActorResult<()> { - use crate::util::progress::ProgressSender; - let mut invalid_entries = BTreeSet::new(); - macro_rules! send { - ($level:expr, $entry:expr, $($arg:tt)*) => { - if let Err(_) = progress.blocking_send(ConsistencyCheckProgress::Update { message: format!($($arg)*), level: $level, entry: $entry }) { - return Ok(()); - } - }; - } - macro_rules! trace { - ($($arg:tt)*) => { - send!(ReportLevel::Trace, None, $($arg)*) - }; - } - macro_rules! info { - ($($arg:tt)*) => { - send!(ReportLevel::Info, None, $($arg)*) - }; - } - macro_rules! warn { - ($($arg:tt)*) => { - send!(ReportLevel::Warn, None, $($arg)*) - }; - } - macro_rules! entry_warn { - ($hash:expr, $($arg:tt)*) => { - send!(ReportLevel::Warn, Some($hash), $($arg)*) - }; - } - macro_rules! entry_info { - ($hash:expr, $($arg:tt)*) => { - send!(ReportLevel::Info, Some($hash), $($arg)*) - }; - } - macro_rules! error { - ($($arg:tt)*) => { - send!(ReportLevel::Error, None, $($arg)*) - }; - } - macro_rules! entry_error { - ($hash:expr, $($arg:tt)*) => { - invalid_entries.insert($hash); - send!(ReportLevel::Error, Some($hash), $($arg)*) - }; - } - let mut delete_after_commit = Default::default(); - let txn = db.begin_write()?; - { - let mut tables = Tables::new(&txn, &mut delete_after_commit)?; - let blobs = &mut tables.blobs; - let inline_data = &mut tables.inline_data; - let inline_outboard = &mut tables.inline_outboard; - let tags = &mut tables.tags; - let mut orphaned_inline_data = BTreeSet::new(); - let mut orphaned_inline_outboard = BTreeSet::new(); - let mut orphaned_data = BTreeSet::new(); - let mut orphaned_outboardard = BTreeSet::new(); - let mut orphaned_sizes = BTreeSet::new(); - // first, dump the entire data content at trace level - trace!("dumping blobs"); - match blobs.iter() { - Ok(iter) => { - for item in iter { - match item { - Ok((k, v)) => { - let hash = k.value(); - let entry = v.value(); - trace!("blob {} -> {:?}", hash.to_hex(), entry); - } - Err(cause) => { - error!("failed to access blob item: {}", cause); - } - } - } - } - Err(cause) => { - error!("failed to iterate blobs: {}", cause); - } - } - trace!("dumping inline_data"); - match inline_data.iter() { - Ok(iter) => { - for item in iter { - match item { - Ok((k, v)) => { - let hash = k.value(); - let data = v.value(); - trace!("inline_data {} -> {:?}", hash.to_hex(), data.len()); - } - Err(cause) => { - error!("failed to access inline data item: {}", cause); - } - } - } - } - Err(cause) => { - error!("failed to iterate inline_data: {}", cause); - } - } - trace!("dumping inline_outboard"); - match inline_outboard.iter() { - Ok(iter) => { - for item in iter { - match item { - Ok((k, v)) => { - let hash = k.value(); - let data = v.value(); - trace!("inline_outboard {} -> {:?}", hash.to_hex(), data.len()); - } - Err(cause) => { - error!("failed to access inline outboard item: {}", cause); - } - } - } - } - Err(cause) => { - error!("failed to iterate inline_outboard: {}", cause); - } - } - trace!("dumping tags"); - match tags.iter() { - Ok(iter) => { - for item in iter { - match item { - Ok((k, v)) => { - let tag = k.value(); - let value = v.value(); - trace!("tags {} -> {:?}", tag, value); - } - Err(cause) => { - error!("failed to access tag item: {}", cause); - } - } - } - } - Err(cause) => { - error!("failed to iterate tags: {}", cause); - } - } - - // perform consistency check for each entry - info!("validating blobs"); - // set of a all hashes that are referenced by the blobs table - let mut entries = BTreeSet::new(); - match blobs.iter() { - Ok(iter) => { - for item in iter { - let Ok((hash, entry)) = item else { - error!("failed to access blob item"); - continue; - }; - let hash = hash.value(); - entries.insert(hash); - entry_info!(hash, "validating blob"); - let entry = entry.value(); - match entry { - EntryState::Complete { - data_location, - outboard_location, - } => { - let data_size = match data_location { - DataLocation::Inline(_) => { - let Ok(inline_data) = inline_data.get(hash) else { - entry_error!(hash, "inline data can not be accessed"); - continue; - }; - let Some(inline_data) = inline_data else { - entry_error!(hash, "inline data missing"); - continue; - }; - inline_data.value().len() as u64 - } - DataLocation::Owned(size) => { - let path = self.options.path.owned_data_path(&hash); - let Ok(metadata) = path.metadata() else { - entry_error!(hash, "owned data file does not exist"); - continue; - }; - if metadata.len() != size { - entry_error!( - hash, - "owned data file size mismatch: {}", - path.display() - ); - continue; - } - size - } - DataLocation::External(paths, size) => { - for path in paths { - let Ok(metadata) = path.metadata() else { - entry_error!( - hash, - "external data file does not exist: {}", - path.display() - ); - invalid_entries.insert(hash); - continue; - }; - if metadata.len() != size { - entry_error!( - hash, - "external data file size mismatch: {}", - path.display() - ); - invalid_entries.insert(hash); - continue; - } - } - size - } - }; - match outboard_location { - OutboardLocation::Inline(_) => { - let Ok(inline_outboard) = inline_outboard.get(hash) else { - entry_error!( - hash, - "inline outboard can not be accessed" - ); - continue; - }; - let Some(inline_outboard) = inline_outboard else { - entry_error!(hash, "inline outboard missing"); - continue; - }; - let outboard_size = inline_outboard.value().len() as u64; - if outboard_size != raw_outboard_size(data_size) { - entry_error!(hash, "inline outboard size mismatch"); - } - } - OutboardLocation::Owned => { - let Ok(metadata) = - self.options.path.owned_outboard_path(&hash).metadata() - else { - entry_error!( - hash, - "owned outboard file does not exist" - ); - continue; - }; - let outboard_size = metadata.len(); - if outboard_size != raw_outboard_size(data_size) { - entry_error!(hash, "owned outboard size mismatch"); - } - } - OutboardLocation::NotNeeded => { - if raw_outboard_size(data_size) != 0 { - entry_error!( - hash, - "outboard not needed but data size is not zero" - ); - } - } - } - } - EntryState::Partial { .. } => { - if !self.options.path.owned_data_path(&hash).exists() { - entry_error!(hash, "persistent partial entry has no data"); - } - if !self.options.path.owned_outboard_path(&hash).exists() { - entry_error!(hash, "persistent partial entry has no outboard"); - } - } - } - } - } - Err(cause) => { - error!("failed to iterate blobs: {}", cause); - } - }; - if repair { - info!("repairing - removing invalid entries found so far"); - for hash in &invalid_entries { - blobs.remove(hash)?; - } - } - info!("checking for orphaned inline data"); - match inline_data.iter() { - Ok(iter) => { - for item in iter { - let Ok((hash, _)) = item else { - error!("failed to access inline data item"); - continue; - }; - let hash = hash.value(); - if !entries.contains(&hash) { - orphaned_inline_data.insert(hash); - entry_error!(hash, "orphaned inline data"); - } - } - } - Err(cause) => { - error!("failed to iterate inline_data: {}", cause); - } - }; - info!("checking for orphaned inline outboard data"); - match inline_outboard.iter() { - Ok(iter) => { - for item in iter { - let Ok((hash, _)) = item else { - error!("failed to access inline outboard item"); - continue; - }; - let hash = hash.value(); - if !entries.contains(&hash) { - orphaned_inline_outboard.insert(hash); - entry_error!(hash, "orphaned inline outboard"); - } - } - } - Err(cause) => { - error!("failed to iterate inline_outboard: {}", cause); - } - }; - info!("checking for unexpected or orphaned files"); - for entry in self.options.path.data_path.read_dir()? { - let entry = entry?; - let path = entry.path(); - if !path.is_file() { - warn!("unexpected entry in data directory: {}", path.display()); - continue; - } - match path.extension().and_then(|x| x.to_str()) { - Some("data") => match path.file_stem().and_then(|x| x.to_str()) { - Some(stem) => { - let mut hash = [0u8; 32]; - let Ok(_) = hex::decode_to_slice(stem, &mut hash) else { - warn!("unexpected data file in data directory: {}", path.display()); - continue; - }; - let hash = Hash::from(hash); - if !entries.contains(&hash) { - orphaned_data.insert(hash); - entry_warn!(hash, "orphaned data file"); - } - } - None => { - warn!("unexpected data file in data directory: {}", path.display()); - } - }, - Some("obao4") => match path.file_stem().and_then(|x| x.to_str()) { - Some(stem) => { - let mut hash = [0u8; 32]; - let Ok(_) = hex::decode_to_slice(stem, &mut hash) else { - warn!( - "unexpected outboard file in data directory: {}", - path.display() - ); - continue; - }; - let hash = Hash::from(hash); - if !entries.contains(&hash) { - orphaned_outboardard.insert(hash); - entry_warn!(hash, "orphaned outboard file"); - } - } - None => { - warn!( - "unexpected outboard file in data directory: {}", - path.display() - ); - } - }, - Some("sizes4") => match path.file_stem().and_then(|x| x.to_str()) { - Some(stem) => { - let mut hash = [0u8; 32]; - let Ok(_) = hex::decode_to_slice(stem, &mut hash) else { - warn!( - "unexpected outboard file in data directory: {}", - path.display() - ); - continue; - }; - let hash = Hash::from(hash); - if !entries.contains(&hash) { - orphaned_sizes.insert(hash); - entry_warn!(hash, "orphaned outboard file"); - } - } - None => { - warn!( - "unexpected outboard file in data directory: {}", - path.display() - ); - } - }, - _ => { - warn!("unexpected file in data directory: {}", path.display()); - } - } - } - if repair { - info!("repairing - removing orphaned files and inline data"); - for hash in orphaned_inline_data { - entry_info!(hash, "deleting orphaned inline data"); - inline_data.remove(&hash)?; - } - for hash in orphaned_inline_outboard { - entry_info!(hash, "deleting orphaned inline outboard"); - inline_outboard.remove(&hash)?; - } - for hash in orphaned_data { - tables.delete_after_commit.insert(hash, [BaoFilePart::Data]); - } - for hash in orphaned_outboardard { - tables - .delete_after_commit - .insert(hash, [BaoFilePart::Outboard]); - } - for hash in orphaned_sizes { - tables - .delete_after_commit - .insert(hash, [BaoFilePart::Sizes]); - } - } - } - txn.commit()?; - if repair { - info!("repairing - deleting orphaned files"); - for (hash, part) in delete_after_commit.into_inner() { - let path = match part { - BaoFilePart::Data => self.options.path.owned_data_path(&hash), - BaoFilePart::Outboard => self.options.path.owned_outboard_path(&hash), - BaoFilePart::Sizes => self.options.path.owned_sizes_path(&hash), - }; - entry_info!(hash, "deleting orphaned file: {}", path.display()); - if let Err(cause) = std::fs::remove_file(&path) { - entry_error!(hash, "failed to delete orphaned file: {}", cause); - } - } - } - Ok(()) - } -} diff --git a/iroh-blobs/src/store/mem.rs b/iroh-blobs/src/store/mem.rs deleted file mode 100644 index 15e037a8337..00000000000 --- a/iroh-blobs/src/store/mem.rs +++ /dev/null @@ -1,464 +0,0 @@ -//! A full in memory database for iroh-blobs -//! -//! Main entry point is [Store]. -use std::{ - collections::{BTreeMap, BTreeSet}, - future::Future, - io, - path::PathBuf, - sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, - time::SystemTime, -}; - -use bao_tree::{ - io::{fsm::Outboard, outboard::PreOrderOutboard, sync::WriteAt}, - BaoTree, -}; -use bytes::{Bytes, BytesMut}; -use futures_lite::{Stream, StreamExt}; -use iroh_base::hash::{BlobFormat, Hash, HashAndFormat}; -use iroh_io::AsyncSliceReader; - -use super::{ - temp_name, BaoBatchWriter, ConsistencyCheckProgress, ExportMode, ExportProgressCb, ImportMode, - ImportProgress, Map, TempCounterMap, -}; -use crate::{ - store::{ - mutable_mem_storage::MutableMemStorage, BaoBlobSize, MapEntry, MapEntryMut, ReadableStore, - }, - util::{ - progress::{BoxedProgressSender, IdGenerator, IgnoreProgressSender, ProgressSender}, - TagCounter, TagDrop, - }, - Tag, TempTag, IROH_BLOCK_SIZE, -}; - -/// A fully featured in memory database for iroh-blobs, including support for -/// partial blobs. -#[derive(Debug, Clone, Default)] -pub struct Store { - inner: Arc, -} - -#[derive(Debug, Default)] -struct StoreInner(RwLock); - -impl TagDrop for StoreInner { - fn on_drop(&self, inner: &HashAndFormat) { - tracing::trace!("temp tag drop: {:?}", inner); - let mut state = self.0.write().unwrap(); - state.temp.dec(inner); - } -} - -impl TagCounter for StoreInner { - fn on_create(&self, inner: &HashAndFormat) { - tracing::trace!("temp tagging: {:?}", inner); - let mut state = self.0.write().unwrap(); - state.temp.inc(inner); - } -} - -impl Store { - /// Create a new in memory store - pub fn new() -> Self { - Self::default() - } - - /// Take a write lock on the store - fn write_lock(&self) -> RwLockWriteGuard<'_, StateInner> { - self.inner.0.write().unwrap() - } - - /// Take a read lock on the store - fn read_lock(&self) -> RwLockReadGuard<'_, StateInner> { - self.inner.0.read().unwrap() - } - - fn import_bytes_sync( - &self, - id: u64, - bytes: Bytes, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result { - progress.blocking_send(ImportProgress::OutboardProgress { id, offset: 0 })?; - let progress2 = progress.clone(); - let cb = move |offset| { - progress2 - .try_send(ImportProgress::OutboardProgress { id, offset }) - .ok(); - }; - let (storage, hash) = MutableMemStorage::complete(bytes, cb); - progress.blocking_send(ImportProgress::OutboardDone { id, hash })?; - use super::Store; - let tag = self.temp_tag(HashAndFormat { hash, format }); - let entry = Entry { - inner: Arc::new(EntryInner { - hash, - data: RwLock::new(storage), - }), - complete: true, - }; - self.write_lock().entries.insert(hash, entry); - Ok(tag) - } - - fn export_sync( - &self, - hash: Hash, - target: PathBuf, - _mode: ExportMode, - progress: impl Fn(u64) -> io::Result<()> + Send + Sync + 'static, - ) -> io::Result<()> { - tracing::trace!("exporting {} to {}", hash, target.display()); - - if !target.is_absolute() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "target path must be absolute", - )); - } - let parent = target.parent().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "target path has no parent directory", - ) - })?; - // create the directory in which the target file is - std::fs::create_dir_all(parent)?; - let state = self.read_lock(); - let entry = state - .entries - .get(&hash) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "hash not found"))?; - let reader = &entry.inner.data; - let size = reader.read().unwrap().current_size(); - let mut file = std::fs::File::create(target)?; - for offset in (0..size).step_by(1024 * 1024) { - let bytes = reader.read().unwrap().read_data_at(offset, 1024 * 1024); - file.write_at(offset, &bytes)?; - progress(offset)?; - } - std::io::Write::flush(&mut file)?; - drop(file); - Ok(()) - } -} - -impl super::Store for Store { - async fn import_file( - &self, - path: std::path::PathBuf, - _mode: ImportMode, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(TempTag, u64)> { - let this = self.clone(); - tokio::task::spawn_blocking(move || { - let id = progress.new_id(); - progress.blocking_send(ImportProgress::Found { - id, - name: path.to_string_lossy().to_string(), - })?; - progress.try_send(ImportProgress::CopyProgress { id, offset: 0 })?; - // todo: provide progress for reading into mem - let bytes: Bytes = std::fs::read(path)?.into(); - let size = bytes.len() as u64; - progress.blocking_send(ImportProgress::Size { id, size })?; - let tag = this.import_bytes_sync(id, bytes, format, progress)?; - Ok((tag, size)) - }) - .await? - } - - async fn import_stream( - &self, - mut data: impl Stream> + Unpin + Send + 'static, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(TempTag, u64)> { - let this = self.clone(); - let id = progress.new_id(); - let name = temp_name(); - progress.send(ImportProgress::Found { id, name }).await?; - let mut bytes = BytesMut::new(); - while let Some(chunk) = data.next().await { - bytes.extend_from_slice(&chunk?); - progress - .try_send(ImportProgress::CopyProgress { - id, - offset: bytes.len() as u64, - }) - .ok(); - } - let bytes = bytes.freeze(); - let size = bytes.len() as u64; - progress.blocking_send(ImportProgress::Size { id, size })?; - let tag = this.import_bytes_sync(id, bytes, format, progress)?; - Ok((tag, size)) - } - - async fn import_bytes(&self, bytes: Bytes, format: BlobFormat) -> io::Result { - let this = self.clone(); - tokio::task::spawn_blocking(move || { - this.import_bytes_sync(0, bytes, format, IgnoreProgressSender::default()) - }) - .await? - } - - async fn set_tag(&self, name: Tag, value: Option) -> io::Result<()> { - let mut state = self.write_lock(); - if let Some(value) = value { - state.tags.insert(name, value); - } else { - state.tags.remove(&name); - } - Ok(()) - } - - async fn create_tag(&self, hash: HashAndFormat) -> io::Result { - let mut state = self.write_lock(); - let tag = Tag::auto(SystemTime::now(), |x| state.tags.contains_key(x)); - state.tags.insert(tag.clone(), hash); - Ok(tag) - } - - fn temp_tag(&self, tag: HashAndFormat) -> TempTag { - self.inner.temp_tag(tag) - } - - async fn gc_run(&self, config: super::GcConfig, protected_cb: G) - where - G: Fn() -> Gut, - Gut: Future> + Send, - { - super::gc_run_loop(self, config, move || async { Ok(()) }, protected_cb).await - } - - async fn delete(&self, hashes: Vec) -> io::Result<()> { - let mut state = self.write_lock(); - for hash in hashes { - if !state.temp.contains(&hash) { - state.entries.remove(&hash); - } - } - Ok(()) - } - - async fn shutdown(&self) {} - - async fn sync(&self) -> io::Result<()> { - Ok(()) - } -} - -#[derive(Debug, Default)] -struct StateInner { - entries: BTreeMap, - tags: BTreeMap, - temp: TempCounterMap, -} - -/// An in memory entry -#[derive(Debug, Clone)] -pub struct Entry { - inner: Arc, - complete: bool, -} - -#[derive(Debug)] -struct EntryInner { - hash: Hash, - data: RwLock, -} - -impl MapEntry for Entry { - fn hash(&self) -> Hash { - self.inner.hash - } - - fn size(&self) -> BaoBlobSize { - let size = self.inner.data.read().unwrap().current_size(); - BaoBlobSize::new(size, self.complete) - } - - fn is_complete(&self) -> bool { - self.complete - } - - async fn outboard(&self) -> io::Result { - let size = self.inner.data.read().unwrap().current_size(); - Ok(PreOrderOutboard { - root: self.hash().into(), - tree: BaoTree::new(size, IROH_BLOCK_SIZE), - data: OutboardReader(self.inner.clone()), - }) - } - - async fn data_reader(&self) -> io::Result { - Ok(DataReader(self.inner.clone())) - } -} - -impl MapEntryMut for Entry { - async fn batch_writer(&self) -> io::Result { - Ok(BatchWriter(self.inner.clone())) - } -} - -struct DataReader(Arc); - -impl AsyncSliceReader for DataReader { - async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result { - Ok(self.0.data.read().unwrap().read_data_at(offset, len)) - } - - async fn size(&mut self) -> std::io::Result { - Ok(self.0.data.read().unwrap().data_len()) - } -} - -struct OutboardReader(Arc); - -impl AsyncSliceReader for OutboardReader { - async fn read_at(&mut self, offset: u64, len: usize) -> std::io::Result { - Ok(self.0.data.read().unwrap().read_outboard_at(offset, len)) - } - - async fn size(&mut self) -> std::io::Result { - Ok(self.0.data.read().unwrap().outboard_len()) - } -} - -struct BatchWriter(Arc); - -impl super::BaoBatchWriter for BatchWriter { - async fn write_batch( - &mut self, - size: u64, - batch: Vec, - ) -> io::Result<()> { - self.0.data.write().unwrap().write_batch(size, &batch) - } - - async fn sync(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl super::Map for Store { - type Entry = Entry; - - async fn get(&self, hash: &Hash) -> std::io::Result> { - Ok(self.inner.0.read().unwrap().entries.get(hash).cloned()) - } -} - -impl super::MapMut for Store { - type EntryMut = Entry; - - async fn get_mut(&self, hash: &Hash) -> std::io::Result> { - self.get(hash).await - } - - async fn get_or_create(&self, hash: Hash, _size: u64) -> std::io::Result { - let entry = Entry { - inner: Arc::new(EntryInner { - hash, - data: RwLock::new(MutableMemStorage::default()), - }), - complete: false, - }; - Ok(entry) - } - - async fn entry_status(&self, hash: &Hash) -> std::io::Result { - self.entry_status_sync(hash) - } - - fn entry_status_sync(&self, hash: &Hash) -> std::io::Result { - Ok(match self.inner.0.read().unwrap().entries.get(hash) { - Some(entry) => { - if entry.complete { - crate::store::EntryStatus::Complete - } else { - crate::store::EntryStatus::Partial - } - } - None => crate::store::EntryStatus::NotFound, - }) - } - - async fn insert_complete(&self, mut entry: Entry) -> std::io::Result<()> { - let hash = entry.hash(); - let mut inner = self.inner.0.write().unwrap(); - let complete = inner - .entries - .get(&hash) - .map(|x| x.complete) - .unwrap_or_default(); - if !complete { - entry.complete = true; - inner.entries.insert(hash, entry); - } - Ok(()) - } -} - -impl ReadableStore for Store { - async fn blobs(&self) -> io::Result> { - let entries = self.read_lock().entries.clone(); - Ok(Box::new( - entries - .into_values() - .filter(|x| x.complete) - .map(|x| Ok(x.hash())), - )) - } - - async fn partial_blobs(&self) -> io::Result> { - let entries = self.read_lock().entries.clone(); - Ok(Box::new( - entries - .into_values() - .filter(|x| !x.complete) - .map(|x| Ok(x.hash())), - )) - } - - async fn tags( - &self, - ) -> io::Result> { - #[allow(clippy::mutable_key_type)] - let tags = self.read_lock().tags.clone(); - Ok(Box::new(tags.into_iter().map(Ok))) - } - - fn temp_tags( - &self, - ) -> Box + Send + Sync + 'static> { - let tags = self.read_lock().temp.keys(); - Box::new(tags) - } - - async fn consistency_check( - &self, - _repair: bool, - _tx: BoxedProgressSender, - ) -> io::Result<()> { - todo!() - } - - async fn export( - &self, - hash: Hash, - target: std::path::PathBuf, - mode: crate::store::ExportMode, - progress: ExportProgressCb, - ) -> io::Result<()> { - let this = self.clone(); - tokio::task::spawn_blocking(move || this.export_sync(hash, target, mode, progress)).await? - } -} diff --git a/iroh-blobs/src/store/mutable_mem_storage.rs b/iroh-blobs/src/store/mutable_mem_storage.rs deleted file mode 100644 index 1bc4b184ac7..00000000000 --- a/iroh-blobs/src/store/mutable_mem_storage.rs +++ /dev/null @@ -1,131 +0,0 @@ -use bao_tree::{ - io::{fsm::BaoContentItem, sync::WriteAt}, - BaoTree, -}; -use bytes::Bytes; - -use crate::{ - util::{compute_outboard, copy_limited_slice, SparseMemFile}, - IROH_BLOCK_SIZE, -}; - -/// Mutable in memory storage for a bao file. -/// -/// This is used for incomplete files if they are not big enough to warrant -/// writing to disk. We must keep track of ranges in both data and outboard -/// that have been written to, and track the most precise known size. -#[derive(Debug, Default)] -pub struct MutableMemStorage { - /// Data file, can be any size. - pub data: SparseMemFile, - /// Outboard file, must be a multiple of 64 bytes. - pub outboard: SparseMemFile, - /// Size that was announced as we wrote that chunk - pub sizes: SizeInfo, -} - -/// Keep track of the most precise size we know of. -/// -/// When in memory, we don't have to write the size for every chunk to a separate -/// slot, but can just keep the best one. -#[derive(Debug, Default)] -pub struct SizeInfo { - pub offset: u64, - pub size: u64, -} - -impl SizeInfo { - /// Create a new size info for a complete file of size `size`. - pub(crate) fn complete(size: u64) -> Self { - let mask = (1 << IROH_BLOCK_SIZE.chunk_log()) - 1; - // offset of the last bao chunk in a file of size `size` - let last_chunk_offset = size & mask; - Self { - offset: last_chunk_offset, - size, - } - } - - /// Write a size at the given offset. The size at the highest offset is going to be kept. - fn write(&mut self, offset: u64, size: u64) { - // >= instead of > because we want to be able to update size 0, the initial value. - if offset >= self.offset { - self.offset = offset; - self.size = size; - } - } - - /// The current size, representing the most correct size we know. - pub fn current_size(&self) -> u64 { - self.size - } -} - -impl MutableMemStorage { - /// Create a new mutable mem storage from the given data - pub fn complete( - bytes: Bytes, - cb: impl Fn(u64) + Send + Sync + 'static, - ) -> (Self, iroh_base::hash::Hash) { - let (hash, outboard) = compute_outboard(&bytes[..], bytes.len() as u64, move |offset| { - cb(offset); - Ok(()) - }) - .unwrap(); - let outboard = outboard.unwrap_or_default(); - let res = Self { - data: bytes.to_vec().into(), - outboard: outboard.into(), - sizes: SizeInfo::complete(bytes.len() as u64), - }; - (res, hash) - } - - pub(super) fn current_size(&self) -> u64 { - self.sizes.current_size() - } - - pub(super) fn read_data_at(&self, offset: u64, len: usize) -> Bytes { - copy_limited_slice(&self.data, offset, len) - } - - pub(super) fn data_len(&self) -> u64 { - self.data.len() as u64 - } - - pub(super) fn read_outboard_at(&self, offset: u64, len: usize) -> Bytes { - copy_limited_slice(&self.outboard, offset, len) - } - - pub(super) fn outboard_len(&self) -> u64 { - self.outboard.len() as u64 - } - - pub(super) fn write_batch( - &mut self, - size: u64, - batch: &[BaoContentItem], - ) -> std::io::Result<()> { - let tree = BaoTree::new(size, IROH_BLOCK_SIZE); - for item in batch { - match item { - BaoContentItem::Parent(parent) => { - if let Some(offset) = tree.pre_order_offset(parent.node) { - let o0 = offset - .checked_mul(64) - .expect("u64 overflow multiplying to hash pair offset"); - let o1 = o0.checked_add(32).expect("u64 overflow"); - let outboard = &mut self.outboard; - outboard.write_all_at(o0, parent.pair.0.as_bytes().as_slice())?; - outboard.write_all_at(o1, parent.pair.1.as_bytes().as_slice())?; - } - } - BaoContentItem::Leaf(leaf) => { - self.sizes.write(leaf.offset, size); - self.data.write_all_at(leaf.offset, leaf.data.as_ref())?; - } - } - } - Ok(()) - } -} diff --git a/iroh-blobs/src/store/readonly_mem.rs b/iroh-blobs/src/store/readonly_mem.rs deleted file mode 100644 index a041615549b..00000000000 --- a/iroh-blobs/src/store/readonly_mem.rs +++ /dev/null @@ -1,345 +0,0 @@ -//! A readonly in memory database for iroh-blobs, usable for testing and sharing static data. -//! -//! Main entry point is [Store]. -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - future::Future, - io, - path::PathBuf, - sync::Arc, -}; - -use bao_tree::{ - blake3, - io::{outboard::PreOrderMemOutboard, sync::Outboard}, -}; -use bytes::Bytes; -use futures_lite::Stream; -use iroh_io::AsyncSliceReader; -use tokio::io::AsyncWriteExt; - -use super::{BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, DbIter, ExportProgressCb}; -use crate::{ - store::{ - EntryStatus, ExportMode, ImportMode, ImportProgress, Map, MapEntry, MapEntryMut, - ReadableStore, - }, - util::{ - progress::{BoxedProgressSender, IdGenerator, ProgressSender}, - Tag, - }, - BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE, -}; - -/// A readonly in memory database for iroh-blobs. -/// -/// This is basically just a HashMap, so it does not allow for any modifications -/// unless you have a mutable reference to it. -/// -/// It is therefore useful mostly for testing and sharing static data. -#[derive(Debug, Clone, Default)] -pub struct Store(Arc, Bytes)>>); - -impl FromIterator<(K, V)> for Store -where - K: Into, - V: AsRef<[u8]>, -{ - fn from_iter>(iter: T) -> Self { - let (db, _m) = Self::new(iter); - db - } -} - -impl Store { - /// Create a new [Store] from a sequence of entries. - /// - /// Returns the database and a map of names to computed blake3 hashes. - /// In case of duplicate names, the last entry is used. - pub fn new( - entries: impl IntoIterator, impl AsRef<[u8]>)>, - ) -> (Self, BTreeMap) { - let mut names = BTreeMap::new(); - let mut res = HashMap::new(); - for (name, data) in entries.into_iter() { - let name = name.into(); - let data: &[u8] = data.as_ref(); - // wrap into the right types - let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE).map_data(Bytes::from); - let hash = outboard.root(); - // add the name, this assumes that names are unique - names.insert(name, hash); - let data = Bytes::from(data.to_vec()); - let hash = Hash::from(hash); - res.insert(hash, (outboard, data)); - } - (Self(Arc::new(res)), names) - } - - /// Insert a new entry into the database, and return the hash of the entry. - /// - /// If the database was shared before, this will make a copy. - pub fn insert(&mut self, data: impl AsRef<[u8]>) -> Hash { - let inner = Arc::make_mut(&mut self.0); - let data: &[u8] = data.as_ref(); - // wrap into the right types - let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE).map_data(Bytes::from); - let hash = outboard.root(); - let data = Bytes::from(data.to_vec()); - let hash = Hash::from(hash); - inner.insert(hash, (outboard, data)); - hash - } - - /// Insert multiple entries into the database, and return the hash of the last entry. - pub fn insert_many( - &mut self, - items: impl IntoIterator>, - ) -> Option { - let mut hash = None; - for item in items.into_iter() { - hash = Some(self.insert(item)); - } - hash - } - - /// Get the bytes associated with a hash, if they exist. - pub fn get_content(&self, hash: &Hash) -> Option { - let entry = self.0.get(hash)?; - Some(entry.1.clone()) - } - - async fn export_impl( - &self, - hash: Hash, - target: PathBuf, - _mode: ExportMode, - progress: impl Fn(u64) -> io::Result<()> + Send + Sync + 'static, - ) -> io::Result<()> { - tracing::trace!("exporting {} to {}", hash, target.display()); - - if !target.is_absolute() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "target path must be absolute", - )); - } - let parent = target.parent().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "target path has no parent directory", - ) - })?; - // create the directory in which the target file is - tokio::fs::create_dir_all(parent).await?; - let data = self - .get_content(&hash) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "hash not found"))?; - - let mut offset = 0u64; - let mut file = tokio::fs::File::create(&target).await?; - for chunk in data.chunks(1024 * 1024) { - progress(offset)?; - file.write_all(chunk).await?; - offset += chunk.len() as u64; - } - file.sync_all().await?; - drop(file); - Ok(()) - } -} - -/// The [MapEntry] implementation for [Store]. -#[derive(Debug, Clone)] -pub struct Entry { - outboard: PreOrderMemOutboard, - data: Bytes, -} - -impl MapEntry for Entry { - fn hash(&self) -> Hash { - self.outboard.root().into() - } - - fn size(&self) -> BaoBlobSize { - BaoBlobSize::Verified(self.data.len() as u64) - } - - async fn outboard(&self) -> io::Result { - Ok(self.outboard.clone()) - } - - async fn data_reader(&self) -> io::Result { - Ok(self.data.clone()) - } - - fn is_complete(&self) -> bool { - true - } -} - -impl Map for Store { - type Entry = Entry; - - async fn get(&self, hash: &Hash) -> io::Result> { - Ok(self.0.get(hash).map(|(o, d)| Entry { - outboard: o.clone(), - data: d.clone(), - })) - } -} - -impl super::MapMut for Store { - type EntryMut = Entry; - - async fn get_mut(&self, hash: &Hash) -> io::Result> { - self.get(hash).await - } - - async fn get_or_create(&self, _hash: Hash, _size: u64) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Other, - "cannot create temp entry in readonly database", - )) - } - - fn entry_status_sync(&self, hash: &Hash) -> io::Result { - Ok(match self.0.contains_key(hash) { - true => EntryStatus::Complete, - false => EntryStatus::NotFound, - }) - } - - async fn entry_status(&self, hash: &Hash) -> io::Result { - self.entry_status_sync(hash) - } - - async fn insert_complete(&self, _entry: Entry) -> io::Result<()> { - // this is unreachable, since we cannot create partial entries - unreachable!() - } -} - -impl ReadableStore for Store { - async fn blobs(&self) -> io::Result> { - Ok(Box::new( - self.0 - .keys() - .copied() - .map(Ok) - .collect::>() - .into_iter(), - )) - } - - async fn tags(&self) -> io::Result> { - Ok(Box::new(std::iter::empty())) - } - - fn temp_tags(&self) -> Box + Send + Sync + 'static> { - Box::new(std::iter::empty()) - } - - async fn consistency_check( - &self, - _repair: bool, - _tx: BoxedProgressSender, - ) -> io::Result<()> { - Ok(()) - } - - async fn export( - &self, - hash: Hash, - target: PathBuf, - mode: ExportMode, - progress: ExportProgressCb, - ) -> io::Result<()> { - self.export_impl(hash, target, mode, progress).await - } - - async fn partial_blobs(&self) -> io::Result> { - Ok(Box::new(std::iter::empty())) - } -} - -impl MapEntryMut for Entry { - async fn batch_writer(&self) -> io::Result { - enum Bar {} - impl BaoBatchWriter for Bar { - async fn write_batch( - &mut self, - _size: u64, - _batch: Vec, - ) -> io::Result<()> { - unreachable!() - } - - async fn sync(&mut self) -> io::Result<()> { - unreachable!() - } - } - - #[allow(unreachable_code)] - Ok(unreachable!() as Bar) - } -} - -impl super::Store for Store { - async fn import_file( - &self, - data: PathBuf, - mode: ImportMode, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(TempTag, u64)> { - let _ = (data, mode, progress, format); - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - /// import a byte slice - async fn import_bytes(&self, bytes: Bytes, format: BlobFormat) -> io::Result { - let _ = (bytes, format); - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - async fn import_stream( - &self, - data: impl Stream> + Unpin + Send, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> io::Result<(TempTag, u64)> { - let _ = (data, format, progress); - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - async fn set_tag(&self, _name: Tag, _hash: Option) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - async fn create_tag(&self, _hash: HashAndFormat) -> io::Result { - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - fn temp_tag(&self, inner: HashAndFormat) -> TempTag { - TempTag::new(inner, None) - } - - async fn gc_run(&self, config: super::GcConfig, protected_cb: G) - where - G: Fn() -> Gut, - Gut: Future> + Send, - { - super::gc_run_loop(self, config, move || async { Ok(()) }, protected_cb).await - } - - async fn delete(&self, _hashes: Vec) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Other, "not implemented")) - } - - async fn shutdown(&self) {} - - async fn sync(&self) -> io::Result<()> { - Ok(()) - } -} diff --git a/iroh-blobs/src/store/traits.rs b/iroh-blobs/src/store/traits.rs deleted file mode 100644 index 816dc8c2c50..00000000000 --- a/iroh-blobs/src/store/traits.rs +++ /dev/null @@ -1,1063 +0,0 @@ -//! Traits for in-memory or persistent maps of blob with bao encoded outboards. -use std::{collections::BTreeSet, future::Future, io, path::PathBuf, time::Duration}; - -pub use bao_tree; -use bao_tree::{ - io::{ - fsm::{ - encode_ranges_validated, BaoContentItem, Outboard, ResponseDecoder, ResponseDecoderNext, - }, - DecodeError, - }, - BaoTree, ChunkRanges, -}; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use genawaiter::rc::{Co, Gen}; -use iroh_base::rpc::RpcError; -use iroh_io::{AsyncSliceReader, AsyncStreamReader, AsyncStreamWriter}; -pub use range_collections; -use serde::{Deserialize, Serialize}; -use tokio::io::AsyncRead; - -use crate::{ - hashseq::parse_hash_seq, - protocol::RangeSpec, - util::{ - local_pool::{self, LocalPool}, - progress::{BoxedProgressSender, IdGenerator, ProgressSender}, - Tag, - }, - BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE, -}; - -/// A fallible but owned iterator over the entries in a store. -pub type DbIter = Box> + Send + Sync + 'static>; - -/// Export trogress callback -pub type ExportProgressCb = Box io::Result<()> + Send + Sync + 'static>; - -/// The availability status of an entry in a store. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum EntryStatus { - /// The entry is completely available. - Complete, - /// The entry is partially available. - Partial, - /// The entry is not in the store. - NotFound, -} - -/// The size of a bao file -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)] -pub enum BaoBlobSize { - /// A remote side told us the size, but we have insufficient data to verify it. - Unverified(u64), - /// We have verified the size. - Verified(u64), -} - -impl BaoBlobSize { - /// Create a new `BaoFileSize` with the given size and verification status. - pub fn new(size: u64, verified: bool) -> Self { - if verified { - BaoBlobSize::Verified(size) - } else { - BaoBlobSize::Unverified(size) - } - } - - /// Get just the value, no matter if it is verified or not. - pub fn value(&self) -> u64 { - match self { - BaoBlobSize::Unverified(size) => *size, - BaoBlobSize::Verified(size) => *size, - } - } -} - -/// An entry for one hash in a bao map -/// -/// The entry has the ability to provide you with an (outboard, data) -/// reader pair. Creating the reader is async and may fail. The futures that -/// create the readers must be `Send`, but the readers themselves don't have to -/// be. -pub trait MapEntry: std::fmt::Debug + Clone + Send + Sync + 'static { - /// The hash of the entry. - fn hash(&self) -> Hash; - /// The size of the entry. - fn size(&self) -> BaoBlobSize; - /// Returns `true` if the entry is complete. - /// - /// Note that this does not actually verify if the bytes on disk are complete, - /// it only checks if the entry was marked as complete in the store. - fn is_complete(&self) -> bool; - /// A future that resolves to a reader that can be used to read the outboard - fn outboard(&self) -> impl Future> + Send; - /// A future that resolves to a reader that can be used to read the data - fn data_reader(&self) -> impl Future> + Send; - - /// Encodes data and outboard into a [`AsyncStreamWriter`]. - /// - /// Data and outboard parts will be interleaved. - /// - /// `offset` is the byte offset in the blob to start the stream from. It will be rounded down to - /// the next chunk group. - /// - /// Returns immediately without error if `start` is equal or larger than the entry's size. - fn write_verifiable_stream<'a>( - &'a self, - offset: u64, - writer: impl AsyncStreamWriter + 'a, - ) -> impl Future> + 'a { - async move { - let size = self.size().value(); - if offset >= size { - return Ok(()); - } - let ranges = range_from_offset_and_length(offset, size - offset); - let (outboard, data) = tokio::try_join!(self.outboard(), self.data_reader())?; - encode_ranges_validated(data, outboard, &ranges, writer).await?; - Ok(()) - } - } -} - -/// A generic map from hashes to bao blobs (blobs with bao outboards). -/// -/// This is the readonly view. To allow updates, a concrete implementation must -/// also implement [`MapMut`]. -/// -/// Entries are *not* guaranteed to be complete for all implementations. -/// They are also not guaranteed to be immutable, since this could be the -/// readonly view of a mutable store. -pub trait Map: Clone + Send + Sync + 'static { - /// The entry type. An entry is a cheaply cloneable handle that can be used - /// to open readers for both the data and the outboard - type Entry: MapEntry; - /// Get an entry for a hash. - /// - /// This can also be used for a membership test by just checking if there - /// is an entry. Creating an entry should be cheap, any expensive ops should - /// be deferred to the creation of the actual readers. - /// - /// It is not guaranteed that the entry is complete. - fn get(&self, hash: &Hash) -> impl Future>> + Send; -} - -/// A partial entry -pub trait MapEntryMut: MapEntry { - /// Get a batch writer - fn batch_writer(&self) -> impl Future> + Send; -} - -/// An async batch interface for writing bao content items to a pair of data and -/// outboard. -/// -/// Details like the chunk group size and the actual storage location are left -/// to the implementation. -pub trait BaoBatchWriter { - /// Write a batch of bao content items to the underlying storage. - /// - /// The batch is guaranteed to be sorted as data is received from the network. - /// So leaves will be sorted by offset, and parents will be sorted by pre order - /// traversal offset. There is no guarantee that they will be consecutive - /// though. - /// - /// The size is the total size of the blob that the remote side told us. - /// It is not guaranteed to be correct, but it is guaranteed to be - /// consistent with all data in the batch. The size therefore represents - /// an upper bound on the maximum offset of all leaf items. - /// So it is guaranteed that `leaf.offset + leaf.size <= size` for all - /// leaf items in the batch. - /// - /// Batches should not become too large. Typically, a batch is just a few - /// parent nodes and a leaf. - /// - /// Batch is a vec so it can be moved into a task, which is unfortunately - /// necessary in typical io code. - fn write_batch( - &mut self, - size: u64, - batch: Vec, - ) -> impl Future>; - - /// Sync the written data to permanent storage, if applicable. - /// E.g. for a file based implementation, this would call sync_data - /// on all files. - fn sync(&mut self) -> impl Future>; -} - -/// Implement BaoBatchWriter for mutable references -impl BaoBatchWriter for &mut W { - async fn write_batch(&mut self, size: u64, batch: Vec) -> io::Result<()> { - (**self).write_batch(size, batch).await - } - - async fn sync(&mut self) -> io::Result<()> { - (**self).sync().await - } -} - -/// A wrapper around a batch writer that calls a progress callback for one leaf -/// per batch. -#[derive(Debug)] -pub(crate) struct FallibleProgressBatchWriter(W, F); - -impl io::Result<()> + 'static> - FallibleProgressBatchWriter -{ - /// Create a new `FallibleProgressBatchWriter` from an inner writer and a progress callback - /// - /// The `on_write` function is called for each write, with the `offset` as the first and the - /// length of the data as the second param. `on_write` must return an `io::Result`. - /// If `on_write` returns an error, the download is aborted. - pub fn new(inner: W, on_write: F) -> Self { - Self(inner, on_write) - } -} - -impl io::Result<()> + 'static> BaoBatchWriter - for FallibleProgressBatchWriter -{ - async fn write_batch(&mut self, size: u64, batch: Vec) -> io::Result<()> { - // find the offset and length of the first (usually only) chunk - let chunk = batch - .iter() - .filter_map(|item| { - if let BaoContentItem::Leaf(leaf) = item { - Some((leaf.offset, leaf.data.len())) - } else { - None - } - }) - .next(); - self.0.write_batch(size, batch).await?; - // call the progress callback - if let Some((offset, len)) = chunk { - (self.1)(offset, len)?; - } - Ok(()) - } - - async fn sync(&mut self) -> io::Result<()> { - self.0.sync().await - } -} - -/// A mutable bao map. -/// -/// This extends the readonly [`Map`] trait with methods to create and modify entries. -pub trait MapMut: Map { - /// An entry that is possibly writable - type EntryMut: MapEntryMut; - - /// Get an existing entry as an EntryMut. - /// - /// For implementations where EntryMut and Entry are the same type, this is just an alias for - /// `get`. - fn get_mut( - &self, - hash: &Hash, - ) -> impl Future>> + Send; - - /// Get an existing partial entry, or create a new one. - /// - /// We need to know the size of the partial entry. This might produce an - /// error e.g. if there is not enough space on disk. - fn get_or_create( - &self, - hash: Hash, - size: u64, - ) -> impl Future> + Send; - - /// Find out if the data behind a `hash` is complete, partial, or not present. - /// - /// Note that this does not actually verify the on-disc data, but only checks in which section - /// of the store the entry is present. - fn entry_status(&self, hash: &Hash) -> impl Future> + Send; - - /// Sync version of `entry_status`, for the doc sync engine until we can get rid of it. - /// - /// Don't count on this to be efficient. - fn entry_status_sync(&self, hash: &Hash) -> io::Result; - - /// Upgrade a partial entry to a complete entry. - fn insert_complete(&self, entry: Self::EntryMut) - -> impl Future> + Send; -} - -/// Extension of [`Map`] to add misc methods used by the rpc calls. -pub trait ReadableStore: Map { - /// list all blobs in the database. This includes both raw blobs that have - /// been imported, and hash sequences that have been created internally. - fn blobs(&self) -> impl Future>> + Send; - /// list all tags (collections or other explicitly added things) in the database - fn tags(&self) -> impl Future>> + Send; - - /// Temp tags - fn temp_tags(&self) -> Box + Send + Sync + 'static>; - - /// Perform a consistency check on the database - fn consistency_check( - &self, - repair: bool, - tx: BoxedProgressSender, - ) -> impl Future> + Send; - - /// list partial blobs in the database - fn partial_blobs(&self) -> impl Future>> + Send; - - /// This trait method extracts a file to a local path. - /// - /// `hash` is the hash of the file - /// `target` is the path to the target file - /// `mode` is a hint how the file should be exported. - /// `progress` is a callback that is called with the total number of bytes that have been written - fn export( - &self, - hash: Hash, - target: PathBuf, - mode: ExportMode, - progress: ExportProgressCb, - ) -> impl Future> + Send; -} - -/// The mutable part of a Bao store. -pub trait Store: ReadableStore + MapMut + std::fmt::Debug { - /// This trait method imports a file from a local path. - /// - /// `data` is the path to the file. - /// `mode` is a hint how the file should be imported. - /// `progress` is a sender that provides a way for the importer to send progress messages - /// when importing large files. This also serves as a way to cancel the import. If the - /// consumer of the progress messages is dropped, subsequent attempts to send progress - /// will fail. - /// - /// Returns the hash of the imported file. The reason to have this method is that some database - /// implementations might be able to import a file without copying it. - fn import_file( - &self, - data: PathBuf, - mode: ImportMode, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> impl Future> + Send; - - /// Import data from memory. - /// - /// It is a special case of `import` that does not use the file system. - fn import_bytes( - &self, - bytes: Bytes, - format: BlobFormat, - ) -> impl Future> + Send; - - /// Import data from a stream of bytes. - fn import_stream( - &self, - data: impl Stream> + Send + Unpin + 'static, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> impl Future> + Send; - - /// Import data from an async byte reader. - fn import_reader( - &self, - data: impl AsyncRead + Send + Unpin + 'static, - format: BlobFormat, - progress: impl ProgressSender + IdGenerator, - ) -> impl Future> + Send { - let stream = tokio_util::io::ReaderStream::new(data); - self.import_stream(stream, format, progress) - } - - /// Import a blob from a verified stream, as emitted by [`MapEntry::write_verifiable_stream`]; - /// - /// `total_size` is the total size of the blob as reported by the remote. - /// `offset` is the byte offset in the blob where the stream starts. It will be rounded - /// to the next chunk group. - fn import_verifiable_stream<'a>( - &'a self, - hash: Hash, - total_size: u64, - offset: u64, - reader: impl AsyncStreamReader + 'a, - ) -> impl Future> + 'a { - async move { - if offset >= total_size { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "offset must not be greater than total_size", - )); - } - let entry = self.get_or_create(hash, total_size).await?; - let mut bw = entry.batch_writer().await?; - - let ranges = range_from_offset_and_length(offset, total_size - offset); - let mut decoder = ResponseDecoder::new( - hash.into(), - ranges, - BaoTree::new(total_size, IROH_BLOCK_SIZE), - reader, - ); - let size = decoder.tree().size(); - let mut buf = Vec::new(); - let is_complete = loop { - decoder = match decoder.next().await { - ResponseDecoderNext::More((decoder, item)) => { - let item = match item { - Err(DecodeError::LeafNotFound(_) | DecodeError::ParentNotFound(_)) => { - break false - } - Err(err) => return Err(err.into()), - Ok(item) => item, - }; - match &item { - BaoContentItem::Parent(_) => { - buf.push(item); - } - BaoContentItem::Leaf(_) => { - buf.push(item); - let batch = std::mem::take(&mut buf); - bw.write_batch(size, batch).await?; - } - } - decoder - } - ResponseDecoderNext::Done(_reader) => { - debug_assert!(buf.is_empty(), "last node of bao tree must be leaf node"); - break true; - } - }; - }; - bw.sync().await?; - drop(bw); - if is_complete { - self.insert_complete(entry).await?; - } - Ok(()) - } - } - - /// Set a tag - fn set_tag( - &self, - name: Tag, - hash: Option, - ) -> impl Future> + Send; - - /// Create a new tag - fn create_tag(&self, hash: HashAndFormat) -> impl Future> + Send; - - /// Create a temporary pin for this store - fn temp_tag(&self, value: HashAndFormat) -> TempTag; - - /// Start the GC loop - /// - /// The gc task will shut down, when dropping the returned future. - fn gc_run(&self, config: super::GcConfig, protected_cb: G) -> impl Future - where - G: Fn() -> Gut, - Gut: Future> + Send; - - /// physically delete the given hashes from the store. - fn delete(&self, hashes: Vec) -> impl Future> + Send; - - /// Shutdown the store. - fn shutdown(&self) -> impl Future + Send; - - /// Sync the store. - fn sync(&self) -> impl Future> + Send; - - /// Validate the database - /// - /// This will check that the file and outboard content is correct for all complete - /// entries, and output valid ranges for all partial entries. - /// - /// It will not check the internal consistency of the database. - fn validate( - &self, - repair: bool, - tx: BoxedProgressSender, - ) -> impl Future> + Send { - validate_impl(self, repair, tx) - } -} - -fn range_from_offset_and_length(offset: u64, length: u64) -> bao_tree::ChunkRanges { - let ranges = bao_tree::ByteRanges::from(offset..(offset + length)); - bao_tree::io::round_up_to_chunks(&ranges) -} - -async fn validate_impl( - store: &impl Store, - repair: bool, - tx: BoxedProgressSender, -) -> io::Result<()> { - use futures_buffered::BufferedStreamExt; - - let validate_parallelism: usize = num_cpus::get(); - let lp = LocalPool::new(local_pool::Config { - threads: validate_parallelism, - ..Default::default() - }); - let complete = store.blobs().await?.collect::>>()?; - let partial = store - .partial_blobs() - .await? - .collect::>>()?; - tx.send(ValidateProgress::Starting { - total: complete.len() as u64, - }) - .await?; - let complete_result = futures_lite::stream::iter(complete) - .map(|hash| { - let store = store.clone(); - let tx = tx.clone(); - lp.spawn(move || async move { - let entry = store - .get(&hash) - .await? - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "entry not found"))?; - let size = entry.size().value(); - let outboard = entry.outboard().await?; - let data = entry.data_reader().await?; - let chunk_ranges = ChunkRanges::all(); - let mut ranges = bao_tree::io::fsm::valid_ranges(outboard, data, &chunk_ranges); - let id = tx.new_id(); - tx.send(ValidateProgress::Entry { - id, - hash, - path: None, - size, - }) - .await?; - let mut actual_chunk_ranges = ChunkRanges::empty(); - while let Some(item) = ranges.next().await { - let item = item?; - let offset = item.start.to_bytes(); - actual_chunk_ranges |= ChunkRanges::from(item); - tx.try_send(ValidateProgress::EntryProgress { id, offset })?; - } - let expected_chunk_range = - ChunkRanges::from(..BaoTree::new(size, IROH_BLOCK_SIZE).chunks()); - let incomplete = actual_chunk_ranges == expected_chunk_range; - let error = if incomplete { - None - } else { - Some(format!( - "expected chunk ranges {:?}, got chunk ranges {:?}", - expected_chunk_range, actual_chunk_ranges - )) - }; - tx.send(ValidateProgress::EntryDone { id, error }).await?; - drop(ranges); - drop(entry); - io::Result::Ok((hash, incomplete)) - }) - }) - .buffered_unordered(validate_parallelism) - .collect::>() - .await; - let partial_result = futures_lite::stream::iter(partial) - .map(|hash| { - let store = store.clone(); - let tx = tx.clone(); - lp.spawn(move || async move { - let entry = store - .get(&hash) - .await? - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "entry not found"))?; - let size = entry.size().value(); - let outboard = entry.outboard().await?; - let data = entry.data_reader().await?; - let chunk_ranges = ChunkRanges::all(); - let mut ranges = bao_tree::io::fsm::valid_ranges(outboard, data, &chunk_ranges); - let id = tx.new_id(); - tx.send(ValidateProgress::PartialEntry { - id, - hash, - path: None, - size, - }) - .await?; - let mut actual_chunk_ranges = ChunkRanges::empty(); - while let Some(item) = ranges.next().await { - let item = item?; - let offset = item.start.to_bytes(); - actual_chunk_ranges |= ChunkRanges::from(item); - tx.try_send(ValidateProgress::PartialEntryProgress { id, offset })?; - } - tx.send(ValidateProgress::PartialEntryDone { - id, - ranges: RangeSpec::new(&actual_chunk_ranges), - }) - .await?; - drop(ranges); - drop(entry); - io::Result::Ok(()) - }) - }) - .buffered_unordered(validate_parallelism) - .collect::>() - .await; - let mut to_downgrade = Vec::new(); - for item in complete_result { - let (hash, incomplete) = item??; - if incomplete { - to_downgrade.push(hash); - } - } - for item in partial_result { - item??; - } - if repair { - return Err(io::Error::new( - io::ErrorKind::Other, - "repair not implemented", - )); - } - Ok(()) -} - -/// Configuration for the GC mark and sweep. -#[derive(derive_more::Debug)] -pub struct GcConfig { - /// The period at which to execute the GC. - pub period: Duration, - /// An optional callback called every time a GC round finishes. - #[debug("done_callback")] - pub done_callback: Option>, -} - -/// Implementation of the gc loop. -pub(super) async fn gc_run_loop( - store: &S, - config: GcConfig, - start_cb: F, - protected_cb: G, -) where - S: Store, - F: Fn() -> Fut, - Fut: Future> + Send, - G: Fn() -> Gut, - Gut: Future> + Send, -{ - tracing::info!("Starting GC task with interval {:?}", config.period); - let mut live = BTreeSet::new(); - 'outer: loop { - if let Err(cause) = start_cb().await { - tracing::debug!("unable to notify the db of GC start: {cause}. Shutting down GC loop."); - break; - } - // do delay before the two phases of GC - tokio::time::sleep(config.period).await; - tracing::debug!("Starting GC"); - live.clear(); - - let p = protected_cb().await; - live.extend(p); - - tracing::debug!("Starting GC mark phase"); - let live_ref = &mut live; - let mut stream = Gen::new(|co| async move { - if let Err(e) = gc_mark_task(store, live_ref, &co).await { - co.yield_(GcMarkEvent::Error(e)).await; - } - }); - while let Some(item) = stream.next().await { - match item { - GcMarkEvent::CustomDebug(text) => { - tracing::debug!("{}", text); - } - GcMarkEvent::CustomWarning(text, _) => { - tracing::warn!("{}", text); - } - GcMarkEvent::Error(err) => { - tracing::error!("Fatal error during GC mark {}", err); - continue 'outer; - } - } - } - drop(stream); - - tracing::debug!("Starting GC sweep phase"); - let live_ref = &live; - let mut stream = Gen::new(|co| async move { - if let Err(e) = gc_sweep_task(store, live_ref, &co).await { - co.yield_(GcSweepEvent::Error(e)).await; - } - }); - while let Some(item) = stream.next().await { - match item { - GcSweepEvent::CustomDebug(text) => { - tracing::debug!("{}", text); - } - GcSweepEvent::CustomWarning(text, _) => { - tracing::warn!("{}", text); - } - GcSweepEvent::Error(err) => { - tracing::error!("Fatal error during GC mark {}", err); - continue 'outer; - } - } - } - if let Some(ref cb) = config.done_callback { - cb(); - } - } -} - -/// Implementation of the gc method. -pub(super) async fn gc_mark_task<'a>( - store: &'a impl Store, - live: &'a mut BTreeSet, - co: &Co, -) -> anyhow::Result<()> { - macro_rules! debug { - ($($arg:tt)*) => { - co.yield_(GcMarkEvent::CustomDebug(format!($($arg)*))).await; - }; - } - macro_rules! warn { - ($($arg:tt)*) => { - co.yield_(GcMarkEvent::CustomWarning(format!($($arg)*), None)).await; - }; - } - let mut roots = BTreeSet::new(); - debug!("traversing tags"); - for item in store.tags().await? { - let (name, haf) = item?; - debug!("adding root {:?} {:?}", name, haf); - roots.insert(haf); - } - debug!("traversing temp roots"); - for haf in store.temp_tags() { - debug!("adding temp pin {:?}", haf); - roots.insert(haf); - } - for HashAndFormat { hash, format } in roots { - // we need to do this for all formats except raw - if live.insert(hash) && !format.is_raw() { - let Some(entry) = store.get(&hash).await? else { - warn!("gc: {} not found", hash); - continue; - }; - if !entry.is_complete() { - warn!("gc: {} is partial", hash); - continue; - } - let Ok(reader) = entry.data_reader().await else { - warn!("gc: {} creating data reader failed", hash); - continue; - }; - let Ok((mut stream, count)) = parse_hash_seq(reader).await else { - warn!("gc: {} parse failed", hash); - continue; - }; - debug!("parsed collection {} {:?}", hash, count); - loop { - let item = match stream.next().await { - Ok(Some(item)) => item, - Ok(None) => break, - Err(_err) => { - warn!("gc: {} parse failed", hash); - break; - } - }; - // if format != raw we would have to recurse here by adding this to current - live.insert(item); - } - } - } - debug!("gc mark done. found {} live blobs", live.len()); - Ok(()) -} - -async fn gc_sweep_task<'a>( - store: &'a impl Store, - live: &BTreeSet, - co: &Co, -) -> anyhow::Result<()> { - let blobs = store.blobs().await?.chain(store.partial_blobs().await?); - let mut count = 0; - let mut batch = Vec::new(); - for hash in blobs { - let hash = hash?; - if !live.contains(&hash) { - batch.push(hash); - count += 1; - } - if batch.len() >= 100 { - store.delete(batch.clone()).await?; - batch.clear(); - } - } - if !batch.is_empty() { - store.delete(batch).await?; - } - co.yield_(GcSweepEvent::CustomDebug(format!( - "deleted {} blobs", - count - ))) - .await; - Ok(()) -} - -/// An event related to GC -#[derive(Debug)] -pub enum GcMarkEvent { - /// A custom event (info) - CustomDebug(String), - /// A custom non critical error - CustomWarning(String, Option), - /// An unrecoverable error during GC - Error(anyhow::Error), -} - -/// An event related to GC -#[derive(Debug)] -pub enum GcSweepEvent { - /// A custom event (debug) - CustomDebug(String), - /// A custom non critical error - CustomWarning(String, Option), - /// An unrecoverable error during GC - Error(anyhow::Error), -} - -/// Progress messages for an import operation -/// -/// An import operation involves computing the outboard of a file, and then -/// either copying or moving the file into the database. -#[allow(missing_docs)] -#[derive(Debug)] -pub enum ImportProgress { - /// Found a path - /// - /// This will be the first message for an id - Found { id: u64, name: String }, - /// Progress when copying the file to the store - /// - /// This will be omitted if the store can use the file in place - /// - /// There will be multiple of these messages for an id - CopyProgress { id: u64, offset: u64 }, - /// Determined the size - /// - /// This will come after `Found` and zero or more `CopyProgress` messages. - /// For unstable files, determining the size will only be done once the file - /// is fully copied. - Size { id: u64, size: u64 }, - /// Progress when computing the outboard - /// - /// There will be multiple of these messages for an id - OutboardProgress { id: u64, offset: u64 }, - /// Done computing the outboard - /// - /// This comes after `Size` and zero or more `OutboardProgress` messages - OutboardDone { id: u64, hash: Hash }, -} - -/// The import mode describes how files will be imported. -/// -/// This is a hint to the import trait method. For some implementations, this -/// does not make any sense. E.g. an in memory implementation will always have -/// to copy the file into memory. Also, a disk based implementation might choose -/// to copy small files even if the mode is `Reference`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -pub enum ImportMode { - /// This mode will copy the file into the database before hashing. - /// - /// This is the safe default because the file can not be accidentally modified - /// after it has been imported. - #[default] - Copy, - /// This mode will try to reference the file in place and assume it is unchanged after import. - /// - /// This has a large performance and storage benefit, but it is less safe since - /// the file might be modified after it has been imported. - /// - /// Stores are allowed to ignore this mode and always copy the file, e.g. - /// if the file is very small or if the store does not support referencing files. - TryReference, -} -/// The import mode describes how files will be imported. -/// -/// This is a hint to the import trait method. For some implementations, this -/// does not make any sense. E.g. an in memory implementation will always have -/// to copy the file into memory. Also, a disk based implementation might choose -/// to copy small files even if the mode is `Reference`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)] -pub enum ExportMode { - /// This mode will copy the file to the target directory. - /// - /// This is the safe default because the file can not be accidentally modified - /// after it has been exported. - #[default] - Copy, - /// This mode will try to move the file to the target directory and then reference it from - /// the database. - /// - /// This has a large performance and storage benefit, but it is less safe since - /// the file might be modified in the target directory after it has been exported. - /// - /// Stores are allowed to ignore this mode and always copy the file, e.g. - /// if the file is very small or if the store does not support referencing files. - TryReference, -} - -/// The expected format of a hash being exported. -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub enum ExportFormat { - /// The hash refers to any blob and will be exported to a single file. - #[default] - Blob, - /// The hash refers to a [`crate::format::collection::Collection`] blob - /// and all children of the collection shall be exported to one file per child. - /// - /// If the blob can be parsed as a [`BlobFormat::HashSeq`], and the first child contains - /// collection metadata, all other children of the collection will be exported to - /// a file each, with their collection name treated as a relative path to the export - /// destination path. - /// - /// If the blob cannot be parsed as a collection, the operation will fail. - Collection, -} - -#[allow(missing_docs)] -#[derive(Debug)] -pub enum ExportProgress { - /// Starting to export to a file - /// - /// This will be the first message for an id - Start { - id: u64, - hash: Hash, - path: PathBuf, - stable: bool, - }, - /// Progress when copying the file to the target - /// - /// This will be omitted if the store can move the file or use copy on write - /// - /// There will be multiple of these messages for an id - Progress { id: u64, offset: u64 }, - /// Done exporting - Done { id: u64 }, -} - -/// Level for generic validation messages -#[derive( - Debug, Clone, Copy, derive_more::Display, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq, -)] -pub enum ReportLevel { - /// Very unimportant info messages - Trace, - /// Info messages - Info, - /// Warnings, something is not quite right - Warn, - /// Errors, something is very wrong - Error, -} - -/// Progress updates for the validate operation -#[derive(Debug, Serialize, Deserialize)] -pub enum ConsistencyCheckProgress { - /// Consistency check started - Start, - /// Consistency check update - Update { - /// The message - message: String, - /// The entry this message is about, if any - entry: Option, - /// The level of the message - level: ReportLevel, - }, - /// Consistency check ended - Done, - /// We got an error and need to abort. - Abort(RpcError), -} - -/// Progress updates for the validate operation -#[derive(Debug, Serialize, Deserialize)] -pub enum ValidateProgress { - /// started validating - Starting { - /// The total number of entries to validate - total: u64, - }, - /// We started validating a complete entry - Entry { - /// a new unique id for this entry - id: u64, - /// the hash of the entry - hash: Hash, - /// location of the entry. - /// - /// In case of a file, this is the path to the file. - /// Otherwise it might be an url or something else to uniquely identify the entry. - path: Option, - /// The size of the entry, in bytes. - size: u64, - }, - /// We got progress ingesting item `id`. - EntryProgress { - /// The unique id of the entry. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done with `id` - EntryDone { - /// The unique id of the entry. - id: u64, - /// An error if we failed to validate the entry. - error: Option, - }, - /// We started validating an entry - PartialEntry { - /// a new unique id for this entry - id: u64, - /// the hash of the entry - hash: Hash, - /// location of the entry. - /// - /// In case of a file, this is the path to the file. - /// Otherwise it might be an url or something else to uniquely identify the entry. - path: Option, - /// The best known size of the entry, in bytes. - size: u64, - }, - /// We got progress ingesting item `id`. - PartialEntryProgress { - /// The unique id of the entry. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done with `id` - PartialEntryDone { - /// The unique id of the entry. - id: u64, - /// Available ranges. - ranges: RangeSpec, - }, - /// We are done with the whole operation. - AllDone, - /// We got an error and need to abort. - Abort(RpcError), -} - -/// Database events -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Event { - /// A GC was started - GcStarted, - /// A GC was completed - GcCompleted, -} diff --git a/iroh-blobs/src/util.rs b/iroh-blobs/src/util.rs deleted file mode 100644 index 5fbe65d2fc6..00000000000 --- a/iroh-blobs/src/util.rs +++ /dev/null @@ -1,365 +0,0 @@ -//! Utility functions and types. -use std::{ - borrow::Borrow, - fmt, - io::{BufReader, Read}, - sync::{Arc, Weak}, - time::SystemTime, -}; - -use bao_tree::{io::outboard::PreOrderOutboard, BaoTree, ChunkRanges}; -use bytes::Bytes; -use derive_more::{Debug, Display, From, Into}; -use range_collections::range_set::RangeSetRange; -use serde::{Deserialize, Serialize}; - -use crate::{BlobFormat, Hash, HashAndFormat, IROH_BLOCK_SIZE}; - -pub mod io; -mod mem_or_file; -pub mod progress; -pub use mem_or_file::MemOrFile; -mod sparse_mem_file; -pub use sparse_mem_file::SparseMemFile; -pub mod local_pool; - -/// A tag -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, From, Into)] -pub struct Tag(pub Bytes); - -#[cfg(feature = "redb")] -mod redb_support { - use bytes::Bytes; - use redb::{Key as RedbKey, Value as RedbValue}; - - use super::Tag; - - impl RedbValue for Tag { - type SelfType<'a> = Self; - - type AsBytes<'a> = bytes::Bytes; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - Self(Bytes::copy_from_slice(data)) - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - value.0.clone() - } - - fn type_name() -> redb::TypeName { - redb::TypeName::new("Tag") - } - } - - impl RedbKey for Tag { - fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { - data1.cmp(data2) - } - } -} - -impl Borrow<[u8]> for Tag { - fn borrow(&self) -> &[u8] { - self.0.as_ref() - } -} - -impl From for Tag { - fn from(value: String) -> Self { - Self(Bytes::from(value)) - } -} - -impl From<&str> for Tag { - fn from(value: &str) -> Self { - Self(Bytes::from(value.to_owned())) - } -} - -impl Display for Tag { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let bytes = self.0.as_ref(); - match std::str::from_utf8(bytes) { - Ok(s) => write!(f, "\"{}\"", s), - Err(_) => write!(f, "{}", hex::encode(bytes)), - } - } -} - -struct DD(T); - -impl fmt::Debug for DD { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -impl Debug for Tag { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Tag").field(&DD(self)).finish() - } -} - -impl Tag { - /// Create a new tag that does not exist yet. - pub fn auto(time: SystemTime, exists: impl Fn(&[u8]) -> bool) -> Self { - let now = chrono::DateTime::::from(time); - let mut i = 0; - loop { - let mut text = format!("auto-{}", now.format("%Y-%m-%dT%H:%M:%S%.3fZ")); - if i != 0 { - text.push_str(&format!("-{}", i)); - } - if !exists(text.as_bytes()) { - return Self::from(text); - } - i += 1; - } - } -} - -/// Option for commands that allow setting a tag -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum SetTagOption { - /// A tag will be automatically generated - Auto, - /// The tag is explicitly named - Named(Tag), -} - -/// Trait used from temp tags to notify an abstract store that a temp tag is -/// being dropped. -pub trait TagDrop: std::fmt::Debug + Send + Sync + 'static { - /// Called on drop - fn on_drop(&self, inner: &HashAndFormat); -} - -/// A trait for things that can track liveness of blobs and collections. -/// -/// This trait works together with [TempTag] to keep track of the liveness of a -/// blob or collection. -/// -/// It is important to include the format in the liveness tracking, since -/// protecting a collection means protecting the blob and all its children, -/// whereas protecting a raw blob only protects the blob itself. -pub trait TagCounter: TagDrop + Sized { - /// Called on creation of a temp tag - fn on_create(&self, inner: &HashAndFormat); - - /// Get this as a weak reference for use in temp tags - fn as_weak(self: &Arc) -> Weak { - let on_drop: Arc = self.clone(); - Arc::downgrade(&on_drop) - } - - /// Create a new temp tag for the given hash and format - fn temp_tag(self: &Arc, inner: HashAndFormat) -> TempTag { - self.on_create(&inner); - TempTag::new(inner, Some(self.as_weak())) - } -} - -/// A hash and format pair that is protected from garbage collection. -/// -/// If format is raw, this will protect just the blob -/// If format is collection, this will protect the collection and all blobs in it -#[derive(Debug)] -pub struct TempTag { - /// The hash and format we are pinning - inner: HashAndFormat, - /// optional callback to call on drop - on_drop: Option>, -} - -impl TempTag { - /// Create a new temp tag for the given hash and format - /// - /// This should only be used by store implementations. - /// - /// The caller is responsible for increasing the refcount on creation and to - /// make sure that temp tags that are created between a mark phase and a sweep - /// phase are protected. - pub fn new(inner: HashAndFormat, on_drop: Option>) -> Self { - Self { inner, on_drop } - } - - /// The hash of the pinned item - pub fn inner(&self) -> &HashAndFormat { - &self.inner - } - - /// The hash of the pinned item - pub fn hash(&self) -> &Hash { - &self.inner.hash - } - - /// The format of the pinned item - pub fn format(&self) -> BlobFormat { - self.inner.format - } - - /// The hash and format of the pinned item - pub fn hash_and_format(&self) -> HashAndFormat { - self.inner - } - - /// Keep the item alive until the end of the process - pub fn leak(mut self) { - // set the liveness tracker to None, so that the refcount is not decreased - // during drop. This means that the refcount will never reach 0 and the - // item will not be gced until the end of the process. - self.on_drop = None; - } -} - -impl Drop for TempTag { - fn drop(&mut self) { - if let Some(on_drop) = self.on_drop.take() { - if let Some(on_drop) = on_drop.upgrade() { - on_drop.on_drop(&self.inner); - } - } - } -} - -/// Get the number of bytes given a set of chunk ranges and the total size. -/// -/// If some ranges are out of bounds, they will be clamped to the size. -pub fn total_bytes(ranges: ChunkRanges, size: u64) -> u64 { - ranges - .iter() - .map(|range| { - let (start, end) = match range { - RangeSetRange::Range(r) => { - (r.start.to_bytes().min(size), r.end.to_bytes().min(size)) - } - RangeSetRange::RangeFrom(range) => (range.start.to_bytes().min(size), size), - }; - end.saturating_sub(start) - }) - .reduce(u64::saturating_add) - .unwrap_or_default() -} - -/// A non-sendable marker type -#[derive(Debug)] -pub(crate) struct NonSend { - _marker: std::marker::PhantomData>, -} - -impl NonSend { - /// Create a new non-sendable marker. - #[allow(dead_code)] - pub const fn new() -> Self { - Self { - _marker: std::marker::PhantomData, - } - } -} - -/// copy a limited slice from a slice as a `Bytes`. -pub(crate) fn copy_limited_slice(bytes: &[u8], offset: u64, len: usize) -> Bytes { - bytes[limited_range(offset, len, bytes.len())] - .to_vec() - .into() -} - -pub(crate) fn limited_range(offset: u64, len: usize, buf_len: usize) -> std::ops::Range { - if offset < buf_len as u64 { - let start = offset as usize; - let end = start.saturating_add(len).min(buf_len); - start..end - } else { - 0..0 - } -} - -/// zero copy get a limited slice from a `Bytes` as a `Bytes`. -#[allow(dead_code)] -pub(crate) fn get_limited_slice(bytes: &Bytes, offset: u64, len: usize) -> Bytes { - bytes.slice(limited_range(offset, len, bytes.len())) -} - -/// Compute raw outboard size, without the size header. -#[allow(dead_code)] -pub(crate) fn raw_outboard_size(size: u64) -> u64 { - BaoTree::new(size, IROH_BLOCK_SIZE).outboard_size() -} - -/// Synchronously compute the outboard of a file, and return hash and outboard. -/// -/// It is assumed that the file is not modified while this is running. -/// -/// If it is modified while or after this is running, the outboard will be -/// invalid, so any attempt to compute a slice from it will fail. -/// -/// If the size of the file is changed while this is running, an error will be -/// returned. -/// -/// The computed outboard is without length prefix. -pub(crate) fn compute_outboard( - read: impl Read, - size: u64, - progress: impl Fn(u64) -> std::io::Result<()> + Send + Sync + 'static, -) -> std::io::Result<(Hash, Option>)> { - use bao_tree::io::sync::CreateOutboard; - - // wrap the reader in a progress reader, so we can report progress. - let reader = ProgressReader::new(read, progress); - // wrap the reader in a buffered reader, so we read in large chunks - // this reduces the number of io ops and also the number of progress reports - let buf_size = usize::try_from(size).unwrap_or(usize::MAX).min(1024 * 1024); - let reader = BufReader::with_capacity(buf_size, reader); - - let ob = PreOrderOutboard::>::create_sized(reader, size, IROH_BLOCK_SIZE)?; - let root = ob.root.into(); - let data = ob.data; - tracing::trace!(%root, "done"); - let data = if !data.is_empty() { Some(data) } else { None }; - Ok((root, data)) -} - -/// Compute raw outboard, without the size header. -#[cfg(test)] -pub(crate) fn raw_outboard(data: &[u8]) -> (Vec, Hash) { - let res = bao_tree::io::outboard::PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); - (res.data, res.root.into()) -} - -/// A reader that calls a callback with the number of bytes read after each read. -pub(crate) struct ProgressReader std::io::Result<()>> { - inner: R, - offset: u64, - cb: F, -} - -impl std::io::Result<()>> ProgressReader { - pub fn new(inner: R, cb: F) -> Self { - Self { - inner, - offset: 0, - cb, - } - } -} - -impl std::io::Result<()>> std::io::Read for ProgressReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let read = self.inner.read(buf)?; - self.offset += read as u64; - (self.cb)(self.offset)?; - Ok(read) - } -} diff --git a/iroh-blobs/src/util/io.rs b/iroh-blobs/src/util/io.rs deleted file mode 100644 index fa75f4dabc1..00000000000 --- a/iroh-blobs/src/util/io.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Utilities for working with tokio io - -use std::{io, pin::Pin, task::Poll}; - -use iroh_io::AsyncStreamReader; -use tokio::io::AsyncWrite; - -/// A reader that tracks the number of bytes read -#[derive(Debug)] -pub struct TrackingReader { - inner: R, - read: u64, -} - -impl TrackingReader { - /// Wrap a reader in a tracking reader - pub fn new(inner: R) -> Self { - Self { inner, read: 0 } - } - - /// Get the number of bytes read - #[allow(dead_code)] - pub fn bytes_read(&self) -> u64 { - self.read - } - - /// Get the inner reader - pub fn into_parts(self) -> (R, u64) { - (self.inner, self.read) - } -} - -impl AsyncStreamReader for TrackingReader -where - R: AsyncStreamReader, -{ - async fn read_bytes(&mut self, len: usize) -> io::Result { - let bytes = self.inner.read_bytes(len).await?; - self.read = self.read.saturating_add(bytes.len() as u64); - Ok(bytes) - } - - async fn read(&mut self) -> io::Result<[u8; L]> { - let res = self.inner.read::().await?; - self.read = self.read.saturating_add(L as u64); - Ok(res) - } -} - -/// A writer that tracks the number of bytes written -#[derive(Debug)] -pub struct TrackingWriter { - inner: W, - written: u64, -} - -impl TrackingWriter { - /// Wrap a writer in a tracking writer - pub fn new(inner: W) -> Self { - Self { inner, written: 0 } - } - - /// Get the number of bytes written - #[allow(dead_code)] - pub fn bytes_written(&self) -> u64 { - self.written - } - - /// Get the inner writer - pub fn into_parts(self) -> (W, u64) { - (self.inner, self.written) - } -} - -impl AsyncWrite for TrackingWriter { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &[u8], - ) -> Poll> { - let this = &mut *self; - let res = Pin::new(&mut this.inner).poll_write(cx, buf); - if let Poll::Ready(Ok(size)) = res { - this.written = this.written.saturating_add(size as u64); - } - res - } - - fn poll_flush( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_flush(cx) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_shutdown(cx) - } -} diff --git a/iroh-blobs/src/util/local_pool.rs b/iroh-blobs/src/util/local_pool.rs deleted file mode 100644 index e4a804a3816..00000000000 --- a/iroh-blobs/src/util/local_pool.rs +++ /dev/null @@ -1,685 +0,0 @@ -//! A local task pool with proper shutdown -use std::{ - any::Any, - future::Future, - ops::Deref, - pin::Pin, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use futures_lite::FutureExt; -use tokio::{ - sync::{Notify, Semaphore}, - task::{JoinError, JoinSet, LocalSet}, -}; - -type BoxedFut = Pin>>; -type SpawnFn = Box BoxedFut + Send + 'static>; - -enum Message { - /// Create a new task and execute it locally - Execute(SpawnFn), - /// Shutdown the thread after finishing all tasks - Finish, -} - -/// A local task pool with proper shutdown -/// -/// Unlike -/// [`LocalPoolHandle`](https://docs.rs/tokio-util/latest/tokio_util/task/struct.LocalPoolHandle.html), -/// this pool will join all its threads when dropped, ensuring that all Drop -/// implementations are run to completion. -/// -/// On drop, this pool will immediately cancel all *tasks* that are currently -/// being executed, and will wait for all threads to finish executing their -/// loops before returning. This means that all drop implementations will be -/// able to run to completion before drop exits. -/// -/// On [`LocalPool::finish`], this pool will notify all threads to shut down, -/// and then wait for all threads to finish executing their loops before -/// returning. This means that all currently executing tasks will be allowed to -/// run to completion. -/// -/// The pool will install the [`tracing::Subscriber`] which was set on the current thread of -/// where it was created as the default subscriber in all spawned threads. -#[derive(Debug)] -pub struct LocalPool { - threads: Vec>, - shutdown_sem: Arc, - cancel_token: CancellationToken, - handle: LocalPoolHandle, -} - -impl Deref for LocalPool { - type Target = LocalPoolHandle; - - fn deref(&self) -> &Self::Target { - &self.handle - } -} - -/// A handle to a [`LocalPool`] -#[derive(Debug, Clone)] -pub struct LocalPoolHandle { - /// The sender half of the channel used to send tasks to the pool - send: async_channel::Sender, -} - -/// What to do when a panic occurs in a pool thread -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PanicMode { - /// Log the panic and continue - /// - /// The panic will be re-thrown when the pool is dropped. - LogAndContinue, - /// Log the panic and immediately shut down the pool. - /// - /// The panic will be re-thrown when the pool is dropped. - Shutdown, -} - -/// Local task pool configuration -#[derive(Clone, Debug)] -pub struct Config { - /// Number of threads in the pool - pub threads: usize, - /// Prefix for thread names - pub thread_name_prefix: &'static str, - /// Ignore panics in pool threads - pub panic_mode: PanicMode, -} - -impl Default for Config { - fn default() -> Self { - Self { - threads: num_cpus::get(), - thread_name_prefix: "local-pool", - panic_mode: PanicMode::Shutdown, - } - } -} - -impl Default for LocalPool { - fn default() -> Self { - Self::new(Default::default()) - } -} - -impl LocalPool { - /// Create a new local pool with a single std thread. - pub fn single() -> Self { - Self::new(Config { - threads: 1, - ..Default::default() - }) - } - - /// Create a new local pool with the given config. - /// - /// This will use the current tokio runtime handle, so it must be called - /// from within a tokio runtime. - pub fn new(config: Config) -> Self { - let Config { - threads, - thread_name_prefix, - panic_mode, - } = config; - let cancel_token = CancellationToken::new(); - let (send, recv) = async_channel::unbounded::(); - let shutdown_sem = Arc::new(Semaphore::new(0)); - let handle = tokio::runtime::Handle::current(); - let handles = (0..threads) - .map(|i| { - Self::spawn_pool_thread( - format!("{thread_name_prefix}-{i}"), - recv.clone(), - cancel_token.clone(), - panic_mode, - shutdown_sem.clone(), - handle.clone(), - ) - }) - .collect::>>() - .expect("invalid thread name"); - Self { - threads: handles, - handle: LocalPoolHandle { send }, - cancel_token, - shutdown_sem, - } - } - - /// Get a cheaply cloneable handle to the pool - /// - /// This is not strictly necessary since we implement deref for - /// LocalPoolHandle, but makes getting a handle more explicit. - pub fn handle(&self) -> &LocalPoolHandle { - &self.handle - } - - /// Spawn a new pool thread. - fn spawn_pool_thread( - thread_name: String, - recv: async_channel::Receiver, - cancel_token: CancellationToken, - panic_mode: PanicMode, - shutdown_sem: Arc, - handle: tokio::runtime::Handle, - ) -> std::io::Result> { - let tracing_dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone()); - std::thread::Builder::new() - .name(thread_name) - .spawn(move || { - let _tracing_guard = tracing::dispatcher::set_default(&tracing_dispatcher); - let mut s = JoinSet::new(); - let mut last_panic = None; - let mut handle_join = |res: Option>| -> bool { - if let Some(Err(e)) = res { - if let Ok(panic) = e.try_into_panic() { - let panic_info = get_panic_info(&panic); - let thread_name = get_thread_name(); - tracing::error!( - "Panic in local pool thread: {}\n{}", - thread_name, - panic_info - ); - last_panic = Some(panic); - } - } - panic_mode == PanicMode::LogAndContinue || last_panic.is_none() - }; - let ls = LocalSet::new(); - let shutdown_mode = handle.block_on(ls.run_until(async { - loop { - tokio::select! { - // poll the set of futures - res = s.join_next(), if !s.is_empty() => { - if !handle_join(res) { - break ShutdownMode::Stop; - } - }, - // if the cancel token is cancelled, break the loop immediately - _ = cancel_token.cancelled() => break ShutdownMode::Stop, - // if we receive a message, execute it - msg = recv.recv() => { - match msg { - // just push into the join set - Ok(Message::Execute(f)) => { - s.spawn_local((f)()); - } - // break with optional semaphore - Ok(Message::Finish) => break ShutdownMode::Finish, - // if the sender is dropped, break the loop immediately - Err(async_channel::RecvError) => break ShutdownMode::Stop, - } - }, - } - } - })); - // soft shutdown mode is just like normal running, except that - // we don't add any more tasks and stop when there are no more - // tasks to run. - if shutdown_mode == ShutdownMode::Finish { - // somebody is asking for a clean shutdown, wait for all tasks to finish - handle.block_on(ls.run_until(async { - loop { - tokio::select! { - res = s.join_next() => { - if res.is_none() || !handle_join(res) { - break; - } - } - _ = cancel_token.cancelled() => break, - } - } - })); - } - // Always add the permit. If nobody is waiting for it, it does - // no harm. - shutdown_sem.add_permits(1); - if let Some(_panic) = last_panic { - // std::panic::resume_unwind(panic); - } - }) - } - - /// A future that resolves when the pool is cancelled - pub async fn cancelled(&self) { - self.cancel_token.cancelled().await - } - - /// Immediately stop polling all tasks and wait for all threads to finish. - /// - /// This is like drop, but waits for thread completion asynchronously. - /// - /// If there was a panic on any of the threads, it will be re-thrown here. - pub async fn shutdown(self) { - self.cancel_token.cancel(); - self.await_thread_completion().await; - // just make it explicit that this is where drop runs - drop(self); - } - - /// Gently shut down the pool - /// - /// Notifies all the pool threads to shut down and waits for them to finish. - /// - /// If you just want to drop the pool without giving the threads a chance to - /// process their remaining tasks, just use [`Self::shutdown`]. - /// - /// If you want to wait for only a limited time for the tasks to finish, - /// you can race this function with a timeout. - pub async fn finish(self) { - // we assume that there are exactly as many threads as there are handles. - // also, we assume that the threads are still running. - for _ in 0..self.threads_u32() { - // send the shutdown message - // sending will fail if all threads are already finished, but - // in that case we don't need to do anything. - // - // Threads will add a permit in any case, so await_thread_completion - // will then immediately return. - self.send.send(Message::Finish).await.ok(); - } - self.await_thread_completion().await; - } - - fn threads_u32(&self) -> u32 { - self.threads - .len() - .try_into() - .expect("invalid number of threads") - } - - async fn await_thread_completion(&self) { - // wait for all threads to finish. - // Each thread will add a permit to the semaphore. - let wait_for_semaphore = async move { - let _ = self - .shutdown_sem - .acquire_many(self.threads_u32()) - .await - .expect("semaphore closed"); - }; - // race the semaphore wait with the cancel token in case somebody - // cancels the pool while we are waiting. - tokio::select! { - _ = wait_for_semaphore => {} - _ = self.cancel_token.cancelled() => {} - } - } -} - -impl Drop for LocalPool { - fn drop(&mut self) { - self.cancel_token.cancel(); - let current_thread_id = std::thread::current().id(); - for handle in self.threads.drain(..) { - // we have no control over from where Drop is called, especially - // if the pool ends up in an Arc. So we need to check if we are - // dropping from within a pool thread and skip it in that case. - if handle.thread().id() == current_thread_id { - tracing::error!("Dropping LocalPool from within a pool thread."); - continue; - } - // Log any panics and resume them - if let Err(panic) = handle.join() { - let panic_info = get_panic_info(&panic); - let thread_name = get_thread_name(); - tracing::error!("Error joining thread: {}\n{}", thread_name, panic_info); - // std::panic::resume_unwind(panic); - } - } - } -} - -/// Errors for spawn failures -#[derive(thiserror::Error, Debug)] -pub enum SpawnError { - /// Task was dropped, either due to a panic or because the pool was shut down. - #[error("cancelled")] - Cancelled, -} - -type SpawnResult = std::result::Result; - -/// Future returned by [`LocalPoolHandle::spawn`] and [`LocalPoolHandle::try_spawn`]. -/// -/// Dropping this future will immediately cancel the task. The task can fail if -/// the pool is shut down or if the task panics. In both cases the future will -/// resolve to [`SpawnError::Cancelled`]. -#[repr(transparent)] -#[derive(Debug)] -pub struct Run(tokio::sync::oneshot::Receiver); - -impl Run { - /// Abort the task - /// - /// Dropping the future will also abort the task. - pub fn abort(&mut self) { - self.0.close(); - } -} - -impl Future for Run { - type Output = std::result::Result; - - fn poll( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - // map a RecvError (other side was dropped) to a SpawnError::Shutdown - // - // The only way the receiver can be dropped is if the pool is shut down. - self.0.poll(cx).map_err(|_| SpawnError::Cancelled) - } -} - -impl From for std::io::Error { - fn from(e: SpawnError) -> Self { - std::io::Error::new(std::io::ErrorKind::Other, e) - } -} - -impl LocalPoolHandle { - /// Get the number of tasks in the queue - /// - /// This is *not* the number of tasks being executed, but the number of - /// tasks waiting to be scheduled for execution. If this number is high, - /// it indicates that the pool is very busy. - /// - /// You might want to use this to throttle or reject requests. - pub fn waiting_tasks(&self) -> usize { - self.send.len() - } - - /// Spawn a task in the pool and return a future that resolves when the task - /// is done. - /// - /// If you don't care about the result, prefer [`LocalPoolHandle::spawn_detached`] - /// since it is more efficient. - pub fn try_spawn(&self, gen: F) -> SpawnResult> - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future + 'static, - T: Send + 'static, - { - let (mut send_res, recv_res) = tokio::sync::oneshot::channel(); - let item = move || async move { - let fut = (gen)(); - tokio::select! { - // send the result to the receiver - res = fut => { send_res.send(res).ok(); } - // immediately stop the task if the receiver is dropped - _ = send_res.closed() => {} - } - }; - self.try_spawn_detached(item)?; - Ok(Run(recv_res)) - } - - /// Spawn a task in the pool. - /// - /// The task will run to completion unless the pool is shut down or the task - /// panics. In case of panic, the pool will either log the panic and continue - /// or immediately shut down, depending on the [`PanicMode`]. - pub fn try_spawn_detached(&self, gen: F) -> SpawnResult<()> - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future + 'static, - { - let gen: SpawnFn = Box::new(move || Box::pin(gen())); - self.try_spawn_detached_boxed(gen) - } - - /// Spawn a task in the pool and await the result. - /// - /// Like [`LocalPoolHandle::try_spawn`], but panics if the pool is shut down. - pub fn spawn(&self, gen: F) -> Run - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future + 'static, - T: Send + 'static, - { - self.try_spawn(gen).expect("pool is shut down") - } - - /// Spawn a task in the pool. - /// - /// Like [`LocalPoolHandle::try_spawn_detached`], but panics if the pool is shut down. - pub fn spawn_detached(&self, gen: F) - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future + 'static, - { - self.try_spawn_detached(gen).expect("pool is shut down") - } - - /// Spawn a task in the pool. - /// - /// This is like [`LocalPoolHandle::try_spawn_detached`], but assuming that the - /// generator function is already boxed. This is the lowest overhead way to - /// spawn a task in the pool. - pub fn try_spawn_detached_boxed(&self, gen: SpawnFn) -> SpawnResult<()> { - self.send - .send_blocking(Message::Execute(gen)) - .map_err(|_| SpawnError::Cancelled) - } -} - -/// Thread shutdown mode -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ShutdownMode { - /// Finish all tasks and then stop - Finish, - /// Stop immediately - Stop, -} - -fn get_panic_info(panic: &Box) -> String { - if let Some(s) = panic.downcast_ref::<&str>() { - s.to_string() - } else if let Some(s) = panic.downcast_ref::() { - s.clone() - } else { - "Panic info unavailable".to_string() - } -} - -fn get_thread_name() -> String { - std::thread::current() - .name() - .unwrap_or("unnamed") - .to_string() -} - -/// A lightweight cancellation token -#[derive(Debug, Clone)] -struct CancellationToken { - inner: Arc, -} - -#[derive(Debug)] -struct CancellationTokenInner { - is_cancelled: AtomicBool, - notify: Notify, -} - -impl CancellationToken { - fn new() -> Self { - Self { - inner: Arc::new(CancellationTokenInner { - is_cancelled: AtomicBool::new(false), - notify: Notify::new(), - }), - } - } - - fn cancel(&self) { - if !self.inner.is_cancelled.swap(true, Ordering::SeqCst) { - self.inner.notify.notify_waiters(); - } - } - - async fn cancelled(&self) { - if self.is_cancelled() { - return; - } - - // Wait for notification if not cancelled - self.inner.notify.notified().await; - } - - fn is_cancelled(&self) -> bool { - self.inner.is_cancelled.load(Ordering::SeqCst) - } -} - -#[cfg(test)] -mod tests { - use std::{sync::atomic::AtomicU64, time::Duration}; - - use tracing::info; - - use super::*; - - /// A struct that simulates a long running drop operation - #[derive(Debug)] - struct TestDrop(Option>); - - impl Drop for TestDrop { - fn drop(&mut self) { - // delay to make sure the drop is executed completely - std::thread::sleep(Duration::from_millis(100)); - // increment the drop counter - if let Some(counter) = self.0.take() { - counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } - } - } - - impl TestDrop { - fn new(counter: Arc) -> Self { - Self(Some(counter)) - } - - fn forget(mut self) { - self.0.take(); - } - } - - /// Create a non-send test future that captures a TestDrop instance - async fn delay_then_drop(x: TestDrop) { - tokio::time::sleep(Duration::from_millis(100)).await; - // drop x at the end. we will never get here when the future is - // no longer polled, but drop should still be called - drop(x); - } - - /// Use a TestDrop instance to test cancellation - async fn delay_then_forget(x: TestDrop, delay: Duration) { - tokio::time::sleep(delay).await; - x.forget(); - } - - #[tokio::test] - async fn test_tracing() { - // This test wants to make sure that logging inside the pool propagates to the - // tracing subscriber that was set for the current thread at the time the pool was - // created. - // - // Look, there should be a custom tracing subscriber here that allows us to inspect - // the messages sent to it so we can verify it received all the messages. But have - // you ever tried to implement a tracing subscriber? In the mean time this test will - // just always pass, to really see the test run it with: - // - // cargo nextest run -p iroh-blobs local_pool::tests::test_tracing --success-output final - // - // and eyeball the output. yolo - let _guard = iroh_test::logging::setup(); - info!("hello from the test"); - let pool = LocalPool::single(); - pool.spawn(|| async move { - info!("hello from the pool"); - }) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_drop() { - let _ = tracing_subscriber::fmt::try_init(); - let pool = LocalPool::new(Config::default()); - let counter = Arc::new(AtomicU64::new(0)); - let n = 4; - for _ in 0..n { - let td = TestDrop::new(counter.clone()); - pool.spawn_detached(move || delay_then_drop(td)); - } - drop(pool); - assert_eq!(counter.load(std::sync::atomic::Ordering::SeqCst), n); - } - - #[tokio::test] - async fn test_finish() { - let _ = tracing_subscriber::fmt::try_init(); - let pool = LocalPool::new(Config::default()); - let counter = Arc::new(AtomicU64::new(0)); - let n = 4; - for _ in 0..n { - let td = TestDrop::new(counter.clone()); - pool.spawn_detached(move || delay_then_drop(td)); - } - pool.finish().await; - assert_eq!(counter.load(std::sync::atomic::Ordering::SeqCst), n); - } - - #[tokio::test] - async fn test_cancel() { - let _ = tracing_subscriber::fmt::try_init(); - let pool = LocalPool::new(Config { - threads: 2, - ..Config::default() - }); - let c1 = Arc::new(AtomicU64::new(0)); - let td1 = TestDrop::new(c1.clone()); - let handle = pool.spawn(move || { - // this one will be aborted anyway, so use a long delay to make sure - // that it does not accidentally run to completion - delay_then_forget(td1, Duration::from_secs(10)) - }); - drop(handle); - let c2 = Arc::new(AtomicU64::new(0)); - let td2 = TestDrop::new(c2.clone()); - let _handle = pool.spawn(move || { - // this one will not be aborted, so use a short delay so the test - // does not take too long - delay_then_forget(td2, Duration::from_millis(100)) - }); - pool.finish().await; - // c1 will be aborted, so drop will run before forget, so the counter will be increased - assert_eq!(c1.load(std::sync::atomic::Ordering::SeqCst), 1); - // c2 will not be aborted, so drop will run after forget, so the counter will not be increased - assert_eq!(c2.load(std::sync::atomic::Ordering::SeqCst), 0); - } - - // #[tokio::test] - // #[should_panic] - // #[ignore = "todo"] - // async fn test_panic() { - // let _ = tracing_subscriber::fmt::try_init(); - // let pool = LocalPool::new(Config { - // threads: 2, - // ..Config::default() - // }); - // pool.spawn_detached(|| async { - // panic!("test panic"); - // }); - // // we can't use shutdown here, because we need to allow time for the - // // panic to happen. - // pool.finish().await; - // } -} diff --git a/iroh-blobs/src/util/mem_or_file.rs b/iroh-blobs/src/util/mem_or_file.rs deleted file mode 100644 index d929a19c978..00000000000 --- a/iroh-blobs/src/util/mem_or_file.rs +++ /dev/null @@ -1,104 +0,0 @@ -use std::{fs::File, io}; - -use bao_tree::io::sync::{ReadAt, Size}; -use bytes::Bytes; - -/// This is a general purpose Either, just like Result, except that the two cases -/// are Mem for something that is in memory, and File for something that is somewhere -/// external and only available via io. -#[derive(Debug)] -pub enum MemOrFile { - /// We got it all in memory - Mem(M), - /// A file - File(F), -} - -/// Helper methods for a common way to use MemOrFile, where the memory part is something -/// like a slice, and the file part is a tuple consisiting of path or file and size. -impl MemOrFile -where - M: AsRef<[u8]>, -{ - /// Get the size of the MemOrFile - pub fn size(&self) -> u64 { - match self { - MemOrFile::Mem(mem) => mem.as_ref().len() as u64, - MemOrFile::File((_, size)) => *size, - } - } -} - -impl ReadAt for MemOrFile { - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { - match self { - MemOrFile::Mem(mem) => mem.as_ref().read_at(offset, buf), - MemOrFile::File(file) => file.read_at(offset, buf), - } - } -} - -impl Size for MemOrFile { - fn size(&self) -> io::Result> { - match self { - MemOrFile::Mem(mem) => Ok(Some(mem.len() as u64)), - MemOrFile::File(file) => file.size(), - } - } -} - -impl Default for MemOrFile { - fn default() -> Self { - MemOrFile::Mem(Default::default()) - } -} - -impl MemOrFile { - /// Turn a reference to a MemOrFile into a MemOrFile of references - pub fn as_ref(&self) -> MemOrFile<&M, &F> { - match self { - MemOrFile::Mem(mem) => MemOrFile::Mem(mem), - MemOrFile::File(file) => MemOrFile::File(file), - } - } - - /// True if this is a Mem - pub fn is_mem(&self) -> bool { - matches!(self, MemOrFile::Mem(_)) - } - - /// Get the mem part - pub fn mem(&self) -> Option<&M> { - match self { - MemOrFile::Mem(mem) => Some(mem), - MemOrFile::File(_) => None, - } - } - - /// Map the file part of this MemOrFile - pub fn map_file(self, f: impl FnOnce(F) -> F2) -> MemOrFile { - match self { - MemOrFile::Mem(mem) => MemOrFile::Mem(mem), - MemOrFile::File(file) => MemOrFile::File(f(file)), - } - } - - /// Try to map the file part of this MemOrFile - pub fn try_map_file( - self, - f: impl FnOnce(F) -> Result, - ) -> Result, E> { - match self { - MemOrFile::Mem(mem) => Ok(MemOrFile::Mem(mem)), - MemOrFile::File(file) => f(file).map(MemOrFile::File), - } - } - - /// Map the memory part of this MemOrFile - pub fn map_mem(self, f: impl FnOnce(M) -> M2) -> MemOrFile { - match self { - MemOrFile::Mem(mem) => MemOrFile::Mem(f(mem)), - MemOrFile::File(file) => MemOrFile::File(file), - } - } -} diff --git a/iroh-blobs/src/util/progress.rs b/iroh-blobs/src/util/progress.rs deleted file mode 100644 index a84d46bc747..00000000000 --- a/iroh-blobs/src/util/progress.rs +++ /dev/null @@ -1,687 +0,0 @@ -//! Utilities for reporting progress. -//! -//! The main entry point is the [ProgressSender] trait. -use std::{future::Future, io, marker::PhantomData, ops::Deref, sync::Arc}; - -use bytes::Bytes; -use iroh_io::AsyncSliceWriter; - -/// A general purpose progress sender. This should be usable for reporting progress -/// from both blocking and non-blocking contexts. -/// -/// # Id generation -/// -/// Any good progress protocol will refer to entities by means of a unique id. -/// E.g. if you want to report progress about some file operation, including details -/// such as the full path of the file would be very wasteful. It is better to -/// introduce a unique id for the file and then report progress using that id. -/// -/// The [IdGenerator] trait provides a method to generate such ids, [IdGenerator::new_id]. -/// -/// # Sending important messages -/// -/// Some messages are important for the receiver to receive. E.g. start and end -/// messages for some operation. If the receiver would miss one of these messages, -/// it would lose the ability to make sense of the progress message stream. -/// -/// This trait provides a method to send such important messages, in both blocking -/// contexts where you have to block until the message is sent [ProgressSender::blocking_send], -/// and non-blocking contexts where you have to yield until the message is sent [ProgressSender::send]. -/// -/// # Sending unimportant messages -/// -/// Some messages are self-contained and not important for the receiver to receive. -/// E.g. if you send millions of progress messages for copying a file that each -/// contain an id and the number of bytes copied so far, it is not important for -/// the receiver to receive every single one of these messages. In fact it is -/// useful to drop some of these messages because waiting for the progress events -/// to be sent can slow down the actual operation. -/// -/// This trait provides a method to send such unimportant messages that can be -/// used in both blocking and non-blocking contexts, [ProgressSender::try_send]. -/// -/// # Errors -/// -/// When the receiver is dropped, sending a message will fail. This provides a way -/// for the receiver to signal that the operation should be stopped. -/// -/// E.g. for a blocking copy operation that reports frequent progress messages, -/// as soon as the receiver is dropped, this is a signal to stop the copy operation. -/// -/// The error type is [ProgressSendError], which can be converted to an [std::io::Error] -/// for convenience. -/// -/// # Transforming the message type -/// -/// Sometimes you have a progress sender that sends a message of type `A` but an -/// operation that reports progress of type `B`. If you have a transformation for -/// every `B` to an `A`, you can use the [ProgressSender::with_map] method to transform the message. -/// -/// This is similar to the `futures::SinkExt::with` method. -/// -/// # Filtering the message type -/// -/// Sometimes you have a progress sender that sends a message of enum `A` but an -/// operation that reports progress of type `B`. You are interested only in some -/// enum cases of `A` that can be transformed to `B`. You can use the [ProgressSender::with_filter_map] -/// method to filter and transform the message. -/// -/// # No-op progress sender -/// -/// If you don't want to report progress, you can use the [IgnoreProgressSender] type. -/// -/// # Async channel progress sender -/// -/// If you want to use an async channel, you can use the [AsyncChannelProgressSender] type. -/// -/// # Implementing your own progress sender -/// -/// Progress senders will frequently be used in a multi-threaded context. -/// -/// They must be **cheap** to clone and send between threads. -/// They must also be thread safe, which is ensured by the [Send] and [Sync] bounds. -/// They must also be unencumbered by lifetimes, which is ensured by the `'static` bound. -/// -/// A typical implementation will wrap the sender part of a channel and an id generator. -pub trait ProgressSender: std::fmt::Debug + Clone + Send + Sync + 'static { - /// The message being sent. - type Msg: Send + Sync + 'static; - - /// Send a message and wait if the receiver is full. - /// - /// Use this to send important progress messages where delivery must be guaranteed. - #[must_use] - fn send(&self, msg: Self::Msg) -> impl Future> + Send; - - /// Try to send a message and drop it if the receiver is full. - /// - /// Use this to send progress messages where delivery is not important, e.g. a self contained progress message. - fn try_send(&self, msg: Self::Msg) -> ProgressSendResult<()>; - - /// Send a message and block if the receiver is full. - /// - /// Use this to send important progress messages where delivery must be guaranteed. - fn blocking_send(&self, msg: Self::Msg) -> ProgressSendResult<()>; - - /// Transform the message type by mapping to the type of this sender. - fn with_map Self::Msg + Send + Sync + Clone + 'static>( - self, - f: F, - ) -> WithMap { - WithMap(self, f, PhantomData) - } - - /// Transform the message type by filter-mapping to the type of this sender. - fn with_filter_map< - U: Send + Sync + 'static, - F: Fn(U) -> Option + Send + Sync + Clone + 'static, - >( - self, - f: F, - ) -> WithFilterMap { - WithFilterMap(self, f, PhantomData) - } - - /// Create a boxed progress sender to get rid of the concrete type. - fn boxed(self) -> BoxedProgressSender - where - Self: IdGenerator, - { - BoxedProgressSender(Arc::new(BoxableProgressSenderWrapper(self))) - } -} - -/// A boxed progress sender -pub struct BoxedProgressSender(Arc>); - -impl Clone for BoxedProgressSender { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl std::fmt::Debug for BoxedProgressSender { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_tuple("BoxedProgressSender").field(&self.0).finish() - } -} - -type BoxFuture<'a, T> = std::pin::Pin + Send + 'a>>; - -/// Boxable progress sender -trait BoxableProgressSender: IdGenerator + std::fmt::Debug + Send + Sync + 'static { - /// Send a message and wait if the receiver is full. - /// - /// Use this to send important progress messages where delivery must be guaranteed. - #[must_use] - fn send(&self, msg: T) -> BoxFuture<'_, ProgressSendResult<()>>; - - /// Try to send a message and drop it if the receiver is full. - /// - /// Use this to send progress messages where delivery is not important, e.g. a self contained progress message. - fn try_send(&self, msg: T) -> ProgressSendResult<()>; - - /// Send a message and block if the receiver is full. - /// - /// Use this to send important progress messages where delivery must be guaranteed. - fn blocking_send(&self, msg: T) -> ProgressSendResult<()>; -} - -impl BoxableProgressSender - for BoxableProgressSenderWrapper -{ - fn send(&self, msg: I::Msg) -> BoxFuture<'_, ProgressSendResult<()>> { - Box::pin(self.0.send(msg)) - } - - fn try_send(&self, msg: I::Msg) -> ProgressSendResult<()> { - self.0.try_send(msg) - } - - fn blocking_send(&self, msg: I::Msg) -> ProgressSendResult<()> { - self.0.blocking_send(msg) - } -} - -/// Boxable progress sender wrapper, used internally. -#[derive(Debug)] -#[repr(transparent)] -struct BoxableProgressSenderWrapper(I); - -impl IdGenerator for BoxableProgressSenderWrapper { - fn new_id(&self) -> u64 { - self.0.new_id() - } -} - -impl IdGenerator for Arc> { - fn new_id(&self) -> u64 { - self.deref().new_id() - } -} - -impl ProgressSender for Arc> { - type Msg = T; - - fn send(&self, msg: T) -> impl Future> + Send { - self.deref().send(msg) - } - - fn try_send(&self, msg: T) -> ProgressSendResult<()> { - self.deref().try_send(msg) - } - - fn blocking_send(&self, msg: T) -> ProgressSendResult<()> { - self.deref().blocking_send(msg) - } -} - -impl IdGenerator for BoxedProgressSender { - fn new_id(&self) -> u64 { - self.0.new_id() - } -} - -impl ProgressSender for BoxedProgressSender { - type Msg = T; - - async fn send(&self, msg: T) -> ProgressSendResult<()> { - self.0.send(msg).await - } - - fn try_send(&self, msg: T) -> ProgressSendResult<()> { - self.0.try_send(msg) - } - - fn blocking_send(&self, msg: T) -> ProgressSendResult<()> { - self.0.blocking_send(msg) - } -} - -impl ProgressSender for Option { - type Msg = T::Msg; - - async fn send(&self, msg: Self::Msg) -> ProgressSendResult<()> { - if let Some(inner) = self { - inner.send(msg).await - } else { - Ok(()) - } - } - - fn try_send(&self, msg: Self::Msg) -> ProgressSendResult<()> { - if let Some(inner) = self { - inner.try_send(msg) - } else { - Ok(()) - } - } - - fn blocking_send(&self, msg: Self::Msg) -> ProgressSendResult<()> { - if let Some(inner) = self { - inner.blocking_send(msg) - } else { - Ok(()) - } - } -} - -/// An id generator, to be combined with a progress sender. -pub trait IdGenerator { - /// Get a new unique id - fn new_id(&self) -> u64; -} - -/// A no-op progress sender. -pub struct IgnoreProgressSender(PhantomData); - -impl Default for IgnoreProgressSender { - fn default() -> Self { - Self(PhantomData) - } -} - -impl Clone for IgnoreProgressSender { - fn clone(&self) -> Self { - Self(PhantomData) - } -} - -impl std::fmt::Debug for IgnoreProgressSender { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("IgnoreProgressSender").finish() - } -} - -impl ProgressSender for IgnoreProgressSender { - type Msg = T; - - async fn send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> { - Ok(()) - } - - fn try_send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> { - Ok(()) - } - - fn blocking_send(&self, _msg: T) -> std::result::Result<(), ProgressSendError> { - Ok(()) - } -} - -impl IdGenerator for IgnoreProgressSender { - fn new_id(&self) -> u64 { - 0 - } -} - -/// Transform the message type by mapping to the type of this sender. -/// -/// See [ProgressSender::with_map]. -pub struct WithMap< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static, ->(I, F, PhantomData); - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static, - > std::fmt::Debug for WithMap -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_tuple("With").field(&self.0).finish() - } -} - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static, - > Clone for WithMap -{ - fn clone(&self) -> Self { - Self(self.0.clone(), self.1.clone(), PhantomData) - } -} - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static, - > ProgressSender for WithMap -{ - type Msg = U; - - async fn send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - let msg = (self.1)(msg); - self.0.send(msg).await - } - - fn try_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - let msg = (self.1)(msg); - self.0.try_send(msg) - } - - fn blocking_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - let msg = (self.1)(msg); - self.0.blocking_send(msg) - } -} - -/// Transform the message type by filter-mapping to the type of this sender. -/// -/// See [ProgressSender::with_filter_map]. -pub struct WithFilterMap(I, F, PhantomData); - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> Option + Clone + Send + Sync + 'static, - > std::fmt::Debug for WithFilterMap -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_tuple("FilterWith").field(&self.0).finish() - } -} - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> Option + Clone + Send + Sync + 'static, - > Clone for WithFilterMap -{ - fn clone(&self) -> Self { - Self(self.0.clone(), self.1.clone(), PhantomData) - } -} - -impl IdGenerator for WithFilterMap { - fn new_id(&self) -> u64 { - self.0.new_id() - } -} - -impl< - I: IdGenerator + ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> I::Msg + Clone + Send + Sync + 'static, - > IdGenerator for WithMap -{ - fn new_id(&self) -> u64 { - self.0.new_id() - } -} - -impl< - I: ProgressSender, - U: Send + Sync + 'static, - F: Fn(U) -> Option + Clone + Send + Sync + 'static, - > ProgressSender for WithFilterMap -{ - type Msg = U; - - async fn send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - if let Some(msg) = (self.1)(msg) { - self.0.send(msg).await - } else { - Ok(()) - } - } - - fn try_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - if let Some(msg) = (self.1)(msg) { - self.0.try_send(msg) - } else { - Ok(()) - } - } - - fn blocking_send(&self, msg: U) -> std::result::Result<(), ProgressSendError> { - if let Some(msg) = (self.1)(msg) { - self.0.blocking_send(msg) - } else { - Ok(()) - } - } -} - -/// A progress sender that uses an async channel. -pub struct AsyncChannelProgressSender { - sender: async_channel::Sender, - id: std::sync::Arc, -} - -impl std::fmt::Debug for AsyncChannelProgressSender { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AsyncChannelProgressSender") - .field("id", &self.id) - .field("sender", &self.sender) - .finish() - } -} - -impl Clone for AsyncChannelProgressSender { - fn clone(&self) -> Self { - Self { - sender: self.sender.clone(), - id: self.id.clone(), - } - } -} - -impl AsyncChannelProgressSender { - /// Create a new progress sender from an async channel sender. - pub fn new(sender: async_channel::Sender) -> Self { - Self { - sender, - id: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - } - } - - /// Returns true if `other` sends on the same `async_channel` channel as `self`. - pub fn same_channel(&self, other: &AsyncChannelProgressSender) -> bool { - same_channel(&self.sender, &other.sender) - } -} - -/// Given a value that is aligned and sized like a pointer, return the value of -/// the pointer as a usize. -fn get_as_ptr(value: &T) -> Option { - use std::mem; - if mem::size_of::() == std::mem::size_of::() - && mem::align_of::() == mem::align_of::() - { - // SAFETY: size and alignment requirements are checked and met - unsafe { Some(mem::transmute_copy(value)) } - } else { - None - } -} - -fn same_channel(a: &async_channel::Sender, b: &async_channel::Sender) -> bool { - // This relies on async_channel::Sender being just a newtype wrapper around - // an Arc>, so if two senders point to the same channel, the - // pointers will be the same. - get_as_ptr(a).unwrap() == get_as_ptr(b).unwrap() -} - -impl IdGenerator for AsyncChannelProgressSender { - fn new_id(&self) -> u64 { - self.id.fetch_add(1, std::sync::atomic::Ordering::SeqCst) - } -} - -impl ProgressSender for AsyncChannelProgressSender { - type Msg = T; - - async fn send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> { - self.sender - .send(msg) - .await - .map_err(|_| ProgressSendError::ReceiverDropped) - } - - fn try_send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> { - match self.sender.try_send(msg) { - Ok(_) => Ok(()), - Err(async_channel::TrySendError::Full(_)) => Ok(()), - Err(async_channel::TrySendError::Closed(_)) => Err(ProgressSendError::ReceiverDropped), - } - } - - fn blocking_send(&self, msg: Self::Msg) -> std::result::Result<(), ProgressSendError> { - match self.sender.send_blocking(msg) { - Ok(_) => Ok(()), - Err(_) => Err(ProgressSendError::ReceiverDropped), - } - } -} - -/// An error that can occur when sending progress messages. -/// -/// Really the only error that can occur is if the receiver is dropped. -#[derive(Debug, Clone, thiserror::Error)] -pub enum ProgressSendError { - /// The receiver was dropped. - #[error("receiver dropped")] - ReceiverDropped, -} - -/// A result type for progress sending. -pub type ProgressSendResult = std::result::Result; - -impl From for std::io::Error { - fn from(e: ProgressSendError) -> Self { - std::io::Error::new(std::io::ErrorKind::BrokenPipe, e) - } -} - -/// A slice writer that adds a synchronous progress callback. -/// -/// This wraps any `AsyncSliceWriter`, passes through all operations to the inner writer, and -/// calls the passed `on_write` callback whenever data is written. -#[derive(Debug)] -pub struct ProgressSliceWriter(W, F); - -impl ProgressSliceWriter { - /// Create a new `ProgressSliceWriter` from an inner writer and a progress callback - /// - /// The `on_write` function is called for each write, with the `offset` as the first and the - /// length of the data as the second param. - pub fn new(inner: W, on_write: F) -> Self { - Self(inner, on_write) - } - - /// Return the inner writer - pub fn into_inner(self) -> W { - self.0 - } -} - -impl AsyncSliceWriter - for ProgressSliceWriter -{ - async fn write_bytes_at(&mut self, offset: u64, data: Bytes) -> io::Result<()> { - (self.1)(offset, data.len()); - self.0.write_bytes_at(offset, data).await - } - - async fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result<()> { - (self.1)(offset, data.len()); - self.0.write_at(offset, data).await - } - - async fn sync(&mut self) -> io::Result<()> { - self.0.sync().await - } - - async fn set_len(&mut self, size: u64) -> io::Result<()> { - self.0.set_len(size).await - } -} - -/// A slice writer that adds a fallible progress callback. -/// -/// This wraps any `AsyncSliceWriter`, passes through all operations to the inner writer, and -/// calls the passed `on_write` callback whenever data is written. `on_write` must return an -/// `io::Result`, and can abort the download by returning an error. -#[derive(Debug)] -pub struct FallibleProgressSliceWriter(W, F); - -impl io::Result<()> + 'static> - FallibleProgressSliceWriter -{ - /// Create a new `ProgressSliceWriter` from an inner writer and a progress callback - /// - /// The `on_write` function is called for each write, with the `offset` as the first and the - /// length of the data as the second param. `on_write` must return a future which resolves to - /// an `io::Result`. If `on_write` returns an error, the download is aborted. - pub fn new(inner: W, on_write: F) -> Self { - Self(inner, on_write) - } - - /// Return the inner writer. - pub fn into_inner(self) -> W { - self.0 - } -} - -impl io::Result<()> + 'static> AsyncSliceWriter - for FallibleProgressSliceWriter -{ - async fn write_bytes_at(&mut self, offset: u64, data: Bytes) -> io::Result<()> { - (self.1)(offset, data.len())?; - self.0.write_bytes_at(offset, data).await - } - - async fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result<()> { - (self.1)(offset, data.len())?; - self.0.write_at(offset, data).await - } - - async fn sync(&mut self) -> io::Result<()> { - self.0.sync().await - } - - async fn set_len(&mut self, size: u64) -> io::Result<()> { - self.0.set_len(size).await - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; - - #[test] - fn get_as_ptr_works() { - struct Wrapper(Arc); - let x = Wrapper(Arc::new(1u64)); - assert_eq!( - get_as_ptr(&x).unwrap(), - Arc::as_ptr(&x.0) as usize - 2 * std::mem::size_of::() - ); - } - - #[test] - fn get_as_ptr_wrong_use() { - struct Wrapper(#[allow(dead_code)] u8); - let x = Wrapper(1); - assert!(get_as_ptr(&x).is_none()); - } - - #[test] - fn test_sender_is_ptr() { - assert_eq!( - std::mem::size_of::(), - std::mem::size_of::>() - ); - assert_eq!( - std::mem::align_of::(), - std::mem::align_of::>() - ); - } -} diff --git a/iroh-blobs/src/util/sparse_mem_file.rs b/iroh-blobs/src/util/sparse_mem_file.rs deleted file mode 100644 index ec6e431542d..00000000000 --- a/iroh-blobs/src/util/sparse_mem_file.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::io; - -use bao_tree::io::sync::{ReadAt, Size, WriteAt}; -use derive_more::Deref; -use range_collections::{range_set::RangeSetRange, RangeSet2}; - -/// A file that is sparse in memory -/// -/// It is not actually using sparse storage to make reading faster, so it will -/// not conserve memory. It is just a way to remember the gaps so we can -/// write it to a file in a sparse way later. -#[derive(derive_more::Debug)] -pub struct SparseMemFile { - /// The data, with gaps filled with zeros - #[debug("{} bytes", data.len())] - data: Vec, - /// The ranges that are not zeros, so we can distinguish between zeros and gaps - ranges: RangeSet2, -} - -impl Default for SparseMemFile { - fn default() -> Self { - Self::new() - } -} - -impl From> for SparseMemFile { - fn from(data: Vec) -> Self { - let ranges = RangeSet2::from(0..data.len()); - Self { data, ranges } - } -} - -impl TryInto> for SparseMemFile { - type Error = io::Error; - - fn try_into(self) -> Result, Self::Error> { - let (data, ranges) = self.into_parts(); - if ranges == RangeSet2::from(0..data.len()) { - Ok(data) - } else { - Err(io::Error::new( - io::ErrorKind::InvalidData, - "SparseMemFile has gaps", - )) - } - } -} - -impl SparseMemFile { - /// Create a new, empty SparseMemFile - pub fn new() -> Self { - Self { - data: Vec::new(), - ranges: RangeSet2::empty(), - } - } - - /// Get the data and the valid ranges - pub fn into_parts(self) -> (Vec, RangeSet2) { - (self.data, self.ranges) - } - - /// Persist the SparseMemFile to a WriteAt - /// - /// This will not persist the gaps, only the data that was written. - pub fn persist(&self, mut target: impl WriteAt) -> io::Result<()> { - let size = self.data.len(); - for range in self.ranges.iter() { - let range = match range { - RangeSetRange::Range(range) => *range.start..*range.end, - RangeSetRange::RangeFrom(range) => *range.start..size, - }; - let start = range.start.try_into().unwrap(); - let buf = &self.data[range]; - target.write_at(start, buf)?; - } - Ok(()) - } -} - -impl AsRef<[u8]> for SparseMemFile { - fn as_ref(&self) -> &[u8] { - &self.data - } -} - -impl Deref for SparseMemFile { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - &self.data - } -} - -impl ReadAt for SparseMemFile { - fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { - self.data.read_at(offset, buf) - } -} - -impl WriteAt for SparseMemFile { - fn write_at(&mut self, offset: u64, buf: &[u8]) -> io::Result { - let start: usize = offset.try_into().map_err(|_| io::ErrorKind::InvalidInput)?; - let end = start - .checked_add(buf.len()) - .ok_or(io::ErrorKind::InvalidInput)?; - let n = self.data.write_at(offset, buf)?; - self.ranges |= RangeSet2::from(start..end); - Ok(n) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl Size for SparseMemFile { - fn size(&self) -> io::Result> { - Ok(Some(self.data.len() as u64)) - } -} diff --git a/iroh-cli/Cargo.toml b/iroh-cli/Cargo.toml index 67fb5ba6d29..d36cbe0111b 100644 --- a/iroh-cli/Cargo.toml +++ b/iroh-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-cli" -version = "0.27.0" +version = "0.28.1" edition = "2021" readme = "README.md" description = "Bytes. Distributed." @@ -40,14 +40,16 @@ futures-util = { version = "0.3.30", features = ["futures-sink"] } hex = "0.4.3" human-time = "0.1.6" indicatif = { version = "0.17", features = ["tokio"] } -iroh = { version = "0.27.0", path = "../iroh", features = ["metrics"] } -iroh-gossip = { version = "0.27.0", path = "../iroh-gossip" } -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics" } +iroh = { version = "0.28.1", path = "../iroh", features = ["metrics"] } +iroh-gossip = "0.28.1" +iroh-docs = { version = "0.28.0", features = ["rpc"]} +iroh-metrics = { version = "0.28.0" } parking_lot = "0.12.1" pkarr = { version = "2.2.0", default-features = false } portable-atomic = "1" +portmapper = { version = "0.1.0", path = "../net-tools/portmapper" } postcard = "1.0.8" -quic-rpc = { version = "0.12", features = ["flume-transport", "quinn-transport"] } +quic-rpc = { version = "0.15", features = ["flume-transport", "quinn-transport"] } rand = "0.8.5" ratatui = "0.26.2" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/iroh-cli/src/commands/blobs.rs b/iroh-cli/src/commands/blobs.rs index 0be5657a87f..a7f7b19f960 100644 --- a/iroh-cli/src/commands/blobs.rs +++ b/iroh-cli/src/commands/blobs.rs @@ -19,6 +19,7 @@ use iroh::{ base::{node_addr::AddrInfoOptions, ticket::BlobTicket}, blobs::{ get::{db::DownloadProgress, progress::BlobProgress, Stats}, + net_protocol::DownloadMode, provider::AddProgress, store::{ ConsistencyCheckProgress, ExportFormat, ExportMode, ReportLevel, ValidateProgress, @@ -28,12 +29,11 @@ use iroh::{ }, client::{ blobs::{ - BlobInfo, BlobStatus, CollectionInfo, DownloadMode, DownloadOptions, - IncompleteBlobInfo, WrapOption, + BlobInfo, BlobStatus, CollectionInfo, DownloadOptions, IncompleteBlobInfo, WrapOption, }, Iroh, }, - net::{key::PublicKey, relay::RelayUrl, NodeAddr}, + net::{key::PublicKey, NodeAddr, RelayUrl}, }; use tokio::io::AsyncWriteExt; @@ -370,7 +370,9 @@ impl BlobCommands { BlobFormat::Raw }; let status = iroh.blobs().status(hash).await?; - let ticket = iroh.blobs().share(hash, format, addr_options).await?; + let mut addr: NodeAddr = iroh.net().node_addr().await?; + addr.apply_options(addr_options); + let ticket = BlobTicket::new(addr, hash, format)?; let (blob_status, size) = match (status, format) { (BlobStatus::Complete { size }, BlobFormat::Raw) => ("blob", size), diff --git a/iroh-cli/src/commands/docs.rs b/iroh-cli/src/commands/docs.rs index 4846db12089..fc827890d67 100644 --- a/iroh-cli/src/commands/docs.rs +++ b/iroh-cli/src/commands/docs.rs @@ -18,17 +18,17 @@ use indicatif::{HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressS use iroh::{ base::{base32::fmt_short, node_addr::AddrInfoOptions}, blobs::{provider::AddProgress, util::SetTagOption, Hash, Tag}, - client::{ - blobs::WrapOption, - docs::{Doc, Entry, LiveEvent, Origin, ShareMode}, - Iroh, - }, + client::{blobs::WrapOption, Doc, Iroh}, docs::{ store::{DownloadPolicy, FilterKind, Query, SortDirection}, AuthorId, DocTicket, NamespaceId, }, util::fs::{path_content_info, path_to_key, PathContent}, }; +use iroh_docs::{ + engine::Origin, + rpc::client::docs::{Entry, LiveEvent, ShareMode}, +}; use tokio::io::AsyncReadExt; use crate::config::ConsoleEnv; @@ -414,7 +414,7 @@ impl DocCommands { let mut stream = doc.get_many(query).await?; while let Some(entry) = stream.try_next().await? { - println!("{}", fmt_entry(&doc, &entry, mode).await); + println!("{}", fmt_entry(&iroh.blobs(), &entry, mode).await); } } Self::Keys { @@ -440,7 +440,7 @@ impl DocCommands { query = query.sort_by(sort.into(), direction); let mut stream = doc.get_many(query).await?; while let Some(entry) = stream.try_next().await? { - println!("{}", fmt_entry(&doc, &entry, mode).await); + println!("{}", fmt_entry(&iroh.blobs(), &entry, mode).await); } } Self::Leave { doc } => { @@ -516,7 +516,7 @@ impl DocCommands { } Some(e) => e, }; - match entry.content_reader(&doc).await { + match iroh.blobs().read(entry.content_hash()).await { Ok(mut content) => { if let Some(dir) = path.parent() { if let Err(err) = std::fs::create_dir_all(dir) { @@ -547,13 +547,14 @@ impl DocCommands { Self::Watch { doc } => { let doc = get_doc(iroh, env, doc).await?; let mut stream = doc.subscribe().await?; + let blobs = iroh.blobs(); while let Some(event) = stream.next().await { let event = event?; match event { LiveEvent::InsertLocal { entry } => { println!( "local change: {}", - fmt_entry(&doc, &entry, DisplayContentMode::Auto).await + fmt_entry(&blobs, &entry, DisplayContentMode::Auto).await ) } LiveEvent::InsertRemote { @@ -563,17 +564,17 @@ impl DocCommands { } => { let content = match content_status { iroh::docs::ContentStatus::Complete => { - fmt_entry(&doc, &entry, DisplayContentMode::Auto).await + fmt_entry(&blobs, &entry, DisplayContentMode::Auto).await } iroh::docs::ContentStatus::Incomplete => { let (Ok(content) | Err(content)) = - fmt_content(&doc, &entry, DisplayContentMode::ShortHash) + fmt_content(&blobs, &entry, DisplayContentMode::ShortHash) .await; format!("", content, human_len(&entry)) } iroh::docs::ContentStatus::Missing => { let (Ok(content) | Err(content)) = - fmt_content(&doc, &entry, DisplayContentMode::ShortHash) + fmt_content(&blobs, &entry, DisplayContentMode::ShortHash) .await; format!("", content, human_len(&entry)) } @@ -679,14 +680,19 @@ impl DocCommands { /// Gets the document given the client, the environment (and maybe the [`NamespaceID`]). async fn get_doc(iroh: &Iroh, env: &ConsoleEnv, id: Option) -> anyhow::Result { + let doc_id = env.doc(id)?; iroh.docs() - .open(env.doc(id)?) + .open(doc_id) .await? .context("Document not found") } /// Formats the content. If an error occurs it's returned in a formatted, friendly way. -async fn fmt_content(doc: &Doc, entry: &Entry, mode: DisplayContentMode) -> Result { +async fn fmt_content( + blobs: &iroh::client::blobs::Client, + entry: &Entry, + mode: DisplayContentMode, +) -> Result { let read_failed = |err: anyhow::Error| format!(""); let encode_hex = |err: std::string::FromUtf8Error| format!("0x{}", hex::encode(err.as_bytes())); let as_utf8 = |buf: Vec| String::from_utf8(buf).map(|repr| format!("\"{repr}\"")); @@ -695,11 +701,17 @@ async fn fmt_content(doc: &Doc, entry: &Entry, mode: DisplayContentMode) -> Resu DisplayContentMode::Auto => { if entry.content_len() < MAX_DISPLAY_CONTENT_LEN { // small content: read fully as UTF-8 - let bytes = entry.content_bytes(doc).await.map_err(read_failed)?; + let bytes = blobs + .read_to_bytes(entry.content_hash()) + .await + .map_err(read_failed)?; Ok(as_utf8(bytes.into()).unwrap_or_else(encode_hex)) } else { // large content: read just the first part as UTF-8 - let mut blob_reader = entry.content_reader(doc).await.map_err(read_failed)?; + let mut blob_reader = blobs + .read(entry.content_hash()) + .await + .map_err(read_failed)?; let mut buf = Vec::with_capacity(MAX_DISPLAY_CONTENT_LEN as usize + 5); blob_reader @@ -714,7 +726,10 @@ async fn fmt_content(doc: &Doc, entry: &Entry, mode: DisplayContentMode) -> Resu } DisplayContentMode::Content => { // read fully as UTF-8 - let bytes = entry.content_bytes(doc).await.map_err(read_failed)?; + let bytes = blobs + .read_to_bytes(entry.content_hash()) + .await + .map_err(read_failed)?; Ok(as_utf8(bytes.into()).unwrap_or_else(encode_hex)) } DisplayContentMode::ShortHash => { @@ -735,12 +750,16 @@ fn human_len(entry: &Entry) -> HumanBytes { /// Formats an entry for display as a `String`. #[must_use = "this won't be printed, you need to print it yourself"] -async fn fmt_entry(doc: &Doc, entry: &Entry, mode: DisplayContentMode) -> String { +async fn fmt_entry( + blobs: &iroh::client::blobs::Client, + entry: &Entry, + mode: DisplayContentMode, +) -> String { let key = std::str::from_utf8(entry.key()) .unwrap_or("") .bold(); let author = fmt_short(entry.author()); - let (Ok(content) | Err(content)) = fmt_content(doc, entry, mode).await; + let (Ok(content) | Err(content)) = fmt_content(blobs, entry, mode).await; let len = human_len(entry); format!("@{author}: {key} = {content} ({len})") } diff --git a/iroh-cli/src/commands/doctor.rs b/iroh-cli/src/commands/doctor.rs index 27a2279d581..77abd0a97ff 100644 --- a/iroh-cli/src/commands/doctor.rs +++ b/iroh-cli/src/commands/doctor.rs @@ -37,10 +37,9 @@ use iroh::{ endpoint::{self, Connection, ConnectionTypeStream, RecvStream, RemoteInfo, SendStream}, key::{PublicKey, SecretKey}, metrics::MagicsockMetrics, - netcheck, portmapper, - relay::{RelayMap, RelayMode, RelayUrl}, + netcheck, ticket::NodeTicket, - Endpoint, NodeAddr, NodeId, + Endpoint, NodeAddr, NodeId, RelayMap, RelayMode, RelayUrl, }, util::{path::IrohPaths, progress::ProgressWriter}, }; diff --git a/iroh-cli/src/commands/gossip.rs b/iroh-cli/src/commands/gossip.rs index 5e5c6bfacc1..a5f8fb964df 100644 --- a/iroh-cli/src/commands/gossip.rs +++ b/iroh-cli/src/commands/gossip.rs @@ -7,10 +7,8 @@ use bao_tree::blake3; use clap::{ArgGroup, Subcommand}; use futures_lite::StreamExt; use futures_util::SinkExt; -use iroh::{ - client::{gossip::SubscribeOpts, Iroh}, - net::NodeId, -}; +use iroh::{client::Iroh, net::NodeId}; +use iroh_gossip::rpc::client::SubscribeOpts; use tokio::io::AsyncBufReadExt; /// Commands to manage gossiping. diff --git a/iroh-cli/src/commands/net.rs b/iroh-cli/src/commands/net.rs index 0d3a6aef9b1..a2b786583c5 100644 --- a/iroh-cli/src/commands/net.rs +++ b/iroh-cli/src/commands/net.rs @@ -12,8 +12,7 @@ use iroh::{ client::Iroh, net::{ endpoint::{DirectAddrInfo, RemoteInfo}, - relay::RelayUrl, - NodeAddr, NodeId, + NodeAddr, NodeId, RelayUrl, }, }; diff --git a/iroh-cli/src/commands/start.rs b/iroh-cli/src/commands/start.rs index 1a6f1e93ce8..daa5f564efd 100644 --- a/iroh-cli/src/commands/start.rs +++ b/iroh-cli/src/commands/start.rs @@ -11,7 +11,7 @@ use anyhow::Result; use colored::Colorize; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; use iroh::{ - net::relay::{RelayMap, RelayMode}, + net::{RelayMap, RelayMode}, node::{Node, RpcStatus, DEFAULT_RPC_ADDR}, }; use tracing::{info_span, trace, Instrument}; diff --git a/iroh-cli/src/config.rs b/iroh-cli/src/config.rs index 479230c2035..297b30a0dd6 100644 --- a/iroh-cli/src/config.rs +++ b/iroh-cli/src/config.rs @@ -13,7 +13,7 @@ use anyhow::{anyhow, bail, Context, Result}; use iroh::{ client::Iroh, docs::{AuthorId, NamespaceId}, - net::relay::{RelayMap, RelayNode}, + net::{RelayMap, RelayNode}, node::GcPolicy, }; use parking_lot::RwLock; diff --git a/iroh-cli/tests/cli.rs b/iroh-cli/tests/cli.rs index 948c15215cb..08a55662233 100644 --- a/iroh-cli/tests/cli.rs +++ b/iroh-cli/tests/cli.rs @@ -154,8 +154,12 @@ fn cli_provide_tree_resume() -> Result<()> { { // import the files into an ephemeral iroh to use the generated blobs db in tests let provider = make_provider_in(&src_iroh_data_dir_pre, Input::Path(src.clone()), false)?; - // small synchronization points: allow iroh to be ready for transfer - std::thread::sleep(std::time::Duration::from_secs(5)); + // small synchronization point: allow iroh to be ready for transfer + #[cfg(target_os = "windows")] + let wait = 10u64; + #[cfg(not(target_os = "windows"))] + let wait = 5u64; + std::thread::sleep(std::time::Duration::from_secs(wait)); let _ticket = match_provide_output(&provider, count, BlobOrCollection::Collection)?; } @@ -236,8 +240,12 @@ fn cli_provide_file_resume() -> Result<()> { { // import the files into an ephemeral iroh to use the generated blobs db in tests let provider = make_provider_in(&src_iroh_data_dir_pre, Input::Path(file.clone()), false)?; - // small synchronization points: allow iroh to be ready for transfer - std::thread::sleep(std::time::Duration::from_secs(5)); + // small synchronization point: allow iroh to be ready for transfer + #[cfg(target_os = "windows")] + let wait = 10u64; + #[cfg(not(target_os = "windows"))] + let wait = 5u64; + std::thread::sleep(std::time::Duration::from_secs(wait)); let _ticket = match_provide_output(&provider, count, BlobOrCollection::Blob)?; } @@ -865,7 +873,8 @@ fn match_provide_output( /// (r"hello world!", 1), /// (r"\S*$", 1), /// (r"\d{2}/\d{2}/\d{4}", 3), -/// ]); +/// ], +/// ); /// ``` fn assert_matches_line(reader: R, expressions: I) -> Vec<(usize, Vec)> where diff --git a/iroh-dns-server/Cargo.toml b/iroh-dns-server/Cargo.toml index eb45cd186d3..6aaf514f94a 100644 --- a/iroh-dns-server/Cargo.toml +++ b/iroh-dns-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-dns-server" -version = "0.27.0" +version = "0.28.0" edition = "2021" description = "A pkarr relay and DNS server" license = "MIT OR Apache-2.0" @@ -24,7 +24,7 @@ governor = "0.6.3" hickory-proto = "=0.25.0-alpha.2" hickory-server = { version = "=0.25.0-alpha.2", features = ["dns-over-rustls"] } http = "1.0.0" -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics" } +iroh-metrics = { version = "0.28.0" } lru = "0.12.3" mainline = "2.0.1" parking_lot = "0.12.1" @@ -53,8 +53,8 @@ z32 = "1.1.1" [dev-dependencies] hickory-resolver = "=0.25.0-alpha.2" -iroh-net = { version = "0.27.0", path = "../iroh-net" } -iroh-test = { path = "../iroh-test" } +iroh-net = { version = "0.28.0" } +iroh-test = "0.28.0" pkarr = { version = "2.2.0", features = ["rand"] } [package.metadata.docs.rs] diff --git a/iroh-dns-server/config.dev.toml b/iroh-dns-server/config.dev.toml index a43b10364af..37569002bf4 100644 --- a/iroh-dns-server/config.dev.toml +++ b/iroh-dns-server/config.dev.toml @@ -1,3 +1,5 @@ +pkarr_put_rate_limit = "disabled" + [http] port = 8080 bind_addr = "127.0.0.1" diff --git a/iroh-dns-server/config.prod.toml b/iroh-dns-server/config.prod.toml index 64b3f88f679..9868f67fc9b 100644 --- a/iroh-dns-server/config.prod.toml +++ b/iroh-dns-server/config.prod.toml @@ -1,3 +1,5 @@ +pkarr_put_rate_limit = "smart" + [https] port = 443 domains = ["irohdns.example.org"] diff --git a/iroh-dns-server/src/config.rs b/iroh-dns-server/src/config.rs index 89222f9daf8..732d65e4e8b 100644 --- a/iroh-dns-server/src/config.rs +++ b/iroh-dns-server/src/config.rs @@ -12,7 +12,7 @@ use tracing::info; use crate::{ dns::DnsConfig, - http::{CertMode, HttpConfig, HttpsConfig}, + http::{CertMode, HttpConfig, HttpsConfig, RateLimitConfig}, }; const DEFAULT_METRICS_ADDR: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9117); @@ -43,6 +43,10 @@ pub struct Config { /// Config for the mainline lookup. pub mainline: Option, + + /// Config for pkarr rate limit + #[serde(default)] + pub pkarr_put_rate_limit: RateLimitConfig, } /// The config for the metrics server. @@ -185,6 +189,7 @@ impl Default for Config { }, metrics: None, mainline: None, + pkarr_put_rate_limit: RateLimitConfig::default(), } } } diff --git a/iroh-dns-server/src/http.rs b/iroh-dns-server/src/http.rs index d7c84993964..06050074918 100644 --- a/iroh-dns-server/src/http.rs +++ b/iroh-dns-server/src/http.rs @@ -30,7 +30,7 @@ mod pkarr; mod rate_limiting; mod tls; -pub use self::tls::CertMode; +pub use self::{rate_limiting::RateLimitConfig, tls::CertMode}; use crate::{config::Config, metrics::Metrics, state::AppState}; /// Config for the HTTP server @@ -71,13 +71,14 @@ impl HttpServer { pub async fn spawn( http_config: Option, https_config: Option, + rate_limit_config: RateLimitConfig, state: AppState, ) -> Result { if http_config.is_none() && https_config.is_none() { bail!("Either http or https config is required"); } - let app = create_app(state); + let app = create_app(state, &rate_limit_config); let mut tasks = JoinSet::new(); @@ -184,7 +185,7 @@ impl HttpServer { } } -pub(crate) fn create_app(state: AppState) -> Router { +pub(crate) fn create_app(state: AppState, rate_limit_config: &RateLimitConfig) -> Router { // configure cors middleware let cors = CorsLayer::new() // allow `GET` and `POST` when accessing the resource @@ -209,7 +210,7 @@ pub(crate) fn create_app(state: AppState) -> Router { }); // configure rate limiting middleware - let rate_limit = rate_limiting::create(); + let rate_limit = rate_limiting::create(rate_limit_config); // configure routes // @@ -218,7 +219,11 @@ pub(crate) fn create_app(state: AppState) -> Router { .route("/dns-query", get(doh::get).post(doh::post)) .route( "/pkarr/:key", - get(pkarr::get).put(pkarr::put.layer(rate_limit)), + if let Some(rate_limit) = rate_limit { + get(pkarr::get).put(pkarr::put.layer(rate_limit)) + } else { + get(pkarr::get).put(pkarr::put) + }, ) .route("/healthcheck", get(|| async { "OK" })) .route("/", get(|| async { "Hi!" })) @@ -232,7 +237,6 @@ pub(crate) fn create_app(state: AppState) -> Router { } /// Record request metrics. -/// // TODO: // * Request duration would be much better tracked as a histogram. // * It would be great to attach labels to the metrics, so that the recorded metrics diff --git a/iroh-dns-server/src/http/rate_limiting.rs b/iroh-dns-server/src/http/rate_limiting.rs index 991ff6e3b09..4ba64b4049f 100644 --- a/iroh-dns-server/src/http/rate_limiting.rs +++ b/iroh-dns-server/src/http/rate_limiting.rs @@ -1,21 +1,68 @@ use std::time::Duration; use governor::{clock::QuantaInstant, middleware::NoOpMiddleware}; +use serde::{Deserialize, Serialize}; use tower_governor::{ - governor::GovernorConfigBuilder, key_extractor::PeerIpKeyExtractor, GovernorLayer, + governor::GovernorConfigBuilder, + key_extractor::{PeerIpKeyExtractor, SmartIpKeyExtractor}, + GovernorLayer, }; +/// Config for http server rate limit. +#[derive(Debug, Deserialize, Default, Serialize, Clone)] +#[serde(rename_all = "lowercase")] +pub enum RateLimitConfig { + /// Disable rate limit. + Disabled, + /// Enable rate limit based on the connection's peer IP address. + /// + /// + #[default] + Simple, + /// Enable rate limit based on headers commonly used by reverse proxies. + /// + /// Uses headers commonly used by reverse proxies to extract the original IP address, + /// falling back to the connection's peer IP address. + /// + Smart, +} + +impl Default for &RateLimitConfig { + fn default() -> Self { + &RateLimitConfig::Simple + } +} + /// Create the default rate-limiting layer. /// /// This spawns a background thread to clean up the rate limiting cache. -pub fn create() -> GovernorLayer<'static, PeerIpKeyExtractor, NoOpMiddleware> { +pub fn create( + rate_limit_config: &RateLimitConfig, +) -> Option>> { + let use_smart_extractor = match rate_limit_config { + RateLimitConfig::Disabled => { + tracing::info!("Rate limiting disabled"); + return None; + } + RateLimitConfig::Simple => false, + RateLimitConfig::Smart => true, + }; + + tracing::info!("Rate limiting enabled ({rate_limit_config:?})"); + // Configure rate limiting: // * allow bursts with up to five requests per IP address // * replenish one element every two seconds - let governor_conf = GovernorConfigBuilder::default() - // .use_headers() - .per_second(4) - .burst_size(2) + let mut governor_conf_builder = GovernorConfigBuilder::default(); + // governor_conf_builder.use_headers() + governor_conf_builder.per_second(4); + governor_conf_builder.burst_size(2); + + if use_smart_extractor { + governor_conf_builder.key_extractor(SmartIpKeyExtractor); + } + + let governor_conf = governor_conf_builder .finish() .expect("failed to build rate-limiting governor"); @@ -34,7 +81,7 @@ pub fn create() -> GovernorLayer<'static, PeerIpKeyExtractor, NoOpMiddleware Range-based set reconciliation is a simple approach to efficiently compute the union of two -sets over a network, based on recursively partitioning the sets and comparing fingerprints of -the partitions to probabilistically detect whether a partition requires further work. - -The crate exposes a generic storage interface with in-memory and persistent, file-based -implementations. The latter makes use of [`redb`], an embedded key-value store, and persists -the whole store with all replicas to a single file. - -[paper]: https://arxiv.org/abs/2212.13567 - - -# License - -This project is licensed under either of - - * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or - http://www.apache.org/licenses/LICENSE-2.0) - * MIT license ([LICENSE-MIT](LICENSE-MIT) or - http://opensource.org/licenses/MIT) - -at your option. - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in this project by you, as defined in the Apache-2.0 license, -shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-docs/proptest-regressions/ranger.txt b/iroh-docs/proptest-regressions/ranger.txt deleted file mode 100644 index 8e8177c0e4f..00000000000 --- a/iroh-docs/proptest-regressions/ranger.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 797e83179f8684388880e25a6fac7b4047eb15b03c55c1fb725b82bdbd0a4369 # shrinks to a = {TestKey("3"): ()}, b = {TestKey(""): (), TestKey("3"): (), TestKey("4"): (), TestKey("5"): (), TestKey("a"): (), TestKey("b"): (), TestKey("c"): ()} -cc f5b7604319ead6181c2ff42e53f05e2c6f0298adf0b38ea4ae4710c43abb7663 # shrinks to input = _SimpleStoreSyncArgs { alice: [(3, ()), (a, ())], bob: [(, ()), (0, ()), (b, ())] } -cc 41d9d33f002235dfe4bed83621fe79348725bbe00931451782025d98c1b81522 # shrinks to input = _SimpleStoreSyncU8Args { alice: [("", 58)], bob: [("", 0)] } diff --git a/iroh-docs/src/actor.rs b/iroh-docs/src/actor.rs deleted file mode 100644 index 98d0225cb6f..00000000000 --- a/iroh-docs/src/actor.rs +++ /dev/null @@ -1,1040 +0,0 @@ -//! This contains an actor spawned on a separate thread to process replica and store operations. - -use std::{ - collections::{hash_map, HashMap}, - num::NonZeroU64, - sync::Arc, - thread::JoinHandle, - time::Duration, -}; - -use anyhow::{anyhow, Context, Result}; -use bytes::Bytes; -use futures_util::FutureExt; -use iroh_base::hash::Hash; -use iroh_metrics::inc; -use serde::{Deserialize, Serialize}; -use tokio::{sync::oneshot, task::JoinSet}; -use tracing::{debug, error, error_span, trace, warn}; - -use crate::{ - metrics::Metrics, - ranger::Message, - store::{ - fs::{ContentHashesIterator, StoreInstance}, - DownloadPolicy, ImportNamespaceOutcome, Query, Store, - }, - Author, AuthorHeads, AuthorId, Capability, CapabilityKind, ContentStatus, - ContentStatusCallback, Event, NamespaceId, NamespaceSecret, PeerIdBytes, Replica, ReplicaInfo, - SignedEntry, SyncOutcome, -}; - -const ACTION_CAP: usize = 1024; -pub(crate) const MAX_COMMIT_DELAY: Duration = Duration::from_millis(500); - -#[derive(derive_more::Debug, derive_more::Display)] -enum Action { - #[display("NewAuthor")] - ImportAuthor { - author: Author, - #[debug("reply")] - reply: oneshot::Sender>, - }, - #[display("ExportAuthor")] - ExportAuthor { - author: AuthorId, - #[debug("reply")] - reply: oneshot::Sender>>, - }, - #[display("DeleteAuthor")] - DeleteAuthor { - author: AuthorId, - #[debug("reply")] - reply: oneshot::Sender>, - }, - #[display("NewReplica")] - ImportNamespace { - capability: Capability, - #[debug("reply")] - reply: oneshot::Sender>, - }, - #[display("ListAuthors")] - ListAuthors { - #[debug("reply")] - reply: async_channel::Sender>, - }, - #[display("ListReplicas")] - ListReplicas { - #[debug("reply")] - reply: async_channel::Sender>, - }, - #[display("ContentHashes")] - ContentHashes { - #[debug("reply")] - reply: oneshot::Sender>, - }, - #[display("FlushStore")] - FlushStore { - #[debug("reply")] - reply: oneshot::Sender>, - }, - #[display("Replica({}, {})", _0.fmt_short(), _1)] - Replica(NamespaceId, ReplicaAction), - #[display("Shutdown")] - Shutdown { - #[debug("reply")] - reply: Option>, - }, -} - -#[derive(derive_more::Debug, strum::Display)] -enum ReplicaAction { - Open { - #[debug("reply")] - reply: oneshot::Sender>, - opts: OpenOpts, - }, - Close { - #[debug("reply")] - reply: oneshot::Sender>, - }, - GetState { - #[debug("reply")] - reply: oneshot::Sender>, - }, - SetSync { - sync: bool, - #[debug("reply")] - reply: oneshot::Sender>, - }, - Subscribe { - sender: async_channel::Sender, - #[debug("reply")] - reply: oneshot::Sender>, - }, - Unsubscribe { - sender: async_channel::Sender, - #[debug("reply")] - reply: oneshot::Sender>, - }, - InsertLocal { - author: AuthorId, - key: Bytes, - hash: Hash, - len: u64, - #[debug("reply")] - reply: oneshot::Sender>, - }, - DeletePrefix { - author: AuthorId, - key: Bytes, - #[debug("reply")] - reply: oneshot::Sender>, - }, - InsertRemote { - entry: SignedEntry, - from: PeerIdBytes, - content_status: ContentStatus, - #[debug("reply")] - reply: oneshot::Sender>, - }, - SyncInitialMessage { - #[debug("reply")] - reply: oneshot::Sender>>, - }, - SyncProcessMessage { - message: Message, - from: PeerIdBytes, - state: SyncOutcome, - #[debug("reply")] - reply: oneshot::Sender>, SyncOutcome)>>, - }, - GetSyncPeers { - #[debug("reply")] - reply: oneshot::Sender>>>, - }, - RegisterUsefulPeer { - peer: PeerIdBytes, - #[debug("reply")] - reply: oneshot::Sender>, - }, - GetExact { - author: AuthorId, - key: Bytes, - include_empty: bool, - reply: oneshot::Sender>>, - }, - GetMany { - query: Query, - reply: async_channel::Sender>, - }, - DropReplica { - reply: oneshot::Sender>, - }, - ExportSecretKey { - reply: oneshot::Sender>, - }, - HasNewsForUs { - heads: AuthorHeads, - #[debug("reply")] - reply: oneshot::Sender>>, - }, - SetDownloadPolicy { - policy: DownloadPolicy, - #[debug("reply")] - reply: oneshot::Sender>, - }, - GetDownloadPolicy { - #[debug("reply")] - reply: oneshot::Sender>, - }, -} - -/// The state for an open replica. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct OpenState { - /// Whether to accept sync requests for this replica. - pub sync: bool, - /// How many event subscriptions are open - pub subscribers: usize, - /// By how many handles the replica is currently held open - pub handles: usize, -} - -#[derive(Debug)] -struct OpenReplica { - info: ReplicaInfo, - sync: bool, - handles: usize, -} - -/// The [`SyncHandle`] controls an actor thread which executes replica and store operations. -/// -/// The [`SyncHandle`] exposes async methods which all send messages into the actor thread, usually -/// returning something via a return channel. The actor thread itself is a regular [`std::thread`] -/// which processes incoming messages sequentially. -/// -/// The handle is cheaply cloneable. Once the last clone is dropped, the actor thread is joined. -/// The thread will finish processing all messages in the channel queue, and then exit. -/// To prevent this last drop from blocking the calling thread, you can call [`SyncHandle::shutdown`] -/// and await its result before dropping the last [`SyncHandle`]. This ensures that -/// waiting for the actor to finish happens in an async context, and therefore that the final -/// [`SyncHandle::drop`] will not block. -#[derive(Debug, Clone)] -pub struct SyncHandle { - tx: async_channel::Sender, - join_handle: Arc>>, -} - -/// Options when opening a replica. -#[derive(Debug, Default)] -pub struct OpenOpts { - /// Set to true to set sync state to true. - pub sync: bool, - /// Optionally subscribe to replica events. - pub subscribe: Option>, -} -impl OpenOpts { - /// Set sync state to true. - pub fn sync(mut self) -> Self { - self.sync = true; - self - } - /// Subscribe to replica events. - pub fn subscribe(mut self, subscribe: async_channel::Sender) -> Self { - self.subscribe = Some(subscribe); - self - } -} - -#[allow(missing_docs)] -impl SyncHandle { - /// Spawn a sync actor and return a handle. - pub fn spawn( - store: Store, - content_status_callback: Option, - me: String, - ) -> SyncHandle { - let (action_tx, action_rx) = async_channel::bounded(ACTION_CAP); - let actor = Actor { - store, - states: Default::default(), - action_rx, - content_status_callback, - tasks: Default::default(), - }; - let join_handle = std::thread::Builder::new() - .name("sync-actor".to_string()) - .spawn(move || { - let span = error_span!("sync", %me); - let _enter = span.enter(); - - if let Err(err) = actor.run() { - error!("Sync actor closed with error: {err:?}"); - } - }) - .expect("failed to spawn thread"); - let join_handle = Arc::new(Some(join_handle)); - SyncHandle { - tx: action_tx, - join_handle, - } - } - - pub async fn open(&self, namespace: NamespaceId, opts: OpenOpts) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::Open { reply, opts }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn close(&self, namespace: NamespaceId) -> Result { - let (reply, rx) = oneshot::channel(); - self.send_replica(namespace, ReplicaAction::Close { reply }) - .await?; - rx.await? - } - - pub async fn subscribe( - &self, - namespace: NamespaceId, - sender: async_channel::Sender, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - self.send_replica(namespace, ReplicaAction::Subscribe { sender, reply }) - .await?; - rx.await? - } - - pub async fn unsubscribe( - &self, - namespace: NamespaceId, - sender: async_channel::Sender, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - self.send_replica(namespace, ReplicaAction::Unsubscribe { sender, reply }) - .await?; - rx.await? - } - - pub async fn set_sync(&self, namespace: NamespaceId, sync: bool) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::SetSync { sync, reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn insert_local( - &self, - namespace: NamespaceId, - author: AuthorId, - key: Bytes, - hash: Hash, - len: u64, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::InsertLocal { - author, - key, - hash, - len, - reply, - }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn delete_prefix( - &self, - namespace: NamespaceId, - author: AuthorId, - key: Bytes, - ) -> Result { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::DeletePrefix { author, key, reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn insert_remote( - &self, - namespace: NamespaceId, - entry: SignedEntry, - from: PeerIdBytes, - content_status: ContentStatus, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::InsertRemote { - entry, - from, - content_status, - reply, - }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn sync_initial_message( - &self, - namespace: NamespaceId, - ) -> Result> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::SyncInitialMessage { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn sync_process_message( - &self, - namespace: NamespaceId, - message: Message, - from: PeerIdBytes, - state: SyncOutcome, - ) -> Result<(Option>, SyncOutcome)> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::SyncProcessMessage { - reply, - message, - from, - state, - }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn get_sync_peers(&self, namespace: NamespaceId) -> Result>> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::GetSyncPeers { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn register_useful_peer( - &self, - namespace: NamespaceId, - peer: PeerIdBytes, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::RegisterUsefulPeer { reply, peer }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn has_news_for_us( - &self, - namespace: NamespaceId, - heads: AuthorHeads, - ) -> Result> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::HasNewsForUs { reply, heads }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn get_many( - &self, - namespace: NamespaceId, - query: Query, - reply: async_channel::Sender>, - ) -> Result<()> { - let action = ReplicaAction::GetMany { query, reply }; - self.send_replica(namespace, action).await?; - Ok(()) - } - - pub async fn get_exact( - &self, - namespace: NamespaceId, - author: AuthorId, - key: Bytes, - include_empty: bool, - ) -> Result> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::GetExact { - author, - key, - include_empty, - reply, - }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn drop_replica(&self, namespace: NamespaceId) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::DropReplica { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn export_secret_key(&self, namespace: NamespaceId) -> Result { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::ExportSecretKey { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn get_state(&self, namespace: NamespaceId) -> Result { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::GetState { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn shutdown(&self) -> Result { - let (reply, rx) = oneshot::channel(); - let action = Action::Shutdown { reply: Some(reply) }; - self.send(action).await?; - let store = rx.await?; - Ok(store) - } - - pub async fn list_authors(&self, reply: async_channel::Sender>) -> Result<()> { - self.send(Action::ListAuthors { reply }).await - } - - pub async fn list_replicas( - &self, - reply: async_channel::Sender>, - ) -> Result<()> { - self.send(Action::ListReplicas { reply }).await - } - - pub async fn import_author(&self, author: Author) -> Result { - let (reply, rx) = oneshot::channel(); - self.send(Action::ImportAuthor { author, reply }).await?; - rx.await? - } - - pub async fn export_author(&self, author: AuthorId) -> Result> { - let (reply, rx) = oneshot::channel(); - self.send(Action::ExportAuthor { author, reply }).await?; - rx.await? - } - - pub async fn delete_author(&self, author: AuthorId) -> Result<()> { - let (reply, rx) = oneshot::channel(); - self.send(Action::DeleteAuthor { author, reply }).await?; - rx.await? - } - - pub async fn import_namespace(&self, capability: Capability) -> Result { - let (reply, rx) = oneshot::channel(); - self.send(Action::ImportNamespace { capability, reply }) - .await?; - rx.await? - } - - pub async fn get_download_policy(&self, namespace: NamespaceId) -> Result { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::GetDownloadPolicy { reply }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn set_download_policy( - &self, - namespace: NamespaceId, - policy: DownloadPolicy, - ) -> Result<()> { - let (reply, rx) = oneshot::channel(); - let action = ReplicaAction::SetDownloadPolicy { reply, policy }; - self.send_replica(namespace, action).await?; - rx.await? - } - - pub async fn content_hashes(&self) -> Result { - let (reply, rx) = oneshot::channel(); - self.send(Action::ContentHashes { reply }).await?; - rx.await? - } - - /// Makes sure that all pending database operations are persisted to disk. - /// - /// Otherwise, database operations are batched into bigger transactions for speed. - /// Use this if you need to make sure something is written to the database - /// before another operation, e.g. to make sure sudden process exits don't corrupt - /// your application state. - /// - /// It's not necessary to call this function before shutdown, as `shutdown` will - /// trigger a flush on its own. - pub async fn flush_store(&self) -> Result<()> { - let (reply, rx) = oneshot::channel(); - self.send(Action::FlushStore { reply }).await?; - rx.await? - } - - async fn send(&self, action: Action) -> Result<()> { - self.tx - .send(action) - .await - .context("sending to iroh_docs actor failed")?; - Ok(()) - } - async fn send_replica(&self, namespace: NamespaceId, action: ReplicaAction) -> Result<()> { - self.send(Action::Replica(namespace, action)).await?; - Ok(()) - } -} - -impl Drop for SyncHandle { - fn drop(&mut self) { - // this means we're dropping the last reference - if let Some(handle) = Arc::get_mut(&mut self.join_handle) { - // this call is the reason tx can not be a tokio mpsc channel. - // we have no control about where drop is called, yet tokio send_blocking panics - // when called from inside a tokio runtime. - self.tx.send_blocking(Action::Shutdown { reply: None }).ok(); - let handle = handle.take().expect("this can only run once"); - if let Err(err) = handle.join() { - warn!(?err, "Failed to join sync actor"); - } - } - } -} - -struct Actor { - store: Store, - states: OpenReplicas, - action_rx: async_channel::Receiver, - content_status_callback: Option, - tasks: JoinSet<()>, -} - -impl Actor { - fn run(self) -> Result<()> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build()?; - let local_set = tokio::task::LocalSet::new(); - local_set.block_on(&rt, async move { self.run_async().await }); - Ok(()) - } - - async fn run_async(mut self) { - let reply = loop { - let timeout = tokio::time::sleep(MAX_COMMIT_DELAY); - tokio::pin!(timeout); - let action = tokio::select! { - _ = &mut timeout => { - if let Err(cause) = self.store.flush() { - error!(?cause, "failed to flush store"); - } - continue; - } - action = self.action_rx.recv() => { - match action { - Ok(action) => action, - Err(async_channel::RecvError) => { - debug!("action channel disconnected"); - break None; - } - - } - } - }; - trace!(%action, "tick"); - inc!(Metrics, actor_tick_main); - match action { - Action::Shutdown { reply } => { - break reply; - } - action => { - if self.on_action(action).is_err() { - warn!("failed to send reply: receiver dropped"); - } - } - } - }; - - if let Err(cause) = self.store.flush() { - warn!(?cause, "failed to flush store"); - } - self.close_all(); - self.tasks.abort_all(); - debug!("docs actor shutdown"); - if let Some(reply) = reply { - reply.send(self.store).ok(); - } - } - - fn on_action(&mut self, action: Action) -> Result<(), SendReplyError> { - match action { - Action::Shutdown { .. } => { - unreachable!("Shutdown is handled in run()") - } - Action::ImportAuthor { author, reply } => { - let id = author.id(); - send_reply(reply, self.store.import_author(author).map(|_| id)) - } - Action::ExportAuthor { author, reply } => { - send_reply(reply, self.store.get_author(&author)) - } - Action::DeleteAuthor { author, reply } => { - send_reply(reply, self.store.delete_author(author)) - } - Action::ImportNamespace { capability, reply } => send_reply_with(reply, self, |this| { - let id = capability.id(); - let outcome = this.store.import_namespace(capability.clone())?; - if let ImportNamespaceOutcome::Upgraded = outcome { - if let Ok(state) = this.states.get_mut(&id) { - state.info.merge_capability(capability)?; - } - } - Ok(id) - }), - Action::ListAuthors { reply } => { - let iter = self - .store - .list_authors() - .map(|a| a.map(|a| a.map(|a| a.id()))); - self.tasks - .spawn_local(iter_to_channel_async(reply, iter).map(|_| ())); - Ok(()) - } - Action::ListReplicas { reply } => { - let iter = self.store.list_namespaces(); - self.tasks - .spawn_local(iter_to_channel_async(reply, iter).map(|_| ())); - Ok(()) - } - Action::ContentHashes { reply } => { - send_reply_with(reply, self, |this| this.store.content_hashes()) - } - Action::FlushStore { reply } => send_reply(reply, self.store.flush()), - Action::Replica(namespace, action) => self.on_replica_action(namespace, action), - } - } - - fn on_replica_action( - &mut self, - namespace: NamespaceId, - action: ReplicaAction, - ) -> Result<(), SendReplyError> { - match action { - ReplicaAction::Open { reply, opts } => { - tracing::trace!("open in"); - let res = self.open(namespace, opts); - tracing::trace!("open out"); - send_reply(reply, res) - } - ReplicaAction::Close { reply } => { - let res = self.close(namespace); - // ignore errors when no receiver is present for close - reply.send(Ok(res)).ok(); - Ok(()) - } - ReplicaAction::Subscribe { sender, reply } => send_reply_with(reply, self, |this| { - let state = this.states.get_mut(&namespace)?; - state.info.subscribe(sender); - Ok(()) - }), - ReplicaAction::Unsubscribe { sender, reply } => send_reply_with(reply, self, |this| { - let state = this.states.get_mut(&namespace)?; - state.info.unsubscribe(&sender); - drop(sender); - Ok(()) - }), - ReplicaAction::SetSync { sync, reply } => send_reply_with(reply, self, |this| { - let state = this.states.get_mut(&namespace)?; - state.sync = sync; - Ok(()) - }), - ReplicaAction::InsertLocal { - author, - key, - hash, - len, - reply, - } => send_reply_with(reply, self, move |this| { - let author = get_author(&mut this.store, &author)?; - let mut replica = this.states.replica(namespace, &mut this.store)?; - replica.insert(&key, &author, hash, len)?; - Ok(()) - }), - ReplicaAction::DeletePrefix { author, key, reply } => { - send_reply_with(reply, self, |this| { - let author = get_author(&mut this.store, &author)?; - let mut replica = this.states.replica(namespace, &mut this.store)?; - let res = replica.delete_prefix(&key, &author)?; - Ok(res) - }) - } - ReplicaAction::InsertRemote { - entry, - from, - content_status, - reply, - } => send_reply_with(reply, self, move |this| { - let mut replica = this - .states - .replica_if_syncing(&namespace, &mut this.store)?; - replica.insert_remote_entry(entry, from, content_status)?; - Ok(()) - }), - - ReplicaAction::SyncInitialMessage { reply } => { - send_reply_with(reply, self, move |this| { - let mut replica = this - .states - .replica_if_syncing(&namespace, &mut this.store)?; - let res = replica.sync_initial_message()?; - Ok(res) - }) - } - ReplicaAction::SyncProcessMessage { - message, - from, - mut state, - reply, - } => send_reply_with(reply, self, move |this| { - let mut replica = this - .states - .replica_if_syncing(&namespace, &mut this.store)?; - let res = replica.sync_process_message(message, from, &mut state)?; - Ok((res, state)) - }), - ReplicaAction::GetSyncPeers { reply } => send_reply_with(reply, self, move |this| { - this.states.ensure_open(&namespace)?; - let peers = this.store.get_sync_peers(&namespace)?; - Ok(peers.map(|iter| iter.collect())) - }), - ReplicaAction::RegisterUsefulPeer { peer, reply } => { - let res = self.store.register_useful_peer(namespace, peer); - send_reply(reply, res) - } - ReplicaAction::GetExact { - author, - key, - include_empty, - reply, - } => send_reply_with(reply, self, move |this| { - this.states.ensure_open(&namespace)?; - this.store.get_exact(namespace, author, key, include_empty) - }), - ReplicaAction::GetMany { query, reply } => { - let iter = self - .states - .ensure_open(&namespace) - .and_then(|_| self.store.get_many(namespace, query)); - self.tasks - .spawn_local(iter_to_channel_async(reply, iter).map(|_| ())); - Ok(()) - } - ReplicaAction::DropReplica { reply } => send_reply_with(reply, self, |this| { - this.close(namespace); - this.store.remove_replica(&namespace) - }), - ReplicaAction::ExportSecretKey { reply } => { - let res = self - .states - .get_mut(&namespace) - .and_then(|state| Ok(state.info.capability.secret_key()?.clone())); - send_reply(reply, res) - } - ReplicaAction::GetState { reply } => send_reply_with(reply, self, move |this| { - let state = this.states.get_mut(&namespace)?; - let handles = state.handles; - let sync = state.sync; - let subscribers = state.info.subscribers_count(); - Ok(OpenState { - handles, - sync, - subscribers, - }) - }), - ReplicaAction::HasNewsForUs { heads, reply } => { - let res = self.store.has_news_for_us(namespace, &heads); - send_reply(reply, res) - } - ReplicaAction::SetDownloadPolicy { policy, reply } => { - send_reply(reply, self.store.set_download_policy(&namespace, policy)) - } - ReplicaAction::GetDownloadPolicy { reply } => { - send_reply(reply, self.store.get_download_policy(&namespace)) - } - } - } - - fn close(&mut self, namespace: NamespaceId) -> bool { - let res = self.states.close(namespace); - if res { - self.store.close_replica(namespace); - } - res - } - - fn close_all(&mut self) { - for id in self.states.close_all() { - self.store.close_replica(id); - } - } - - fn open(&mut self, namespace: NamespaceId, opts: OpenOpts) -> Result<()> { - let open_cb = || { - let mut info = self.store.load_replica_info(&namespace)?; - if let Some(cb) = &self.content_status_callback { - info.set_content_status_callback(Arc::clone(cb)); - } - Ok(info) - }; - self.states.open_with(namespace, opts, open_cb) - } -} - -#[derive(Default)] -struct OpenReplicas(HashMap); - -impl OpenReplicas { - fn replica<'a, 'b>( - &'a mut self, - namespace: NamespaceId, - store: &'b mut Store, - ) -> Result> { - let state = self.get_mut(&namespace)?; - Ok(Replica::new( - StoreInstance::new(state.info.capability.id(), store), - &mut state.info, - )) - } - - fn replica_if_syncing<'a, 'b>( - &'a mut self, - namespace: &NamespaceId, - store: &'b mut Store, - ) -> Result> { - let state = self.get_mut(namespace)?; - anyhow::ensure!(state.sync, "sync is not enabled for replica"); - Ok(Replica::new( - StoreInstance::new(state.info.capability.id(), store), - &mut state.info, - )) - } - - fn get_mut(&mut self, namespace: &NamespaceId) -> Result<&mut OpenReplica> { - self.0.get_mut(namespace).context("replica not open") - } - - fn is_open(&self, namespace: &NamespaceId) -> bool { - self.0.contains_key(namespace) - } - - fn ensure_open(&self, namespace: &NamespaceId) -> Result<()> { - match self.is_open(namespace) { - true => Ok(()), - false => Err(anyhow!("replica not open")), - } - } - fn open_with( - &mut self, - namespace: NamespaceId, - opts: OpenOpts, - mut open_cb: impl FnMut() -> Result, - ) -> Result<()> { - match self.0.entry(namespace) { - hash_map::Entry::Vacant(e) => { - let mut info = open_cb()?; - if let Some(sender) = opts.subscribe { - info.subscribe(sender); - } - debug!(namespace = %namespace.fmt_short(), "open"); - let state = OpenReplica { - info, - sync: opts.sync, - handles: 1, - }; - e.insert(state); - } - hash_map::Entry::Occupied(mut e) => { - let state = e.get_mut(); - state.handles += 1; - state.sync = state.sync || opts.sync; - if let Some(sender) = opts.subscribe { - state.info.subscribe(sender); - } - } - } - Ok(()) - } - fn close(&mut self, namespace: NamespaceId) -> bool { - match self.0.entry(namespace) { - hash_map::Entry::Vacant(_e) => { - warn!(namespace = %namespace.fmt_short(), "received close request for closed replica"); - true - } - hash_map::Entry::Occupied(mut e) => { - let state = e.get_mut(); - state.handles = state.handles.wrapping_sub(1); - if state.handles == 0 { - let _ = e.remove_entry(); - debug!(namespace = %namespace.fmt_short(), "close"); - true - } else { - false - } - } - } - } - - fn close_all(&mut self) -> impl Iterator + '_ { - self.0.drain().map(|(n, _s)| n) - } -} - -async fn iter_to_channel_async( - channel: async_channel::Sender>, - iter: Result>>, -) -> Result<(), SendReplyError> { - match iter { - Err(err) => channel.send(Err(err)).await.map_err(send_reply_error)?, - Ok(iter) => { - for item in iter { - channel.send(item).await.map_err(send_reply_error)?; - } - } - } - Ok(()) -} - -fn get_author(store: &mut Store, id: &AuthorId) -> Result { - store.get_author(id)?.context("author not found") -} - -#[derive(Debug)] -struct SendReplyError; - -fn send_reply(sender: oneshot::Sender, value: T) -> Result<(), SendReplyError> { - sender.send(value).map_err(send_reply_error) -} - -fn send_reply_with( - sender: oneshot::Sender>, - this: &mut Actor, - f: impl FnOnce(&mut Actor) -> Result, -) -> Result<(), SendReplyError> { - sender.send(f(this)).map_err(send_reply_error) -} - -fn send_reply_error(_err: T) -> SendReplyError { - SendReplyError -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::store; - #[tokio::test] - async fn open_close() -> anyhow::Result<()> { - let store = store::Store::memory(); - let sync = SyncHandle::spawn(store, None, "foo".into()); - let namespace = NamespaceSecret::new(&mut rand::rngs::OsRng {}); - let id = namespace.id(); - sync.import_namespace(namespace.into()).await?; - sync.open(id, Default::default()).await?; - let (tx, rx) = async_channel::bounded(10); - sync.subscribe(id, tx).await?; - sync.close(id).await?; - assert!(rx.recv().await.is_err()); - Ok(()) - } -} diff --git a/iroh-docs/src/engine.rs b/iroh-docs/src/engine.rs deleted file mode 100644 index ed8bb5fe91b..00000000000 --- a/iroh-docs/src/engine.rs +++ /dev/null @@ -1,410 +0,0 @@ -//! Handlers and actors to for live syncing replicas. -//! -//! [`crate::Replica`] is also called documents here. - -use std::{ - io, - path::PathBuf, - str::FromStr, - sync::{Arc, RwLock}, -}; - -use anyhow::{bail, Context, Result}; -use futures_lite::{Stream, StreamExt}; -use iroh_blobs::{downloader::Downloader, store::EntryStatus, Hash}; -use iroh_gossip::net::Gossip; -use iroh_net::{key::PublicKey, Endpoint, NodeAddr}; -use serde::{Deserialize, Serialize}; -use tokio::sync::{mpsc, oneshot}; -use tokio_util::task::AbortOnDropHandle; -use tracing::{error, error_span, Instrument}; - -use self::live::{LiveActor, ToLiveActor}; -pub use self::{ - live::SyncEvent, - state::{Origin, SyncReason}, -}; -use crate::{ - actor::SyncHandle, Author, AuthorId, ContentStatus, ContentStatusCallback, Entry, NamespaceId, -}; - -mod gossip; -mod live; -mod state; - -/// Capacity of the channel for the [`ToLiveActor`] messages. -const ACTOR_CHANNEL_CAP: usize = 64; -/// Capacity for the channels for [`Engine::subscribe`]. -const SUBSCRIBE_CHANNEL_CAP: usize = 256; - -/// The sync engine coordinates actors that manage open documents, set-reconciliation syncs with -/// peers and a gossip swarm for each syncing document. -#[derive(derive_more::Debug, Clone)] -pub struct Engine { - /// [`Endpoint`] used by the engine. - pub endpoint: Endpoint, - /// Handle to the actor thread. - pub sync: SyncHandle, - /// The persistent default author for this engine. - pub default_author: Arc, - to_live_actor: mpsc::Sender, - #[allow(dead_code)] - actor_handle: Arc>, - #[debug("ContentStatusCallback")] - content_status_cb: ContentStatusCallback, -} - -impl Engine { - /// Start the sync engine. - /// - /// This will spawn two tokio tasks for the live sync coordination and gossip actors, and a - /// thread for the [`crate::actor::SyncHandle`]. - pub async fn spawn( - endpoint: Endpoint, - gossip: Gossip, - replica_store: crate::store::Store, - bao_store: B, - downloader: Downloader, - default_author_storage: DefaultAuthorStorage, - ) -> anyhow::Result { - let (live_actor_tx, to_live_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); - let me = endpoint.node_id().fmt_short(); - - let content_status_cb = { - let bao_store = bao_store.clone(); - Arc::new(move |hash| entry_to_content_status(bao_store.entry_status_sync(&hash))) - }; - let sync = SyncHandle::spawn(replica_store, Some(content_status_cb.clone()), me.clone()); - - let actor = LiveActor::new( - sync.clone(), - endpoint.clone(), - gossip.clone(), - bao_store, - downloader, - to_live_actor_recv, - live_actor_tx.clone(), - ); - let actor_handle = tokio::task::spawn( - async move { - if let Err(err) = actor.run().await { - error!("sync actor failed: {err:?}"); - } - } - .instrument(error_span!("sync", %me)), - ); - - let default_author = match DefaultAuthor::load(default_author_storage, &sync).await { - Ok(author) => author, - Err(err) => { - // If loading the default author failed, make sure to shutdown the sync actor before - // returning. - let _store = sync.shutdown().await.ok(); - return Err(err); - } - }; - - Ok(Self { - endpoint, - sync, - to_live_actor: live_actor_tx, - actor_handle: Arc::new(AbortOnDropHandle::new(actor_handle)), - content_status_cb, - default_author: Arc::new(default_author), - }) - } - - /// Start to sync a document. - /// - /// If `peers` is non-empty, it will both do an initial set-reconciliation sync with each peer, - /// and join an iroh-gossip swarm with these peers to receive and broadcast document updates. - pub async fn start_sync(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { - let (reply, reply_rx) = oneshot::channel(); - self.to_live_actor - .send(ToLiveActor::StartSync { - namespace, - peers, - reply, - }) - .await?; - reply_rx.await??; - Ok(()) - } - - /// Stop the live sync for a document and leave the gossip swarm. - /// - /// If `kill_subscribers` is true, all existing event subscribers will be dropped. This means - /// they will receive `None` and no further events in case of rejoining the document. - pub async fn leave(&self, namespace: NamespaceId, kill_subscribers: bool) -> Result<()> { - let (reply, reply_rx) = oneshot::channel(); - self.to_live_actor - .send(ToLiveActor::Leave { - namespace, - kill_subscribers, - reply, - }) - .await?; - reply_rx.await??; - Ok(()) - } - - /// Subscribe to replica and sync progress events. - pub async fn subscribe( - &self, - namespace: NamespaceId, - ) -> Result> + Unpin + 'static> { - let content_status_cb = self.content_status_cb.clone(); - - // Create a future that sends channel senders to the respective actors. - // We clone `self` so that the future does not capture any lifetimes. - let this = self.clone(); - - // Subscribe to insert events from the replica. - let a = { - let (s, r) = async_channel::bounded(SUBSCRIBE_CHANNEL_CAP); - this.sync.subscribe(namespace, s).await?; - Box::pin(r).map(move |ev| LiveEvent::from_replica_event(ev, &content_status_cb)) - }; - - // Subscribe to events from the [`live::Actor`]. - let b = { - let (s, r) = async_channel::bounded(SUBSCRIBE_CHANNEL_CAP); - let r = Box::pin(r); - let (reply, reply_rx) = oneshot::channel(); - this.to_live_actor - .send(ToLiveActor::Subscribe { - namespace, - sender: s, - reply, - }) - .await?; - reply_rx.await??; - r.map(|event| Ok(LiveEvent::from(event))) - }; - - Ok(a.or(b)) - } - - /// Handle an incoming iroh-docs connection. - pub async fn handle_connection( - &self, - conn: iroh_net::endpoint::Connecting, - ) -> anyhow::Result<()> { - self.to_live_actor - .send(ToLiveActor::HandleConnection { conn }) - .await?; - Ok(()) - } - - /// Shutdown the engine. - pub async fn shutdown(&self) -> Result<()> { - let (reply, reply_rx) = oneshot::channel(); - self.to_live_actor - .send(ToLiveActor::Shutdown { reply }) - .await?; - reply_rx.await?; - Ok(()) - } -} - -/// Converts an [`EntryStatus`] into a ['ContentStatus']. -pub fn entry_to_content_status(entry: io::Result) -> ContentStatus { - match entry { - Ok(EntryStatus::Complete) => ContentStatus::Complete, - Ok(EntryStatus::Partial) => ContentStatus::Incomplete, - Ok(EntryStatus::NotFound) => ContentStatus::Missing, - Err(cause) => { - tracing::warn!("Error while checking entry status: {cause:?}"); - ContentStatus::Missing - } - } -} - -/// Events informing about actions of the live sync progress. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] -pub enum LiveEvent { - /// A local insertion. - InsertLocal { - /// The inserted entry. - entry: Entry, - }, - /// Received a remote insert. - InsertRemote { - /// The peer that sent us the entry. - from: PublicKey, - /// The inserted entry. - entry: Entry, - /// If the content is available at the local node - content_status: ContentStatus, - }, - /// The content of an entry was downloaded and is now available at the local node - ContentReady { - /// The content hash of the newly available entry content - hash: Hash, - }, - /// All pending content is now ready. - /// - /// This event signals that all queued content downloads from the last sync run have either - /// completed or failed. - /// - /// It will only be emitted after a [`Self::SyncFinished`] event, never before. - /// - /// Receiving this event does not guarantee that all content in the document is available. If - /// blobs failed to download, this event will still be emitted after all operations completed. - PendingContentReady, - /// We have a new neighbor in the swarm. - NeighborUp(PublicKey), - /// We lost a neighbor in the swarm. - NeighborDown(PublicKey), - /// A set-reconciliation sync finished. - SyncFinished(SyncEvent), -} - -impl From for LiveEvent { - fn from(ev: live::Event) -> Self { - match ev { - live::Event::ContentReady { hash } => Self::ContentReady { hash }, - live::Event::NeighborUp(peer) => Self::NeighborUp(peer), - live::Event::NeighborDown(peer) => Self::NeighborDown(peer), - live::Event::SyncFinished(ev) => Self::SyncFinished(ev), - live::Event::PendingContentReady => Self::PendingContentReady, - } - } -} - -impl LiveEvent { - fn from_replica_event( - ev: crate::Event, - content_status_cb: &ContentStatusCallback, - ) -> Result { - Ok(match ev { - crate::Event::LocalInsert { entry, .. } => Self::InsertLocal { - entry: entry.into(), - }, - crate::Event::RemoteInsert { entry, from, .. } => Self::InsertRemote { - content_status: content_status_cb(entry.content_hash()), - entry: entry.into(), - from: PublicKey::from_bytes(&from)?, - }, - }) - } -} - -/// Where to persist the default author. -/// -/// If set to `Mem`, a new author will be created in the docs store before spawning the sync -/// engine. Changing the default author will not be persisted. -/// -/// If set to `Persistent`, the default author will be loaded from and persisted to the specified -/// path (as base32 encoded string of the author's public key). -#[derive(Debug)] -pub enum DefaultAuthorStorage { - /// Memory storage. - Mem, - /// File based persistent storage. - Persistent(PathBuf), -} - -impl DefaultAuthorStorage { - /// Load the default author from the storage. - /// - /// Will create and save a new author if the storage is empty. - /// - /// Returns an error if the author can't be parsed or if the uathor does not exist in the docs - /// store. - pub async fn load(&self, docs_store: &SyncHandle) -> anyhow::Result { - match self { - Self::Mem => { - let author = Author::new(&mut rand::thread_rng()); - let author_id = author.id(); - docs_store.import_author(author).await?; - Ok(author_id) - } - Self::Persistent(ref path) => { - if path.exists() { - let data = tokio::fs::read_to_string(path).await.with_context(|| { - format!( - "Failed to read the default author file at `{}`", - path.to_string_lossy() - ) - })?; - let author_id = AuthorId::from_str(&data).with_context(|| { - format!( - "Failed to parse the default author from `{}`", - path.to_string_lossy() - ) - })?; - if docs_store.export_author(author_id).await?.is_none() { - bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) - } - Ok(author_id) - } else { - let author = Author::new(&mut rand::thread_rng()); - let author_id = author.id(); - docs_store.import_author(author).await?; - // Make sure to write the default author to the store - // *before* we write the default author ID file. - // Otherwise the default author ID file is effectively a dangling reference. - docs_store.flush_store().await?; - self.persist(author_id).await?; - Ok(author_id) - } - } - } - } - - /// Save a new default author. - pub async fn persist(&self, author_id: AuthorId) -> anyhow::Result<()> { - match self { - Self::Mem => { - // persistence is not possible for the mem storage so this is a noop. - } - Self::Persistent(ref path) => { - tokio::fs::write(path, author_id.to_string()) - .await - .with_context(|| { - format!( - "Failed to write the default author to `{}`", - path.to_string_lossy() - ) - })?; - } - } - Ok(()) - } -} - -/// Persistent default author for a docs engine. -#[derive(Debug)] -pub struct DefaultAuthor { - value: RwLock, - storage: DefaultAuthorStorage, -} - -impl DefaultAuthor { - /// Load the default author from storage. - /// - /// If the storage is empty creates a new author and persists it. - pub async fn load(storage: DefaultAuthorStorage, docs_store: &SyncHandle) -> Result { - let value = storage.load(docs_store).await?; - Ok(Self { - value: RwLock::new(value), - storage, - }) - } - - /// Get the current default author. - pub fn get(&self) -> AuthorId { - *self.value.read().unwrap() - } - - /// Set the default author. - pub async fn set(&self, author_id: AuthorId, docs_store: &SyncHandle) -> Result<()> { - if docs_store.export_author(author_id).await?.is_none() { - bail!("The author does not exist"); - } - self.storage.persist(author_id).await?; - *self.value.write().unwrap() = author_id; - Ok(()) - } -} diff --git a/iroh-docs/src/engine/gossip.rs b/iroh-docs/src/engine/gossip.rs deleted file mode 100644 index ff98931c404..00000000000 --- a/iroh-docs/src/engine/gossip.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::collections::{hash_map, HashMap}; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use futures_lite::StreamExt; -use futures_util::FutureExt; -use iroh_gossip::net::{Event, Gossip, GossipEvent, GossipReceiver, GossipSender, JoinOptions}; -use iroh_net::NodeId; -use tokio::{ - sync::mpsc, - task::{AbortHandle, JoinSet}, -}; -use tracing::{debug, instrument, warn}; - -use super::live::{Op, ToLiveActor}; -use crate::{actor::SyncHandle, ContentStatus, NamespaceId}; - -#[derive(Debug)] -struct ActiveState { - sender: GossipSender, - abort_handle: AbortHandle, -} - -#[derive(Debug)] -pub struct GossipState { - gossip: Gossip, - sync: SyncHandle, - to_live_actor: mpsc::Sender, - active: HashMap, - active_tasks: JoinSet<(NamespaceId, Result<()>)>, -} - -impl GossipState { - pub fn new(gossip: Gossip, sync: SyncHandle, to_live_actor: mpsc::Sender) -> Self { - Self { - gossip, - sync, - to_live_actor, - active: Default::default(), - active_tasks: Default::default(), - } - } - - pub async fn join(&mut self, namespace: NamespaceId, bootstrap: Vec) -> Result<()> { - match self.active.entry(namespace) { - hash_map::Entry::Occupied(entry) => { - if !bootstrap.is_empty() { - entry.get().sender.join_peers(bootstrap).await?; - } - } - hash_map::Entry::Vacant(entry) => { - let sub = self - .gossip - .join_with_opts(namespace.into(), JoinOptions::with_bootstrap(bootstrap)); - let (sender, stream) = sub.split(); - let abort_handle = self.active_tasks.spawn( - receive_loop( - namespace, - stream, - self.to_live_actor.clone(), - self.sync.clone(), - ) - .map(move |res| (namespace, res)), - ); - entry.insert(ActiveState { - sender, - abort_handle, - }); - } - } - Ok(()) - } - - pub fn quit(&mut self, topic: &NamespaceId) { - if let Some(state) = self.active.remove(topic) { - state.abort_handle.abort(); - } - } - - pub async fn shutdown(&mut self) -> Result<()> { - for (_, state) in self.active.drain() { - state.abort_handle.abort(); - } - self.progress().await - } - - pub async fn broadcast(&self, namespace: &NamespaceId, message: Bytes) { - if let Some(state) = self.active.get(namespace) { - state.sender.broadcast(message).await.ok(); - } - } - - pub async fn broadcast_neighbors(&self, namespace: &NamespaceId, message: Bytes) { - if let Some(state) = self.active.get(namespace) { - state.sender.broadcast_neighbors(message).await.ok(); - } - } - - pub fn max_message_size(&self) -> usize { - self.gossip.max_message_size() - } - - pub fn is_empty(&self) -> bool { - self.active.is_empty() - } - - /// Progress the internal task queues. - /// - /// Returns an error if any of the active tasks panic. - /// - /// ## Cancel safety - /// - /// This function is fully cancel-safe. - pub async fn progress(&mut self) -> Result<()> { - while let Some(res) = self.active_tasks.join_next().await { - match res { - Err(err) if err.is_cancelled() => continue, - Err(err) => return Err(err).context("gossip receive loop panicked"), - Ok((namespace, res)) => { - self.active.remove(&namespace); - if let Err(err) = res { - warn!(?err, ?namespace, "gossip receive loop failed") - } - } - } - } - Ok(()) - } -} - -#[instrument("gossip-recv", skip_all, fields(namespace=%namespace.fmt_short()))] -async fn receive_loop( - namespace: NamespaceId, - mut recv: GossipReceiver, - to_sync_actor: mpsc::Sender, - sync: SyncHandle, -) -> Result<()> { - for peer in recv.neighbors() { - to_sync_actor - .send(ToLiveActor::NeighborUp { namespace, peer }) - .await?; - } - while let Some(event) = recv.try_next().await? { - let event = match event { - Event::Gossip(event) => event, - Event::Lagged => { - debug!("gossip loop lagged - dropping gossip event"); - continue; - } - }; - match event { - GossipEvent::Received(msg) => { - let op: Op = postcard::from_bytes(&msg.content)?; - match op { - Op::Put(entry) => { - debug!(peer = %msg.delivered_from.fmt_short(), namespace = %namespace.fmt_short(), "received entry via gossip"); - // Insert the entry into our replica. - // If the message was broadcast with neighbor scope, or is received - // directly from the author, we assume that the content is available at - // that peer. Otherwise we don't. - // The download is not triggered here, but in the `on_replica_event` - // handler for the `InsertRemote` event. - let content_status = match msg.scope.is_direct() { - true => ContentStatus::Complete, - false => ContentStatus::Missing, - }; - let from = *msg.delivered_from.as_bytes(); - if let Err(err) = sync - .insert_remote(namespace, entry, from, content_status) - .await - { - debug!("ignoring entry received via gossip: {err}"); - } - } - Op::ContentReady(hash) => { - to_sync_actor - .send(ToLiveActor::NeighborContentReady { - namespace, - node: msg.delivered_from, - hash, - }) - .await?; - } - Op::SyncReport(report) => { - to_sync_actor - .send(ToLiveActor::IncomingSyncReport { - from: msg.delivered_from, - report, - }) - .await?; - } - } - } - GossipEvent::NeighborUp(peer) => { - to_sync_actor - .send(ToLiveActor::NeighborUp { namespace, peer }) - .await?; - } - GossipEvent::NeighborDown(peer) => { - to_sync_actor - .send(ToLiveActor::NeighborDown { namespace, peer }) - .await?; - } - GossipEvent::Joined(peers) => { - for peer in peers { - to_sync_actor - .send(ToLiveActor::NeighborUp { namespace, peer }) - .await?; - } - } - } - } - Ok(()) -} diff --git a/iroh-docs/src/engine/live.rs b/iroh-docs/src/engine/live.rs deleted file mode 100644 index 100ed0a5f2a..00000000000 --- a/iroh-docs/src/engine/live.rs +++ /dev/null @@ -1,963 +0,0 @@ -#![allow(missing_docs)] - -use std::{ - collections::{HashMap, HashSet}, - time::SystemTime, -}; - -use anyhow::{Context, Result}; -use futures_lite::FutureExt; -use iroh_blobs::{ - downloader::{DownloadError, DownloadRequest, Downloader}, - get::Stats, - store::EntryStatus, - Hash, HashAndFormat, -}; -use iroh_gossip::net::Gossip; -use iroh_metrics::inc; -use iroh_net::{key::PublicKey, Endpoint, NodeAddr, NodeId}; -use serde::{Deserialize, Serialize}; -use tokio::{ - sync::{self, mpsc, oneshot}, - task::JoinSet, -}; -use tracing::{debug, error, info, instrument, trace, warn, Instrument, Span}; - -// use super::gossip::{GossipActor, ToGossipActor}; -use super::state::{NamespaceStates, Origin, SyncReason}; -use crate::{ - actor::{OpenOpts, SyncHandle}, - engine::gossip::GossipState, - metrics::Metrics, - net::{ - connect_and_sync, handle_connection, AbortReason, AcceptError, AcceptOutcome, ConnectError, - SyncFinished, - }, - AuthorHeads, ContentStatus, NamespaceId, SignedEntry, -}; - -/// Name used for logging when new node addresses are added from the docs engine. -const SOURCE_NAME: &str = "docs_engine"; - -/// An iroh-docs operation -/// -/// This is the message that is broadcast over iroh-gossip. -#[derive(Debug, Clone, Serialize, Deserialize, strum::Display)] -pub enum Op { - /// A new entry was inserted into the document. - Put(SignedEntry), - /// A peer now has content available for a hash. - ContentReady(Hash), - /// We synced with another peer, here's the news. - SyncReport(SyncReport), -} - -/// Report of a successful sync with the new heads. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SyncReport { - namespace: NamespaceId, - /// Encoded [`AuthorHeads`] - heads: Vec, -} - -/// Messages to the sync actor -#[derive(derive_more::Debug, strum::Display)] -pub enum ToLiveActor { - StartSync { - namespace: NamespaceId, - peers: Vec, - #[debug("onsehot::Sender")] - reply: sync::oneshot::Sender>, - }, - Leave { - namespace: NamespaceId, - kill_subscribers: bool, - #[debug("onsehot::Sender")] - reply: sync::oneshot::Sender>, - }, - Shutdown { - reply: sync::oneshot::Sender<()>, - }, - Subscribe { - namespace: NamespaceId, - #[debug("sender")] - sender: async_channel::Sender, - #[debug("oneshot::Sender")] - reply: sync::oneshot::Sender>, - }, - HandleConnection { - conn: iroh_net::endpoint::Connecting, - }, - AcceptSyncRequest { - namespace: NamespaceId, - peer: PublicKey, - #[debug("oneshot::Sender")] - reply: sync::oneshot::Sender, - }, - - IncomingSyncReport { - from: PublicKey, - report: SyncReport, - }, - NeighborContentReady { - namespace: NamespaceId, - node: PublicKey, - hash: Hash, - }, - NeighborUp { - namespace: NamespaceId, - peer: PublicKey, - }, - NeighborDown { - namespace: NamespaceId, - peer: PublicKey, - }, -} - -/// Events informing about actions of the live sync progress. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] -pub enum Event { - /// The content of an entry was downloaded and is now available at the local node - ContentReady { - /// The content hash of the newly available entry content - hash: Hash, - }, - /// We have a new neighbor in the swarm. - NeighborUp(PublicKey), - /// We lost a neighbor in the swarm. - NeighborDown(PublicKey), - /// A set-reconciliation sync finished. - SyncFinished(SyncEvent), - /// All pending content is now ready. - /// - /// This event is only emitted after a sync completed and `Self::SyncFinished` was emitted at - /// least once. It signals that all currently pending downloads have been completed. - /// - /// Receiving this event does not guarantee that all content in the document is available. If - /// blobs failed to download, this event will still be emitted after all operations completed. - PendingContentReady, -} - -type SyncConnectRes = ( - NamespaceId, - PublicKey, - SyncReason, - Result, -); -type SyncAcceptRes = Result; -type DownloadRes = (NamespaceId, Hash, Result); - -// Currently peers might double-sync in both directions. -pub struct LiveActor { - /// Receiver for actor messages. - inbox: mpsc::Receiver, - sync: SyncHandle, - endpoint: Endpoint, - bao_store: B, - downloader: Downloader, - replica_events_tx: async_channel::Sender, - replica_events_rx: async_channel::Receiver, - - /// Send messages to self. - /// Note: Must not be used in methods called from `Self::run` directly to prevent deadlocks. - /// Only clone into newly spawned tasks. - sync_actor_tx: mpsc::Sender, - gossip: GossipState, - - /// Running sync futures (from connect). - running_sync_connect: JoinSet, - /// Running sync futures (from accept). - running_sync_accept: JoinSet, - /// Running download futures. - download_tasks: JoinSet, - /// Content hashes which are wanted but not yet queued because no provider was found. - missing_hashes: HashSet, - /// Content hashes queued in downloader. - queued_hashes: QueuedHashes, - - /// Subscribers to actor events - subscribers: SubscribersMap, - - /// Sync state per replica and peer - state: NamespaceStates, -} -impl LiveActor { - /// Create the live actor. - #[allow(clippy::too_many_arguments)] - pub fn new( - sync: SyncHandle, - endpoint: Endpoint, - gossip: Gossip, - bao_store: B, - downloader: Downloader, - inbox: mpsc::Receiver, - sync_actor_tx: mpsc::Sender, - ) -> Self { - let (replica_events_tx, replica_events_rx) = async_channel::bounded(1024); - let gossip_state = GossipState::new(gossip, sync.clone(), sync_actor_tx.clone()); - Self { - inbox, - sync, - replica_events_rx, - replica_events_tx, - endpoint, - gossip: gossip_state, - bao_store, - downloader, - sync_actor_tx, - running_sync_connect: Default::default(), - running_sync_accept: Default::default(), - subscribers: Default::default(), - download_tasks: Default::default(), - state: Default::default(), - missing_hashes: Default::default(), - queued_hashes: Default::default(), - } - } - - /// Run the actor loop. - pub async fn run(mut self) -> Result<()> { - let shutdown_reply = self.run_inner().await; - if let Err(err) = self.shutdown().await { - error!(?err, "Error during shutdown"); - } - drop(self); - match shutdown_reply { - Ok(reply) => { - reply.send(()).ok(); - Ok(()) - } - Err(err) => Err(err), - } - } - - async fn run_inner(&mut self) -> Result> { - let mut i = 0; - loop { - i += 1; - trace!(?i, "tick wait"); - inc!(Metrics, doc_live_tick_main); - tokio::select! { - biased; - msg = self.inbox.recv() => { - let msg = msg.context("to_actor closed")?; - trace!(?i, %msg, "tick: to_actor"); - inc!(Metrics, doc_live_tick_actor); - match msg { - ToLiveActor::Shutdown { reply } => { - break Ok(reply); - } - msg => { - self.on_actor_message(msg).await.context("on_actor_message")?; - } - } - } - event = self.replica_events_rx.recv() => { - trace!(?i, "tick: replica_event"); - inc!(Metrics, doc_live_tick_replica_event); - let event = event.context("replica_events closed")?; - if let Err(err) = self.on_replica_event(event).await { - error!(?err, "Failed to process replica event"); - } - } - Some(res) = self.running_sync_connect.join_next(), if !self.running_sync_connect.is_empty() => { - trace!(?i, "tick: running_sync_connect"); - inc!(Metrics, doc_live_tick_running_sync_connect); - let (namespace, peer, reason, res) = res.context("running_sync_connect closed")?; - self.on_sync_via_connect_finished(namespace, peer, reason, res).await; - - } - Some(res) = self.running_sync_accept.join_next(), if !self.running_sync_accept.is_empty() => { - trace!(?i, "tick: running_sync_accept"); - inc!(Metrics, doc_live_tick_running_sync_accept); - let res = res.context("running_sync_accept closed")?; - self.on_sync_via_accept_finished(res).await; - } - Some(res) = self.download_tasks.join_next(), if !self.download_tasks.is_empty() => { - trace!(?i, "tick: pending_downloads"); - inc!(Metrics, doc_live_tick_pending_downloads); - let (namespace, hash, res) = res.context("pending_downloads closed")?; - self.on_download_ready(namespace, hash, res).await; - } - res = self.gossip.progress(), if !self.gossip.is_empty() => { - if let Err(error) = res { - warn!(?error, "gossip state failed"); - } - } - } - } - } - - async fn on_actor_message(&mut self, msg: ToLiveActor) -> anyhow::Result { - match msg { - ToLiveActor::Shutdown { .. } => { - unreachable!("handled in run"); - } - ToLiveActor::IncomingSyncReport { from, report } => { - self.on_sync_report(from, report).await - } - ToLiveActor::NeighborUp { namespace, peer } => { - debug!(peer = %peer.fmt_short(), namespace = %namespace.fmt_short(), "neighbor up"); - self.sync_with_peer(namespace, peer, SyncReason::NewNeighbor); - self.subscribers - .send(&namespace, Event::NeighborUp(peer)) - .await; - } - ToLiveActor::NeighborDown { namespace, peer } => { - debug!(peer = %peer.fmt_short(), namespace = %namespace.fmt_short(), "neighbor down"); - self.subscribers - .send(&namespace, Event::NeighborDown(peer)) - .await; - } - ToLiveActor::StartSync { - namespace, - peers, - reply, - } => { - let res = self.start_sync(namespace, peers).await; - reply.send(res).ok(); - } - ToLiveActor::Leave { - namespace, - kill_subscribers, - reply, - } => { - let res = self.leave(namespace, kill_subscribers).await; - reply.send(res).ok(); - } - ToLiveActor::Subscribe { - namespace, - sender, - reply, - } => { - self.subscribers.subscribe(namespace, sender); - reply.send(Ok(())).ok(); - } - ToLiveActor::HandleConnection { conn } => { - self.handle_connection(conn).await; - } - ToLiveActor::AcceptSyncRequest { - namespace, - peer, - reply, - } => { - let outcome = self.accept_sync_request(namespace, peer); - reply.send(outcome).ok(); - } - ToLiveActor::NeighborContentReady { - namespace, - node, - hash, - } => { - self.on_neighbor_content_ready(namespace, node, hash).await; - } - }; - Ok(true) - } - - #[instrument("connect", skip_all, fields(peer = %peer.fmt_short(), namespace = %namespace.fmt_short()))] - fn sync_with_peer(&mut self, namespace: NamespaceId, peer: PublicKey, reason: SyncReason) { - if !self.state.start_connect(&namespace, peer, reason) { - return; - } - let endpoint = self.endpoint.clone(); - let sync = self.sync.clone(); - let fut = async move { - let res = connect_and_sync(&endpoint, &sync, namespace, NodeAddr::new(peer)).await; - (namespace, peer, reason, res) - } - .instrument(Span::current()); - self.running_sync_connect.spawn(fut); - } - - async fn shutdown(&mut self) -> anyhow::Result<()> { - // cancel all subscriptions - self.subscribers.clear(); - let (gossip_shutdown_res, _store) = tokio::join!( - // quit the gossip topics and task loops. - self.gossip.shutdown(), - // shutdown sync thread - self.sync.shutdown() - ); - gossip_shutdown_res?; - // TODO: abort_all and join_next all JoinSets to catch panics - // (they are aborted on drop, but that swallows panics) - Ok(()) - } - - async fn start_sync(&mut self, namespace: NamespaceId, mut peers: Vec) -> Result<()> { - debug!(?namespace, peers = peers.len(), "start sync"); - // update state to allow sync - if !self.state.is_syncing(&namespace) { - let opts = OpenOpts::default() - .sync() - .subscribe(self.replica_events_tx.clone()); - self.sync.open(namespace, opts).await?; - self.state.insert(namespace); - } - // add the peers stored for this document - match self.sync.get_sync_peers(namespace).await { - Ok(None) => { - // no peers for this document - } - Ok(Some(known_useful_peers)) => { - let as_node_addr = known_useful_peers.into_iter().filter_map(|peer_id_bytes| { - // peers are stored as bytes, don't fail the operation if they can't be - // decoded: simply ignore the peer - match PublicKey::from_bytes(&peer_id_bytes) { - Ok(public_key) => Some(NodeAddr::new(public_key)), - Err(_signing_error) => { - warn!("potential db corruption: peers per doc can't be decoded"); - None - } - } - }); - peers.extend(as_node_addr); - } - Err(e) => { - // try to continue if peers per doc can't be read since they are not vital for sync - warn!(%e, "db error reading peers per document") - } - } - self.join_peers(namespace, peers).await?; - Ok(()) - } - - async fn leave( - &mut self, - namespace: NamespaceId, - kill_subscribers: bool, - ) -> anyhow::Result<()> { - // self.subscribers.remove(&namespace); - if self.state.remove(&namespace) { - self.sync.set_sync(namespace, false).await?; - self.sync - .unsubscribe(namespace, self.replica_events_tx.clone()) - .await?; - self.sync.close(namespace).await?; - self.gossip.quit(&namespace); - } - if kill_subscribers { - self.subscribers.remove(&namespace); - } - Ok(()) - } - - async fn join_peers(&mut self, namespace: NamespaceId, peers: Vec) -> Result<()> { - let mut peer_ids = Vec::new(); - - // add addresses of peers to our endpoint address book - for peer in peers.into_iter() { - let peer_id = peer.node_id; - // adding a node address without any addressing info fails with an error, - // but we still want to include those peers because node discovery might find addresses for them - if peer.info.is_empty() { - peer_ids.push(peer_id) - } else { - match self.endpoint.add_node_addr_with_source(peer, SOURCE_NAME) { - Ok(()) => { - peer_ids.push(peer_id); - } - Err(err) => { - warn!(peer = %peer_id.fmt_short(), "failed to add known addrs: {err:?}"); - } - } - } - } - - // tell gossip to join - self.gossip.join(namespace, peer_ids.clone()).await?; - - if !peer_ids.is_empty() { - // trigger initial sync with initial peers - for peer in peer_ids { - self.sync_with_peer(namespace, peer, SyncReason::DirectJoin); - } - } - Ok(()) - } - - #[instrument("connect", skip_all, fields(peer = %peer.fmt_short(), namespace = %namespace.fmt_short()))] - async fn on_sync_via_connect_finished( - &mut self, - namespace: NamespaceId, - peer: PublicKey, - reason: SyncReason, - result: Result, - ) { - match result { - Err(ConnectError::RemoteAbort(AbortReason::AlreadySyncing)) => { - debug!(?reason, "remote abort, already syncing"); - } - res => { - self.on_sync_finished( - namespace, - peer, - Origin::Connect(reason), - res.map_err(Into::into), - ) - .await - } - } - } - - #[instrument("accept", skip_all, fields(peer = %fmt_accept_peer(&res), namespace = %fmt_accept_namespace(&res)))] - async fn on_sync_via_accept_finished(&mut self, res: Result) { - match res { - Ok(state) => { - self.on_sync_finished(state.namespace, state.peer, Origin::Accept, Ok(state)) - .await - } - Err(AcceptError::Abort { reason, .. }) if reason == AbortReason::AlreadySyncing => { - // In case we aborted the sync: do nothing (our outgoing sync is in progress) - debug!(?reason, "aborted by us"); - } - Err(err) => { - if let (Some(peer), Some(namespace)) = (err.peer(), err.namespace()) { - self.on_sync_finished( - namespace, - peer, - Origin::Accept, - Err(anyhow::Error::from(err)), - ) - .await; - } else { - debug!(?err, "failed before reading the first message"); - } - } - } - } - - async fn on_sync_finished( - &mut self, - namespace: NamespaceId, - peer: PublicKey, - origin: Origin, - result: Result, - ) { - match &result { - Err(ref err) => { - warn!(?origin, ?err, "sync failed"); - } - Ok(ref details) => { - info!( - sent = %details.outcome.num_sent, - recv = %details.outcome.num_recv, - t_connect = ?details.timings.connect, - t_process = ?details.timings.process, - "sync finished", - ); - - // register the peer as useful for the document - if let Err(e) = self - .sync - .register_useful_peer(namespace, *peer.as_bytes()) - .await - { - debug!(%e, "failed to register peer for document") - } - - // broadcast a sync report to our neighbors, but only if we received new entries. - if details.outcome.num_recv > 0 { - info!("broadcast sync report to neighbors"); - match details - .outcome - .heads_received - .encode(Some(self.gossip.max_message_size())) - { - Err(err) => warn!(?err, "Failed to encode author heads for sync report"), - Ok(heads) => { - let report = SyncReport { namespace, heads }; - self.broadcast_neighbors(namespace, &Op::SyncReport(report)) - .await; - } - } - } - } - }; - - let result_for_event = match &result { - Ok(details) => Ok(details.into()), - Err(err) => Err(err.to_string()), - }; - - let Some((started, resync)) = self.state.finish(&namespace, peer, &origin, result) else { - return; - }; - - let ev = SyncEvent { - peer, - origin, - result: result_for_event, - finished: SystemTime::now(), - started, - }; - self.subscribers - .send(&namespace, Event::SyncFinished(ev)) - .await; - - // Check if there are queued pending content hashes for this namespace. - // If hashes are pending, mark this namespace to be eglible for a PendingContentReady event once all - // pending hashes have completed downloading. - // If no hashes are pending, emit the PendingContentReady event right away. The next - // PendingContentReady event may then only be emitted after the next sync completes. - if self.queued_hashes.contains_namespace(&namespace) { - self.state.set_may_emit_ready(&namespace, true); - } else { - self.subscribers - .send(&namespace, Event::PendingContentReady) - .await; - self.state.set_may_emit_ready(&namespace, false); - } - - if resync { - self.sync_with_peer(namespace, peer, SyncReason::Resync); - } - } - - async fn broadcast_neighbors(&self, namespace: NamespaceId, op: &Op) { - if !self.state.is_syncing(&namespace) { - return; - } - - let msg = match postcard::to_stdvec(op) { - Ok(msg) => msg, - Err(err) => { - error!(?err, ?op, "Failed to serialize message:"); - return; - } - }; - // TODO: We should debounce and merge these neighbor announcements likely. - self.gossip - .broadcast_neighbors(&namespace, msg.into()) - .await; - } - - async fn on_download_ready( - &mut self, - namespace: NamespaceId, - hash: Hash, - res: Result, - ) { - let completed_namespaces = self.queued_hashes.remove_hash(&hash); - debug!(namespace=%namespace.fmt_short(), success=res.is_ok(), completed_namespaces=completed_namespaces.len(), "download ready"); - if res.is_ok() { - self.subscribers - .send(&namespace, Event::ContentReady { hash }) - .await; - // Inform our neighbors that we have new content ready. - self.broadcast_neighbors(namespace, &Op::ContentReady(hash)) - .await; - } else { - self.missing_hashes.insert(hash); - } - for namespace in completed_namespaces.iter() { - if let Some(true) = self.state.may_emit_ready(namespace) { - self.subscribers - .send(namespace, Event::PendingContentReady) - .await; - } - } - } - - async fn on_neighbor_content_ready( - &mut self, - namespace: NamespaceId, - node: NodeId, - hash: Hash, - ) { - self.start_download(namespace, hash, node, true).await; - } - - #[instrument("on_sync_report", skip_all, fields(peer = %from.fmt_short(), namespace = %report.namespace.fmt_short()))] - async fn on_sync_report(&mut self, from: PublicKey, report: SyncReport) { - let namespace = report.namespace; - if !self.state.is_syncing(&namespace) { - return; - } - let heads = match AuthorHeads::decode(&report.heads) { - Ok(heads) => heads, - Err(err) => { - warn!(?err, "failed to decode AuthorHeads"); - return; - } - }; - match self.sync.has_news_for_us(report.namespace, heads).await { - Ok(Some(updated_authors)) => { - info!(%updated_authors, "news reported: sync now"); - self.sync_with_peer(report.namespace, from, SyncReason::SyncReport); - } - Ok(None) => { - debug!("no news reported: nothing to do"); - } - Err(err) => { - warn!("sync actor error: {err:?}"); - } - } - } - - async fn on_replica_event(&mut self, event: crate::Event) -> Result<()> { - match event { - crate::Event::LocalInsert { namespace, entry } => { - debug!(namespace=%namespace.fmt_short(), "replica event: LocalInsert"); - // A new entry was inserted locally. Broadcast a gossip message. - if self.state.is_syncing(&namespace) { - let op = Op::Put(entry.clone()); - let message = postcard::to_stdvec(&op)?.into(); - self.gossip.broadcast(&namespace, message).await; - } - } - crate::Event::RemoteInsert { - namespace, - entry, - from, - should_download, - remote_content_status, - } => { - debug!(namespace=%namespace.fmt_short(), "replica event: RemoteInsert"); - // A new entry was inserted from initial sync or gossip. Queue downloading the - // content. - if should_download { - let hash = entry.content_hash(); - if matches!(remote_content_status, ContentStatus::Complete) { - let node_id = PublicKey::from_bytes(&from)?; - self.start_download(namespace, hash, node_id, false).await; - } else { - self.missing_hashes.insert(hash); - } - } - } - } - - Ok(()) - } - - async fn start_download( - &mut self, - namespace: NamespaceId, - hash: Hash, - node: PublicKey, - only_if_missing: bool, - ) { - let entry_status = self.bao_store.entry_status(&hash).await; - if matches!(entry_status, Ok(EntryStatus::Complete)) { - self.missing_hashes.remove(&hash); - return; - } - if self.queued_hashes.contains_hash(&hash) { - self.queued_hashes.insert(hash, namespace); - self.downloader.nodes_have(hash, vec![node]).await; - } else if !only_if_missing || self.missing_hashes.contains(&hash) { - let req = DownloadRequest::new(HashAndFormat::raw(hash), vec![node]); - let handle = self.downloader.queue(req).await; - - self.queued_hashes.insert(hash, namespace); - self.missing_hashes.remove(&hash); - self.download_tasks - .spawn(async move { (namespace, hash, handle.await) }); - } - } - - #[instrument("accept", skip_all)] - pub async fn handle_connection(&mut self, conn: iroh_net::endpoint::Connecting) { - let to_actor_tx = self.sync_actor_tx.clone(); - let accept_request_cb = move |namespace, peer| { - let to_actor_tx = to_actor_tx.clone(); - async move { - let (reply_tx, reply_rx) = oneshot::channel(); - to_actor_tx - .send(ToLiveActor::AcceptSyncRequest { - namespace, - peer, - reply: reply_tx, - }) - .await - .ok(); - match reply_rx.await { - Ok(outcome) => outcome, - Err(err) => { - warn!( - "accept request callback failed to retrieve reply from actor: {err:?}" - ); - AcceptOutcome::Reject(AbortReason::InternalServerError) - } - } - } - .boxed() - }; - debug!("incoming connection"); - let sync = self.sync.clone(); - self.running_sync_accept.spawn( - async move { handle_connection(sync, conn, accept_request_cb).await } - .instrument(Span::current()), - ); - } - - pub fn accept_sync_request( - &mut self, - namespace: NamespaceId, - peer: PublicKey, - ) -> AcceptOutcome { - self.state - .accept_request(&self.endpoint.node_id(), &namespace, peer) - } -} - -/// Event emitted when a sync operation completes -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct SyncEvent { - /// Peer we synced with - pub peer: PublicKey, - /// Origin of the sync exchange - pub origin: Origin, - /// Timestamp when the sync started - pub finished: SystemTime, - /// Timestamp when the sync finished - pub started: SystemTime, - /// Result of the sync operation - pub result: std::result::Result, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub struct SyncDetails { - /// Number of entries received - pub entries_received: usize, - /// Number of entries sent - pub entries_sent: usize, -} - -impl From<&SyncFinished> for SyncDetails { - fn from(value: &SyncFinished) -> Self { - Self { - entries_received: value.outcome.num_recv, - entries_sent: value.outcome.num_sent, - } - } -} - -#[derive(Debug, Default)] -struct SubscribersMap(HashMap); - -impl SubscribersMap { - fn subscribe(&mut self, namespace: NamespaceId, sender: async_channel::Sender) { - self.0.entry(namespace).or_default().subscribe(sender); - } - - async fn send(&mut self, namespace: &NamespaceId, event: Event) -> bool { - debug!(namespace=%namespace.fmt_short(), %event, "emit event"); - let Some(subscribers) = self.0.get_mut(namespace) else { - return false; - }; - - if !subscribers.send(event).await { - self.0.remove(namespace); - } - true - } - - fn remove(&mut self, namespace: &NamespaceId) { - self.0.remove(namespace); - } - - fn clear(&mut self) { - self.0.clear(); - } -} - -#[derive(Debug, Default)] -struct QueuedHashes { - by_hash: HashMap>, - by_namespace: HashMap>, -} - -impl QueuedHashes { - fn insert(&mut self, hash: Hash, namespace: NamespaceId) { - self.by_hash.entry(hash).or_default().insert(namespace); - self.by_namespace.entry(namespace).or_default().insert(hash); - } - - /// Remove a hash from the set of queued hashes. - /// - /// Returns a list of namespaces that are now complete (have no queued hashes anymore). - fn remove_hash(&mut self, hash: &Hash) -> Vec { - let namespaces = self.by_hash.remove(hash).unwrap_or_default(); - let mut removed_namespaces = vec![]; - for namespace in namespaces { - if let Some(hashes) = self.by_namespace.get_mut(&namespace) { - hashes.remove(hash); - if hashes.is_empty() { - self.by_namespace.remove(&namespace); - removed_namespaces.push(namespace); - } - } - } - removed_namespaces - } - - fn contains_hash(&self, hash: &Hash) -> bool { - self.by_hash.contains_key(hash) - } - - fn contains_namespace(&self, namespace: &NamespaceId) -> bool { - self.by_namespace.contains_key(namespace) - } -} - -#[derive(Debug, Default)] -struct Subscribers(Vec>); - -impl Subscribers { - fn subscribe(&mut self, sender: async_channel::Sender) { - self.0.push(sender) - } - - async fn send(&mut self, event: Event) -> bool { - let futs = self.0.iter().map(|sender| sender.send(event.clone())); - let res = futures_buffered::join_all(futs).await; - // reverse the order so removing does not shift remaining indices - for (i, res) in res.into_iter().enumerate().rev() { - if res.is_err() { - self.0.remove(i); - } - } - !self.0.is_empty() - } -} - -fn fmt_accept_peer(res: &Result) -> String { - match res { - Ok(res) => res.peer.fmt_short(), - Err(err) => err - .peer() - .map(|x| x.fmt_short()) - .unwrap_or_else(|| "unknown".to_string()), - } -} - -fn fmt_accept_namespace(res: &Result) -> String { - match res { - Ok(res) => res.namespace.fmt_short(), - Err(err) => err - .namespace() - .map(|x| x.fmt_short()) - .unwrap_or_else(|| "unknown".to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_sync_remove() { - let pk = PublicKey::from_bytes(&[1; 32]).unwrap(); - let (a_tx, a_rx) = async_channel::unbounded(); - let (b_tx, b_rx) = async_channel::unbounded(); - let mut subscribers = Subscribers::default(); - subscribers.subscribe(a_tx); - subscribers.subscribe(b_tx); - drop(a_rx); - drop(b_rx); - subscribers.send(Event::NeighborUp(pk)).await; - } -} diff --git a/iroh-docs/src/engine/state.rs b/iroh-docs/src/engine/state.rs deleted file mode 100644 index 83dc4ef9932..00000000000 --- a/iroh-docs/src/engine/state.rs +++ /dev/null @@ -1,260 +0,0 @@ -use std::{ - collections::BTreeMap, - time::{Instant, SystemTime}, -}; - -use anyhow::Result; -use iroh_net::NodeId; -use serde::{Deserialize, Serialize}; -use tracing::{debug, warn}; - -use crate::{ - net::{AbortReason, AcceptOutcome, SyncFinished}, - NamespaceId, -}; - -/// Why we started a sync request -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Copy)] -pub enum SyncReason { - /// Direct join request via API - DirectJoin, - /// Peer showed up as new neighbor in the gossip swarm - NewNeighbor, - /// We synced after receiving a sync report that indicated news for us - SyncReport, - /// We received a sync report while a sync was running, so run again afterwars - Resync, -} - -/// Why we performed a sync exchange -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -pub enum Origin { - /// We initiated the exchange - Connect(SyncReason), - /// A node connected to us and we accepted the exchange - Accept, -} - -/// The state we're in for a node and a namespace -#[derive(Debug, Clone)] -pub enum SyncState { - Idle, - Running { start: SystemTime, origin: Origin }, -} - -impl Default for SyncState { - fn default() -> Self { - Self::Idle - } -} - -/// Contains an entry for each active (syncing) namespace, and in there an entry for each node we -/// synced with. -#[derive(Default)] -pub struct NamespaceStates(BTreeMap); - -#[derive(Default)] -struct NamespaceState { - nodes: BTreeMap, - may_emit_ready: bool, -} - -impl NamespaceStates { - /// Are we syncing this namespace? - pub fn is_syncing(&self, namespace: &NamespaceId) -> bool { - self.0.contains_key(namespace) - } - - /// Insert a namespace into the set of syncing namespaces. - pub fn insert(&mut self, namespace: NamespaceId) { - self.0.entry(namespace).or_default(); - } - - /// Start a sync request. - /// - /// Returns true if the request should be performed, and false if it should be aborted. - pub fn start_connect( - &mut self, - namespace: &NamespaceId, - node: NodeId, - reason: SyncReason, - ) -> bool { - match self.entry(namespace, node) { - None => { - debug!("abort connect: namespace is not in sync set"); - false - } - Some(state) => state.start_connect(reason), - } - } - - /// Accept a sync request. - /// - /// Returns the [`AcceptOutcome`] to be performed. - pub fn accept_request( - &mut self, - me: &NodeId, - namespace: &NamespaceId, - node: NodeId, - ) -> AcceptOutcome { - let Some(state) = self.entry(namespace, node) else { - return AcceptOutcome::Reject(AbortReason::NotFound); - }; - state.accept_request(me, &node) - } - - /// Insert a finished sync operation into the state. - /// - /// Returns the time when the operation was started, and a `bool` that is true if another sync - /// request should be triggered right afterwards. - /// - /// Returns `None` if the namespace is not syncing or the sync state doesn't expect a finish - /// event. - pub fn finish( - &mut self, - namespace: &NamespaceId, - node: NodeId, - origin: &Origin, - result: Result, - ) -> Option<(SystemTime, bool)> { - let state = self.entry(namespace, node)?; - state.finish(origin, result) - } - - /// Set whether a [`super::live::Event::PendingContentReady`] may be emitted once the pending queue - /// becomes empty. - /// - /// This should be set to `true` if there are pending content hashes after a sync finished, and - /// to `false` whenever a `PendingContentReady` was emitted. - pub fn set_may_emit_ready(&mut self, namespace: &NamespaceId, value: bool) -> Option<()> { - let state = self.0.get_mut(namespace)?; - state.may_emit_ready = value; - Some(()) - } - /// Returns whether a [`super::live::Event::PendingContentReady`] event may be emitted once the - /// pending queue becomes empty. - /// - /// If this returns `false`, an event should not be emitted even if the queue becomes empty, - /// because a currently running sync did not yet terminate. Once it terminates, the event will - /// be emitted from the handler for finished syncs. - pub fn may_emit_ready(&mut self, namespace: &NamespaceId) -> Option { - let state = self.0.get_mut(namespace)?; - if state.may_emit_ready { - state.may_emit_ready = false; - Some(true) - } else { - Some(false) - } - } - - /// Remove a namespace from the set of syncing namespaces. - pub fn remove(&mut self, namespace: &NamespaceId) -> bool { - self.0.remove(namespace).is_some() - } - - /// Get the [`PeerState`] for a namespace and node. - /// If the namespace is syncing and the node so far unknown, initialize and return a default [`PeerState`]. - /// If the namespace is not syncing return None. - fn entry(&mut self, namespace: &NamespaceId, node: NodeId) -> Option<&mut PeerState> { - self.0 - .get_mut(namespace) - .map(|n| n.nodes.entry(node).or_default()) - } -} - -/// State of a node with regard to a namespace. -#[derive(Default)] -struct PeerState { - state: SyncState, - resync_requested: bool, - last_sync: Option<(Instant, Result)>, -} - -impl PeerState { - fn finish( - &mut self, - origin: &Origin, - result: Result, - ) -> Option<(SystemTime, bool)> { - let start = match &self.state { - SyncState::Running { - start, - origin: origin2, - } => { - if origin2 != origin { - warn!(actual = ?origin, expected = ?origin2, "finished sync origin does not match state") - } - Some(*start) - } - SyncState::Idle => { - warn!("sync state finish called but not in running state"); - None - } - }; - - self.last_sync = Some((Instant::now(), result)); - self.state = SyncState::Idle; - start.map(|s| (s, self.resync_requested)) - } - - fn start_connect(&mut self, reason: SyncReason) -> bool { - debug!(?reason, "start connect"); - match self.state { - // never run two syncs at the same time - SyncState::Running { .. } => { - debug!("abort connect: sync already running"); - if matches!(reason, SyncReason::SyncReport) { - debug!("resync queued"); - self.resync_requested = true; - } - false - } - SyncState::Idle => { - self.set_sync_running(Origin::Connect(reason)); - true - } - } - } - - fn accept_request(&mut self, me: &NodeId, node: &NodeId) -> AcceptOutcome { - let outcome = match &self.state { - SyncState::Idle => AcceptOutcome::Allow, - SyncState::Running { origin, .. } => match origin { - Origin::Accept => AcceptOutcome::Reject(AbortReason::AlreadySyncing), - // Incoming sync request while we are dialing ourselves. - // In this case, compare the binary representations of our and the other node's id - // to deterministically decide which of the two concurrent connections will succeed. - Origin::Connect(_reason) => match expected_sync_direction(me, node) { - SyncDirection::Accept => AcceptOutcome::Allow, - SyncDirection::Connect => AcceptOutcome::Reject(AbortReason::AlreadySyncing), - }, - }, - }; - if let AcceptOutcome::Allow = outcome { - self.set_sync_running(Origin::Accept); - } - outcome - } - - fn set_sync_running(&mut self, origin: Origin) { - self.state = SyncState::Running { - origin, - start: SystemTime::now(), - }; - self.resync_requested = false; - } -} - -#[derive(Debug)] -enum SyncDirection { - Accept, - Connect, -} - -fn expected_sync_direction(self_node_id: &NodeId, other_node_id: &NodeId) -> SyncDirection { - if self_node_id.as_bytes() > other_node_id.as_bytes() { - SyncDirection::Accept - } else { - SyncDirection::Connect - } -} diff --git a/iroh-docs/src/heads.rs b/iroh-docs/src/heads.rs deleted file mode 100644 index baa923ae4b4..00000000000 --- a/iroh-docs/src/heads.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! Author heads - -use std::{collections::BTreeMap, num::NonZeroU64}; - -use anyhow::Result; - -use crate::AuthorId; - -type Timestamp = u64; - -/// Timestamps of the latest entry for each author. -#[derive(Debug, Clone, Eq, PartialEq, Default)] -pub struct AuthorHeads { - heads: BTreeMap, -} - -impl AuthorHeads { - /// Insert a new timestamp. - pub fn insert(&mut self, author: AuthorId, timestamp: Timestamp) { - self.heads - .entry(author) - .and_modify(|t| *t = (*t).max(timestamp)) - .or_insert(timestamp); - } - - /// Number of author-timestamp pairs. - pub fn len(&self) -> usize { - self.heads.len() - } - - /// Whether this [`AuthorHeads`] is empty. - pub fn is_empty(&self) -> bool { - self.heads.is_empty() - } - - /// Get the timestamp for an author. - pub fn get(&self, author: &AuthorId) -> Option { - self.heads.get(author).copied() - } - - /// Can this state offer newer stuff to `other`? - pub fn has_news_for(&self, other: &Self) -> Option { - let mut updates = 0; - for (author, ts_ours) in self.iter() { - if other - .get(author) - .map(|ts_theirs| *ts_ours > ts_theirs) - .unwrap_or(true) - { - updates += 1; - } - } - NonZeroU64::new(updates) - } - - /// Merge another author head state into this one. - pub fn merge(&mut self, other: &Self) { - for (a, t) in other.iter() { - self.insert(*a, *t); - } - } - - /// Create an iterator over the entries in this state. - pub fn iter(&self) -> std::collections::btree_map::Iter { - self.heads.iter() - } - - /// Encode into a byte array with a limited size. - /// - /// Will skip oldest entries if the size limit is reached. - /// Returns a byte array with a maximum length of `size_limit`. - pub fn encode(&self, size_limit: Option) -> Result> { - let mut by_timestamp = BTreeMap::new(); - for (author, ts) in self.iter() { - by_timestamp.insert(*ts, *author); - } - let mut items = Vec::new(); - for (ts, author) in by_timestamp.into_iter().rev() { - items.push((ts, author)); - if let Some(size_limit) = size_limit { - if postcard::experimental::serialized_size(&items)? > size_limit { - items.pop(); - break; - } - } - } - let encoded = postcard::to_stdvec(&items)?; - debug_assert!(size_limit.map(|s| encoded.len() <= s).unwrap_or(true)); - Ok(encoded) - } - - /// Decode from byte slice created with [`Self::encode`]. - pub fn decode(bytes: &[u8]) -> Result { - let items: Vec<(Timestamp, AuthorId)> = postcard::from_bytes(bytes)?; - let mut heads = AuthorHeads::default(); - for (ts, author) in items { - heads.insert(author, ts); - } - Ok(heads) - } -} - -impl FromIterator<(AuthorId, Timestamp)> for AuthorHeads { - fn from_iter>(iter: T) -> Self { - Self { - heads: iter.into_iter().collect(), - } - } -} - -impl FromIterator<(Timestamp, AuthorId)> for AuthorHeads { - fn from_iter>(iter: T) -> Self { - Self { - heads: iter.into_iter().map(|(ts, author)| (author, ts)).collect(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Record; - #[test] - fn author_heads_encode_decode() -> Result<()> { - let mut heads = AuthorHeads::default(); - let start = Record::empty_current().timestamp(); - for i in 0..10u64 { - heads.insert(AuthorId::from(&[i as u8; 32]), start + i); - } - let encoded = heads.encode(Some(256))?; - let decoded = AuthorHeads::decode(&encoded)?; - assert_eq!(decoded.len(), 6); - let expected: AuthorHeads = (0u64..6) - .map(|n| (AuthorId::from(&[9 - n as u8; 32]), start + (9 - n))) - .collect(); - assert_eq!(expected, decoded); - Ok(()) - } - - #[test] - fn author_heads_compare() -> Result<()> { - let a = [ - (AuthorId::from(&[0u8; 32]), 5), - (AuthorId::from(&[1u8; 32]), 7), - ]; - let b = [ - (AuthorId::from(&[0u8; 32]), 4), - (AuthorId::from(&[1u8; 32]), 6), - (AuthorId::from(&[2u8; 32]), 7), - ]; - let a: AuthorHeads = a.into_iter().collect(); - let b: AuthorHeads = b.into_iter().collect(); - assert_eq!(a.has_news_for(&b), NonZeroU64::new(2)); - assert_eq!(b.has_news_for(&a), NonZeroU64::new(1)); - Ok(()) - } -} diff --git a/iroh-docs/src/keys.rs b/iroh-docs/src/keys.rs deleted file mode 100644 index 9efbeee5607..00000000000 --- a/iroh-docs/src/keys.rs +++ /dev/null @@ -1,516 +0,0 @@ -//! Keys used in iroh-docs - -use std::{cmp::Ordering, fmt, str::FromStr}; - -use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; -use iroh_base::base32; -use rand_core::CryptoRngCore; -use serde::{Deserialize, Serialize}; - -use crate::store::PublicKeyStore; - -/// Author key to insert entries in a [`crate::Replica`] -/// -/// Internally, an author is a [`SigningKey`] which is used to sign entries. -#[derive(Clone, Serialize, Deserialize)] -pub struct Author { - signing_key: SigningKey, -} -impl Author { - /// Create a new [`Author`] with a random key. - pub fn new(rng: &mut R) -> Self { - let signing_key = SigningKey::generate(rng); - Author { signing_key } - } - - /// Create an [`Author`] from a byte array. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { - SigningKey::from_bytes(bytes).into() - } - - /// Returns the [`Author`] byte representation. - pub fn to_bytes(&self) -> [u8; 32] { - self.signing_key.to_bytes() - } - - /// Get the [`AuthorPublicKey`] for this author. - pub fn public_key(&self) -> AuthorPublicKey { - AuthorPublicKey(self.signing_key.verifying_key()) - } - - /// Get the [`AuthorId`] for this author. - pub fn id(&self) -> AuthorId { - AuthorId::from(self.public_key()) - } - - /// Sign a message with this [`Author`] key. - pub fn sign(&self, msg: &[u8]) -> Signature { - self.signing_key.sign(msg) - } - - /// Strictly verify a signature on a message with this [`Author`]'s public key. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.signing_key.verify_strict(msg, signature) - } -} - -/// Identifier for an [`Author`] -/// -/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can -/// be used to verify [`Signature`]s. -#[derive(Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash, derive_more::From)] -pub struct AuthorPublicKey(VerifyingKey); - -impl AuthorPublicKey { - /// Verify that a signature matches the `msg` bytes and was created with the [`Author`] - /// that corresponds to this [`AuthorId`]. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.0.verify_strict(msg, signature) - } - - /// Get the byte representation of this [`AuthorId`]. - pub fn as_bytes(&self) -> &[u8; 32] { - self.0.as_bytes() - } - - /// Create from a slice of bytes. - /// - /// Will return an error if the input bytes do not represent a valid [`ed25519_dalek`] - /// curve point. Will never fail for a byte array returned from [`Self::as_bytes`]. - /// See [`VerifyingKey::from_bytes`] for details. - pub fn from_bytes(bytes: &[u8; 32]) -> Result { - Ok(AuthorPublicKey(VerifyingKey::from_bytes(bytes)?)) - } -} - -/// Namespace key of a [`crate::Replica`]. -/// -/// Holders of this key can insert new entries into a [`crate::Replica`]. -/// Internally, a [`NamespaceSecret`] is a [`SigningKey`] which is used to sign entries. -#[derive(Clone, Serialize, Deserialize)] -pub struct NamespaceSecret { - signing_key: SigningKey, -} - -impl NamespaceSecret { - /// Create a new [`NamespaceSecret`] with a random key. - pub fn new(rng: &mut R) -> Self { - let signing_key = SigningKey::generate(rng); - - NamespaceSecret { signing_key } - } - - /// Create a [`NamespaceSecret`] from a byte array. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { - SigningKey::from_bytes(bytes).into() - } - - /// Returns the [`NamespaceSecret`] byte representation. - pub fn to_bytes(&self) -> [u8; 32] { - self.signing_key.to_bytes() - } - - /// Get the [`NamespacePublicKey`] for this namespace. - pub fn public_key(&self) -> NamespacePublicKey { - NamespacePublicKey(self.signing_key.verifying_key()) - } - - /// Get the [`NamespaceId`] for this namespace. - pub fn id(&self) -> NamespaceId { - NamespaceId::from(self.public_key()) - } - - /// Sign a message with this [`NamespaceSecret`] key. - pub fn sign(&self, msg: &[u8]) -> Signature { - self.signing_key.sign(msg) - } - - /// Strictly verify a signature on a message with this [`NamespaceSecret`]'s public key. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.signing_key.verify_strict(msg, signature) - } -} - -/// The corresponding [`VerifyingKey`] for a [`NamespaceSecret`]. -/// It is used as an identifier, and can be used to verify [`Signature`]s. -#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, derive_more::From)] -pub struct NamespacePublicKey(VerifyingKey); - -impl NamespacePublicKey { - /// Verify that a signature matches the `msg` bytes and was created with the [`NamespaceSecret`] - /// that corresponds to this [`NamespaceId`]. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.0.verify_strict(msg, signature) - } - - /// Get the byte representation of this [`NamespaceId`]. - pub fn as_bytes(&self) -> &[u8; 32] { - self.0.as_bytes() - } - - /// Create from a slice of bytes. - /// - /// Will return an error if the input bytes do not represent a valid [`ed25519_dalek`] - /// curve point. Will never fail for a byte array returned from [`Self::as_bytes`]. - /// See [`VerifyingKey::from_bytes`] for details. - pub fn from_bytes(bytes: &[u8; 32]) -> Result { - Ok(NamespacePublicKey(VerifyingKey::from_bytes(bytes)?)) - } -} - -impl fmt::Display for Author { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.to_bytes())) - } -} - -impl fmt::Display for NamespaceSecret { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.to_bytes())) - } -} - -impl fmt::Display for AuthorPublicKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.as_bytes())) - } -} - -impl fmt::Display for NamespacePublicKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.as_bytes())) - } -} - -impl fmt::Display for AuthorId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.as_bytes())) - } -} - -impl fmt::Display for NamespaceId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(self.as_bytes())) - } -} - -impl fmt::Debug for NamespaceSecret { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Namespace({})", self) - } -} - -impl fmt::Debug for NamespaceId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NamespaceId({})", base32::fmt_short(self.0)) - } -} - -impl fmt::Debug for AuthorId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "AuthorId({})", base32::fmt_short(self.0)) - } -} - -impl fmt::Debug for Author { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Author({})", self) - } -} - -impl fmt::Debug for NamespacePublicKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NamespacePublicKey({})", self) - } -} - -impl fmt::Debug for AuthorPublicKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "AuthorPublicKey({})", self) - } -} - -impl FromStr for Author { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - Ok(Self::from_bytes(&base32::parse_array(s)?)) - } -} - -impl FromStr for NamespaceSecret { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - Ok(Self::from_bytes(&base32::parse_array(s)?)) - } -} - -impl FromStr for AuthorPublicKey { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - Self::from_bytes(&base32::parse_array(s)?).map_err(Into::into) - } -} - -impl FromStr for NamespacePublicKey { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - Self::from_bytes(&base32::parse_array(s)?).map_err(Into::into) - } -} - -impl From for Author { - fn from(signing_key: SigningKey) -> Self { - Self { signing_key } - } -} - -impl From for NamespaceSecret { - fn from(signing_key: SigningKey) -> Self { - Self { signing_key } - } -} - -impl PartialOrd for NamespacePublicKey { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for NamespacePublicKey { - fn cmp(&self, other: &Self) -> Ordering { - self.0.as_bytes().cmp(other.0.as_bytes()) - } -} - -impl PartialOrd for AuthorPublicKey { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for AuthorPublicKey { - fn cmp(&self, other: &Self) -> Ordering { - self.0.as_bytes().cmp(other.0.as_bytes()) - } -} - -impl From for NamespacePublicKey { - fn from(value: NamespaceSecret) -> Self { - value.public_key() - } -} - -impl From for AuthorPublicKey { - fn from(value: Author) -> Self { - value.public_key() - } -} - -impl From<&NamespaceSecret> for NamespacePublicKey { - fn from(value: &NamespaceSecret) -> Self { - value.public_key() - } -} - -impl From<&Author> for AuthorPublicKey { - fn from(value: &Author) -> Self { - value.public_key() - } -} - -/// [`NamespacePublicKey`] in bytes -#[derive( - Default, - Clone, - Copy, - PartialOrd, - Ord, - Eq, - PartialEq, - Hash, - derive_more::From, - derive_more::Into, - derive_more::AsRef, - Serialize, - Deserialize, -)] -pub struct NamespaceId([u8; 32]); - -/// [`AuthorPublicKey`] in bytes -#[derive( - Default, - Clone, - Copy, - PartialOrd, - Ord, - Eq, - PartialEq, - Hash, - derive_more::From, - derive_more::Into, - derive_more::AsRef, - Serialize, - Deserialize, -)] -pub struct AuthorId([u8; 32]); - -impl AuthorId { - /// Convert to byte array. - pub fn to_bytes(&self) -> [u8; 32] { - self.0 - } - - /// Convert to byte slice. - pub fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - /// Convert into [`AuthorPublicKey`] by fetching from a [`PublicKeyStore`]. - /// - /// Fails if the bytes of this [`AuthorId`] are not a valid [`ed25519_dalek`] curve point. - pub fn public_key( - &self, - store: &S, - ) -> Result { - store.author_key(self) - } - - /// Convert into [`AuthorPublicKey`]. - /// - /// Fails if the bytes of this [`AuthorId`] are not a valid [`ed25519_dalek`] curve point. - pub fn into_public_key(&self) -> Result { - AuthorPublicKey::from_bytes(&self.0) - } - - /// Convert to a base32 string limited to the first 10 bytes for a friendly string - /// representation of the key. - pub fn fmt_short(&self) -> String { - base32::fmt_short(self.0) - } -} - -impl NamespaceId { - /// Convert to byte array. - pub fn to_bytes(&self) -> [u8; 32] { - self.0 - } - - /// Convert to byte slice. - pub fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - /// Convert into [`NamespacePublicKey`] by fetching from a [`PublicKeyStore`]. - /// - /// Fails if the bytes of this [`NamespaceId`] are not a valid [`ed25519_dalek`] curve point. - pub fn public_key( - &self, - store: &S, - ) -> Result { - store.namespace_key(self) - } - - /// Convert into [`NamespacePublicKey`]. - /// - /// Fails if the bytes of this [`NamespaceId`] are not a valid [`ed25519_dalek`] curve point. - pub fn into_public_key(&self) -> Result { - NamespacePublicKey::from_bytes(&self.0) - } - - /// Convert to a base32 string limited to the first 10 bytes for a friendly string - /// representation of the key. - pub fn fmt_short(&self) -> String { - base32::fmt_short(self.0) - } -} - -impl From<&[u8; 32]> for NamespaceId { - fn from(value: &[u8; 32]) -> Self { - Self(*value) - } -} - -impl From<&[u8; 32]> for AuthorId { - fn from(value: &[u8; 32]) -> Self { - Self(*value) - } -} - -impl AsRef<[u8]> for NamespaceId { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl AsRef<[u8]> for AuthorId { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl From for AuthorId { - fn from(value: AuthorPublicKey) -> Self { - Self(*value.as_bytes()) - } -} -impl From for NamespaceId { - fn from(value: NamespacePublicKey) -> Self { - Self(*value.as_bytes()) - } -} - -impl From<&AuthorPublicKey> for AuthorId { - fn from(value: &AuthorPublicKey) -> Self { - Self(*value.as_bytes()) - } -} -impl From<&NamespacePublicKey> for NamespaceId { - fn from(value: &NamespacePublicKey) -> Self { - Self(*value.as_bytes()) - } -} - -impl From for AuthorId { - fn from(value: Author) -> Self { - value.id() - } -} -impl From for NamespaceId { - fn from(value: NamespaceSecret) -> Self { - value.id() - } -} - -impl TryFrom for NamespacePublicKey { - type Error = SignatureError; - fn try_from(value: NamespaceId) -> Result { - Self::from_bytes(&value.0) - } -} - -impl TryFrom for AuthorPublicKey { - type Error = SignatureError; - fn try_from(value: AuthorId) -> Result { - Self::from_bytes(&value.0) - } -} - -impl FromStr for AuthorId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - AuthorPublicKey::from_str(s).map(|x| x.into()) - } -} - -impl FromStr for NamespaceId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - NamespacePublicKey::from_str(s).map(|x| x.into()) - } -} diff --git a/iroh-docs/src/lib.rs b/iroh-docs/src/lib.rs deleted file mode 100644 index b7bac5f8f45..00000000000 --- a/iroh-docs/src/lib.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! Multi-dimensional key-value documents with an efficient synchronization protocol -//! -//! The crate operates on [Replicas](Replica). A replica contains an unlimited number of -//! [Entries][Entry]. Each entry is identified by a key, its author, and the replica's -//! namespace. Its value is the [32-byte BLAKE3 hash](iroh_base::hash::Hash) -//! of the entry's content data, the size of this content data, and a timestamp. -//! The content data itself is not stored or transferred through a replica. -//! -//! All entries in a replica are signed with two keypairs: -//! -//! * The [`NamespaceSecret`] key, as a token of write capability. The public key is the -//! [`NamespaceId`], which also serves as the unique identifier for a replica. -//! * The [Author] key, as a proof of authorship. Any number of authors may be created, and -//! their semantic meaning is application-specific. The public key of an author is the [AuthorId]. -//! -//! Replicas can be synchronized between peers by exchanging messages. The synchronization algorithm -//! is based on a technique called *range-based set reconciliation*, based on [this paper][paper] by -//! Aljoscha Meyer: -//! -//! > Range-based set reconciliation is a simple approach to efficiently compute the union of two -//! > sets over a network, based on recursively partitioning the sets and comparing fingerprints of -//! > the partitions to probabilistically detect whether a partition requires further work. -//! -//! The crate exposes a [generic storage interface](store::Store). There is an implementation -//! of this interface, [store::fs::Store], that can be used either -//! [in-memory](store::fs::Store::memory) or in -//! [persistent, file-based](store::fs::Store::persistent) mode. -//! -//! Both modes make use of [`redb`], an embedded key-value store. When used -//! in-memory, the store is backed by a `Vec`. When used in persistent mode, -//! the store is backed by a single file on disk. -//! -//! [paper]: https://arxiv.org/abs/2212.13567 -#![deny(missing_docs, rustdoc::broken_intra_doc_links)] -#![cfg_attr(iroh_docsrs, feature(doc_cfg))] - -pub mod metrics; -#[cfg(feature = "net")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "net")))] -pub mod net; -#[cfg(feature = "net")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "net")))] -mod ticket; - -#[cfg(feature = "engine")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "engine")))] -pub mod engine; - -pub mod actor; -pub mod store; -pub mod sync; - -mod heads; -mod keys; -mod ranger; - -#[cfg(feature = "net")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "net")))] -pub use self::ticket::DocTicket; -pub use self::{heads::*, keys::*, sync::*}; diff --git a/iroh-docs/src/metrics.rs b/iroh-docs/src/metrics.rs deleted file mode 100644 index 69d7ab6733f..00000000000 --- a/iroh-docs/src/metrics.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Metrics for iroh-docs - -use iroh_metrics::{ - core::{Counter, Metric}, - struct_iterable::Iterable, -}; - -/// Metrics for iroh-docs -#[allow(missing_docs)] -#[derive(Debug, Clone, Iterable)] -pub struct Metrics { - pub new_entries_local: Counter, - pub new_entries_remote: Counter, - pub new_entries_local_size: Counter, - pub new_entries_remote_size: Counter, - pub sync_via_connect_success: Counter, - pub sync_via_connect_failure: Counter, - pub sync_via_accept_success: Counter, - pub sync_via_accept_failure: Counter, - - pub actor_tick_main: Counter, - - pub doc_gossip_tick_main: Counter, - pub doc_gossip_tick_event: Counter, - pub doc_gossip_tick_actor: Counter, - pub doc_gossip_tick_pending_join: Counter, - - pub doc_live_tick_main: Counter, - pub doc_live_tick_actor: Counter, - pub doc_live_tick_replica_event: Counter, - pub doc_live_tick_running_sync_connect: Counter, - pub doc_live_tick_running_sync_accept: Counter, - pub doc_live_tick_pending_downloads: Counter, -} - -impl Default for Metrics { - fn default() -> Self { - Self { - new_entries_local: Counter::new("Number of document entries added locally"), - new_entries_remote: Counter::new("Number of document entries added by peers"), - new_entries_local_size: Counter::new("Total size of entry contents added locally"), - new_entries_remote_size: Counter::new("Total size of entry contents added by peers"), - sync_via_accept_success: Counter::new("Number of successful syncs (via accept)"), - sync_via_accept_failure: Counter::new("Number of failed syncs (via accept)"), - sync_via_connect_success: Counter::new("Number of successful syncs (via connect)"), - sync_via_connect_failure: Counter::new("Number of failed syncs (via connect)"), - - actor_tick_main: Counter::new("Number of times the main actor loop ticked"), - - doc_gossip_tick_main: Counter::new("Number of times the gossip actor loop ticked"), - doc_gossip_tick_event: Counter::new( - "Number of times the gossip actor processed an event", - ), - doc_gossip_tick_actor: Counter::new( - "Number of times the gossip actor processed an actor event", - ), - doc_gossip_tick_pending_join: Counter::new( - "Number of times the gossip actor processed a pending join", - ), - - doc_live_tick_main: Counter::new("Number of times the live actor loop ticked"), - doc_live_tick_actor: Counter::new( - "Number of times the live actor processed an actor event", - ), - doc_live_tick_replica_event: Counter::new( - "Number of times the live actor processed a replica event", - ), - doc_live_tick_running_sync_connect: Counter::new( - "Number of times the live actor processed a running sync connect", - ), - doc_live_tick_running_sync_accept: Counter::new( - "Number of times the live actor processed a running sync accept", - ), - doc_live_tick_pending_downloads: Counter::new( - "Number of times the live actor processed a pending download", - ), - } - } -} - -impl Metric for Metrics { - fn name() -> &'static str { - "iroh_docs" - } -} diff --git a/iroh-docs/src/net.rs b/iroh-docs/src/net.rs deleted file mode 100644 index 42353c3e25b..00000000000 --- a/iroh-docs/src/net.rs +++ /dev/null @@ -1,368 +0,0 @@ -//! Network implementation of the iroh-docs protocol - -use std::{ - future::Future, - time::{Duration, Instant}, -}; - -#[cfg(feature = "metrics")] -use iroh_metrics::inc; -use iroh_net::{endpoint::get_remote_node_id, key::PublicKey, Endpoint, NodeAddr}; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error_span, trace, Instrument}; - -#[cfg(feature = "metrics")] -use crate::metrics::Metrics; -use crate::{ - actor::SyncHandle, - net::codec::{run_alice, BobState}, - NamespaceId, SyncOutcome, -}; - -/// The ALPN identifier for the iroh-docs protocol -pub const DOCS_ALPN: &[u8] = b"/iroh-sync/1"; - -mod codec; - -/// Connect to a peer and sync a replica -pub async fn connect_and_sync( - endpoint: &Endpoint, - sync: &SyncHandle, - namespace: NamespaceId, - peer: NodeAddr, -) -> Result { - let t_start = Instant::now(); - let peer_id = peer.node_id; - trace!("connect"); - let connection = endpoint - .connect(peer, DOCS_ALPN) - .await - .map_err(ConnectError::connect)?; - - let (mut send_stream, mut recv_stream) = - connection.open_bi().await.map_err(ConnectError::connect)?; - - let t_connect = t_start.elapsed(); - debug!(?t_connect, "connected"); - - let res = run_alice(&mut send_stream, &mut recv_stream, sync, namespace, peer_id).await; - - send_stream.finish().map_err(ConnectError::close)?; - send_stream.stopped().await.map_err(ConnectError::close)?; - recv_stream - .read_to_end(0) - .await - .map_err(ConnectError::close)?; - - #[cfg(feature = "metrics")] - if res.is_ok() { - inc!(Metrics, sync_via_connect_success); - } else { - inc!(Metrics, sync_via_connect_failure); - } - - let t_process = t_start.elapsed() - t_connect; - match &res { - Ok(res) => { - debug!( - ?t_connect, - ?t_process, - sent = %res.num_sent, - recv = %res.num_recv, - "done, ok" - ); - } - Err(err) => { - debug!(?t_connect, ?t_process, ?err, "done, failed"); - } - } - - let outcome = res?; - - let timings = Timings { - connect: t_connect, - process: t_process, - }; - - let res = SyncFinished { - namespace, - peer: peer_id, - outcome, - timings, - }; - - Ok(res) -} - -/// Whether we want to accept or reject an incoming sync request. -#[derive(Debug, Clone)] -pub enum AcceptOutcome { - /// Accept the sync request. - Allow, - /// Decline the sync request - Reject(AbortReason), -} - -/// Handle an iroh-docs connection and sync all shared documents in the replica store. -pub async fn handle_connection( - sync: SyncHandle, - connecting: iroh_net::endpoint::Connecting, - accept_cb: F, -) -> Result -where - F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future, -{ - let t_start = Instant::now(); - let connection = connecting.await.map_err(AcceptError::connect)?; - let peer = get_remote_node_id(&connection).map_err(AcceptError::connect)?; - let (mut send_stream, mut recv_stream) = connection - .accept_bi() - .await - .map_err(|e| AcceptError::open(peer, e))?; - - let t_connect = t_start.elapsed(); - let span = error_span!("accept", peer = %peer.fmt_short(), namespace = tracing::field::Empty); - span.in_scope(|| { - debug!(?t_connect, "connection established"); - }); - - let mut state = BobState::new(peer); - let res = state - .run(&mut send_stream, &mut recv_stream, sync, accept_cb) - .instrument(span.clone()) - .await; - - #[cfg(feature = "metrics")] - if res.is_ok() { - inc!(Metrics, sync_via_accept_success); - } else { - inc!(Metrics, sync_via_accept_failure); - } - - let namespace = state.namespace(); - let outcome = state.into_outcome(); - - send_stream - .finish() - .map_err(|error| AcceptError::close(peer, namespace, error))?; - send_stream - .stopped() - .await - .map_err(|error| AcceptError::close(peer, namespace, error))?; - recv_stream - .read_to_end(0) - .await - .map_err(|error| AcceptError::close(peer, namespace, error))?; - - let t_process = t_start.elapsed() - t_connect; - span.in_scope(|| match &res { - Ok(_res) => { - debug!( - ?t_connect, - ?t_process, - sent = %outcome.num_sent, - recv = %outcome.num_recv, - "done, ok" - ); - } - Err(err) => { - debug!(?t_connect, ?t_process, ?err, "done, failed"); - } - }); - - let namespace = res?; - - let timings = Timings { - connect: t_connect, - process: t_process, - }; - let res = SyncFinished { - namespace, - outcome, - peer, - timings, - }; - - Ok(res) -} - -/// Details of a finished sync operation. -#[derive(Debug, Clone)] -pub struct SyncFinished { - /// The namespace that was synced. - pub namespace: NamespaceId, - /// The peer we syned with. - pub peer: PublicKey, - /// The outcome of the sync operation - pub outcome: SyncOutcome, - /// The time this operation took - pub timings: Timings, -} - -/// Time a sync operation took -#[derive(Debug, Default, Clone)] -pub struct Timings { - /// Time to establish connection - pub connect: Duration, - /// Time to run sync exchange - pub process: Duration, -} - -/// Errors that may occur on handling incoming sync connections. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum AcceptError { - /// Failed to establish connection - #[error("Failed to establish connection")] - Connect { - #[source] - error: anyhow::Error, - }, - /// Failed to open replica - #[error("Failed to open replica with {peer:?}")] - Open { - peer: PublicKey, - #[source] - error: anyhow::Error, - }, - /// We aborted the sync request. - #[error("Aborted sync of {namespace:?} with {peer:?}: {reason:?}")] - Abort { - peer: PublicKey, - namespace: NamespaceId, - reason: AbortReason, - }, - /// Failed to run sync - #[error("Failed to sync {namespace:?} with {peer:?}")] - Sync { - peer: PublicKey, - namespace: Option, - #[source] - error: anyhow::Error, - }, - /// Failed to close - #[error("Failed to close {namespace:?} with {peer:?}")] - Close { - peer: PublicKey, - namespace: Option, - #[source] - error: anyhow::Error, - }, -} - -/// Errors that may occur on outgoing sync requests. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum ConnectError { - /// Failed to establish connection - #[error("Failed to establish connection")] - Connect { - #[source] - error: anyhow::Error, - }, - /// The remote peer aborted the sync request. - #[error("Remote peer aborted sync: {0:?}")] - RemoteAbort(AbortReason), - /// Failed to run sync - #[error("Failed to sync")] - Sync { - #[source] - error: anyhow::Error, - }, - /// Failed to close - #[error("Failed to close connection1")] - Close { - #[source] - error: anyhow::Error, - }, -} - -/// Reason why we aborted an incoming sync request. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -pub enum AbortReason { - /// Namespace is not available. - NotFound, - /// We are already syncing this namespace. - AlreadySyncing, - /// We experienced an error while trying to provide the requested resource - InternalServerError, -} - -impl AcceptError { - fn connect(error: impl Into) -> Self { - Self::Connect { - error: error.into(), - } - } - fn open(peer: PublicKey, error: impl Into) -> Self { - Self::Open { - peer, - error: error.into(), - } - } - pub(crate) fn sync( - peer: PublicKey, - namespace: Option, - error: impl Into, - ) -> Self { - Self::Sync { - peer, - namespace, - error: error.into(), - } - } - fn close( - peer: PublicKey, - namespace: Option, - error: impl Into, - ) -> Self { - Self::Close { - peer, - namespace, - error: error.into(), - } - } - /// Get the peer's node ID (if available) - pub fn peer(&self) -> Option { - match self { - AcceptError::Connect { .. } => None, - AcceptError::Open { peer, .. } => Some(*peer), - AcceptError::Sync { peer, .. } => Some(*peer), - AcceptError::Close { peer, .. } => Some(*peer), - AcceptError::Abort { peer, .. } => Some(*peer), - } - } - - /// Get the namespace (if available) - pub fn namespace(&self) -> Option { - match self { - AcceptError::Connect { .. } => None, - AcceptError::Open { .. } => None, - AcceptError::Sync { namespace, .. } => namespace.to_owned(), - AcceptError::Close { namespace, .. } => namespace.to_owned(), - AcceptError::Abort { namespace, .. } => Some(*namespace), - } - } -} - -impl ConnectError { - fn connect(error: impl Into) -> Self { - Self::Connect { - error: error.into(), - } - } - fn close(error: impl Into) -> Self { - Self::Close { - error: error.into(), - } - } - pub(crate) fn sync(error: impl Into) -> Self { - Self::Sync { - error: error.into(), - } - } - pub(crate) fn remote_abort(reason: AbortReason) -> Self { - Self::RemoteAbort(reason) - } -} diff --git a/iroh-docs/src/net/codec.rs b/iroh-docs/src/net/codec.rs deleted file mode 100644 index e4e164d466c..00000000000 --- a/iroh-docs/src/net/codec.rs +++ /dev/null @@ -1,693 +0,0 @@ -use std::future::Future; - -use anyhow::{anyhow, ensure}; -use bytes::{Buf, BufMut, BytesMut}; -use futures_util::SinkExt; -use iroh_net::key::PublicKey; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio_stream::StreamExt; -use tokio_util::codec::{Decoder, Encoder, FramedRead, FramedWrite}; -use tracing::{debug, trace, Span}; - -use crate::{ - actor::SyncHandle, - net::{AbortReason, AcceptError, AcceptOutcome, ConnectError}, - NamespaceId, SyncOutcome, -}; - -#[derive(Debug, Default)] -struct SyncCodec; - -const MAX_MESSAGE_SIZE: usize = 1024 * 1024 * 1024; // This is likely too large, but lets have some restrictions - -impl Decoder for SyncCodec { - type Item = Message; - type Error = anyhow::Error; - fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - if src.len() < 4 { - return Ok(None); - } - let bytes: [u8; 4] = src[..4].try_into().unwrap(); - let frame_len = u32::from_be_bytes(bytes) as usize; - ensure!( - frame_len <= MAX_MESSAGE_SIZE, - "received message that is too large: {}", - frame_len - ); - if src.len() < 4 + frame_len { - return Ok(None); - } - - let message: Message = postcard::from_bytes(&src[4..4 + frame_len])?; - src.advance(4 + frame_len); - Ok(Some(message)) - } -} - -impl Encoder for SyncCodec { - type Error = anyhow::Error; - - fn encode(&mut self, item: Message, dst: &mut BytesMut) -> Result<(), Self::Error> { - let len = - postcard::serialize_with_flavor(&item, postcard::ser_flavors::Size::default()).unwrap(); - ensure!( - len <= MAX_MESSAGE_SIZE, - "attempting to send message that is too large {}", - len - ); - - dst.put_u32(u32::try_from(len).expect("already checked")); - if dst.len() < 4 + len { - dst.resize(4 + len, 0u8); - } - postcard::to_slice(&item, &mut dst[4..])?; - - Ok(()) - } -} - -/// Sync Protocol -/// -/// - Init message: signals which namespace is being synced -/// - N Sync messages -/// -/// On any error and on success the substream is closed. -#[derive(Debug, Clone, Serialize, Deserialize)] -enum Message { - /// Init message (sent by the dialing peer) - Init { - /// Namespace to sync - namespace: NamespaceId, - /// Initial message - message: crate::sync::ProtocolMessage, - }, - /// Sync messages (sent by both peers) - Sync(crate::sync::ProtocolMessage), - /// Abort message (sent by the accepting peer to decline a request) - Abort { reason: AbortReason }, -} - -/// Runs the initiator side of the sync protocol. -pub(super) async fn run_alice( - writer: &mut W, - reader: &mut R, - handle: &SyncHandle, - namespace: NamespaceId, - peer: PublicKey, -) -> Result { - let peer_bytes = *peer.as_bytes(); - let mut reader = FramedRead::new(reader, SyncCodec); - let mut writer = FramedWrite::new(writer, SyncCodec); - - let mut progress = Some(SyncOutcome::default()); - - // Init message - - let message = handle - .sync_initial_message(namespace) - .await - .map_err(ConnectError::sync)?; - let init_message = Message::Init { namespace, message }; - trace!("send init message"); - writer - .send(init_message) - .await - .map_err(ConnectError::sync)?; - - // Sync message loop - while let Some(msg) = reader.next().await { - let msg = msg.map_err(ConnectError::sync)?; - match msg { - Message::Init { .. } => { - return Err(ConnectError::sync(anyhow!("unexpected init message"))); - } - Message::Sync(msg) => { - trace!("recv process message"); - let current_progress = progress.take().unwrap(); - let (reply, next_progress) = handle - .sync_process_message(namespace, msg, peer_bytes, current_progress) - .await - .map_err(ConnectError::sync)?; - progress = Some(next_progress); - if let Some(msg) = reply { - trace!("send process message"); - writer - .send(Message::Sync(msg)) - .await - .map_err(ConnectError::sync)?; - } else { - break; - } - } - Message::Abort { reason } => { - return Err(ConnectError::remote_abort(reason)); - } - } - } - - trace!("done"); - Ok(progress.unwrap()) -} - -/// Runs the receiver side of the sync protocol. -#[cfg(test)] -pub(super) async fn run_bob( - writer: &mut W, - reader: &mut R, - handle: SyncHandle, - accept_cb: F, - peer: PublicKey, -) -> Result<(NamespaceId, SyncOutcome), AcceptError> -where - R: AsyncRead + Unpin, - W: AsyncWrite + Unpin, - F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future, -{ - let mut state = BobState::new(peer); - let namespace = state.run(writer, reader, handle, accept_cb).await?; - Ok((namespace, state.into_outcome())) -} - -/// State for the receiver side of the sync protocol. -pub struct BobState { - namespace: Option, - peer: PublicKey, - progress: Option, -} - -impl BobState { - /// Create a new state for a single connection. - pub fn new(peer: PublicKey) -> Self { - Self { - peer, - namespace: None, - progress: Some(Default::default()), - } - } - - fn fail(&self, reason: impl Into) -> AcceptError { - AcceptError::sync(self.peer, self.namespace(), reason.into()) - } - - /// Handle connection and run to end. - pub async fn run( - &mut self, - writer: W, - reader: R, - sync: SyncHandle, - accept_cb: F, - ) -> Result - where - R: AsyncRead + Unpin, - W: AsyncWrite + Unpin, - F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future, - { - let mut reader = FramedRead::new(reader, SyncCodec); - let mut writer = FramedWrite::new(writer, SyncCodec); - while let Some(msg) = reader.next().await { - let msg = msg.map_err(|e| self.fail(e))?; - let next = match (msg, self.namespace.as_ref()) { - (Message::Init { namespace, message }, None) => { - Span::current() - .record("namespace", tracing::field::display(&namespace.fmt_short())); - trace!("recv init message"); - let accept = accept_cb(namespace, self.peer).await; - match accept { - AcceptOutcome::Allow => { - trace!("allow request"); - } - AcceptOutcome::Reject(reason) => { - debug!(?reason, "reject request"); - writer - .send(Message::Abort { reason }) - .await - .map_err(|e| self.fail(e))?; - return Err(AcceptError::Abort { - namespace, - peer: self.peer, - reason, - }); - } - } - let last_progress = self.progress.take().unwrap(); - let next = sync - .sync_process_message( - namespace, - message, - *self.peer.as_bytes(), - last_progress, - ) - .await; - self.namespace = Some(namespace); - next - } - (Message::Sync(msg), Some(namespace)) => { - trace!("recv process message"); - let last_progress = self.progress.take().unwrap(); - sync.sync_process_message(*namespace, msg, *self.peer.as_bytes(), last_progress) - .await - } - (Message::Init { .. }, Some(_)) => { - return Err(self.fail(anyhow!("double init message"))) - } - (Message::Sync(_), None) => { - return Err(self.fail(anyhow!("unexpected sync message before init"))) - } - (Message::Abort { .. }, _) => { - return Err(self.fail(anyhow!("unexpected sync abort message"))) - } - }; - let (reply, progress) = next.map_err(|e| self.fail(e))?; - self.progress = Some(progress); - match reply { - Some(msg) => { - trace!("send process message"); - writer - .send(Message::Sync(msg)) - .await - .map_err(|e| self.fail(e))?; - } - None => break, - } - } - - trace!("done"); - - self.namespace() - .ok_or_else(|| self.fail(anyhow!("Stream closed before init message"))) - } - - /// Get the namespace that is synced, if available. - pub fn namespace(&self) -> Option { - self.namespace - } - - /// Consume self and get the [`SyncOutcome`] for this connection. - pub fn into_outcome(self) -> SyncOutcome { - self.progress.unwrap() - } -} - -#[cfg(test)] -mod tests { - use anyhow::Result; - use iroh_base::hash::Hash; - use iroh_net::key::SecretKey; - use rand_core::{CryptoRngCore, SeedableRng}; - - use super::*; - use crate::{ - actor::OpenOpts, - store::{self, Query, Store}, - AuthorId, NamespaceSecret, - }; - - #[tokio::test] - async fn test_sync_simple() -> Result<()> { - let mut rng = rand::thread_rng(); - let alice_peer_id = SecretKey::from_bytes(&[1u8; 32]).public(); - let bob_peer_id = SecretKey::from_bytes(&[2u8; 32]).public(); - - let mut alice_store = store::Store::memory(); - // For now uses same author on both sides. - let author = alice_store.new_author(&mut rng).unwrap(); - - let namespace = NamespaceSecret::new(&mut rng); - - let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); - let alice_replica_id = alice_replica.id(); - alice_replica - .hash_and_insert("hello bob", &author, "from alice") - .unwrap(); - - let mut bob_store = store::Store::memory(); - let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); - let bob_replica_id = bob_replica.id(); - bob_replica - .hash_and_insert("hello alice", &author, "from bob") - .unwrap(); - - assert_eq!( - bob_store - .get_many(bob_replica_id, Query::all(),) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 1 - ); - assert_eq!( - alice_store - .get_many(alice_replica_id, Query::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 1 - ); - - // close the replicas because now the async actor will take over - alice_store.close_replica(alice_replica_id); - bob_store.close_replica(bob_replica_id); - - let (alice, bob) = tokio::io::duplex(64); - - let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); - let alice_handle = SyncHandle::spawn(alice_store, None, "alice".to_string()); - alice_handle - .open(namespace.id(), OpenOpts::default().sync()) - .await?; - let namespace_id = namespace.id(); - let alice_handle2 = alice_handle.clone(); - let alice_task = tokio::task::spawn(async move { - run_alice( - &mut alice_writer, - &mut alice_reader, - &alice_handle2, - namespace_id, - bob_peer_id, - ) - .await - }); - - let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_handle = SyncHandle::spawn(bob_store, None, "bob".to_string()); - bob_handle - .open(namespace.id(), OpenOpts::default().sync()) - .await?; - let bob_handle2 = bob_handle.clone(); - let bob_task = tokio::task::spawn(async move { - run_bob( - &mut bob_writer, - &mut bob_reader, - bob_handle2, - |_namespace, _peer| std::future::ready(AcceptOutcome::Allow), - alice_peer_id, - ) - .await - }); - - alice_task.await??; - bob_task.await??; - - let mut alice_store = alice_handle.shutdown().await?; - let mut bob_store = bob_handle.shutdown().await?; - - assert_eq!( - bob_store - .get_many(namespace.id(), Query::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 2 - ); - assert_eq!( - alice_store - .get_many(namespace.id(), Query::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 2 - ); - - Ok(()) - } - - #[tokio::test] - async fn test_sync_many_authors_memory() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let alice_store = store::Store::memory(); - let bob_store = store::Store::memory(); - test_sync_many_authors(alice_store, bob_store).await - } - - #[tokio::test] - async fn test_sync_many_authors_fs() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let tmpdir = tempfile::tempdir()?; - let alice_store = store::fs::Store::persistent(tmpdir.path().join("a.db"))?; - let bob_store = store::fs::Store::persistent(tmpdir.path().join("b.db"))?; - test_sync_many_authors(alice_store, bob_store).await - } - - type Message = (AuthorId, Vec, Hash); - - fn insert_messages( - mut rng: impl CryptoRngCore, - replica: &mut crate::sync::Replica, - num_authors: usize, - msgs_per_author: usize, - key_value_fn: impl Fn(&AuthorId, usize) -> (String, String), - ) -> Vec { - let mut res = vec![]; - let authors: Vec<_> = (0..num_authors) - .map(|_| replica.store.store.new_author(&mut rng).unwrap()) - .collect(); - - for i in 0..msgs_per_author { - for author in authors.iter() { - let (key, value) = key_value_fn(&author.id(), i); - let hash = replica.hash_and_insert(key.clone(), author, value).unwrap(); - res.push((author.id(), key.as_bytes().to_vec(), hash)); - } - } - res.sort(); - res - } - - fn get_messages(store: &mut Store, namespace: NamespaceId) -> Vec { - let mut msgs = store - .get_many(namespace, Query::all()) - .unwrap() - .map(|entry| { - entry.map(|entry| { - ( - entry.author_bytes(), - entry.key().to_vec(), - entry.content_hash(), - ) - }) - }) - .collect::>>() - .unwrap(); - msgs.sort(); - msgs - } - - async fn test_sync_many_authors(mut alice_store: Store, mut bob_store: Store) -> Result<()> { - let num_messages = &[1, 2, 5, 10]; - let num_authors = &[2, 3, 4, 5, 10]; - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - - for num_messages in num_messages { - for num_authors in num_authors { - println!( - "bob & alice each using {num_authors} authors and inserting {num_messages} messages per author" - ); - - let alice_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); - let bob_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); - let namespace = NamespaceSecret::new(&mut rng); - - let mut all_messages = vec![]; - - let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); - let alice_messages = insert_messages( - &mut rng, - &mut alice_replica, - *num_authors, - *num_messages, - |author, i| { - ( - format!("hello bob {i}"), - format!("from alice by {author}: {i}"), - ) - }, - ); - all_messages.extend_from_slice(&alice_messages); - - let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); - let bob_messages = insert_messages( - &mut rng, - &mut bob_replica, - *num_authors, - *num_messages, - |author, i| { - ( - format!("hello bob {i}"), - format!("from bob by {author}: {i}"), - ) - }, - ); - all_messages.extend_from_slice(&bob_messages); - - all_messages.sort(); - - let res = get_messages(&mut alice_store, namespace.id()); - assert_eq!(res, alice_messages); - - let res = get_messages(&mut bob_store, namespace.id()); - assert_eq!(res, bob_messages); - - // replicas can be opened only once so close the replicas before spawning the - // actors - alice_store.close_replica(namespace.id()); - let alice_handle = SyncHandle::spawn(alice_store, None, "alice".to_string()); - - bob_store.close_replica(namespace.id()); - let bob_handle = SyncHandle::spawn(bob_store, None, "bob".to_string()); - - run_sync( - alice_handle.clone(), - alice_node_pubkey, - bob_handle.clone(), - bob_node_pubkey, - namespace.id(), - ) - .await?; - - alice_store = alice_handle.shutdown().await?; - bob_store = bob_handle.shutdown().await?; - - let res = get_messages(&mut bob_store, namespace.id()); - assert_eq!(res.len(), all_messages.len()); - assert_eq!(res, all_messages); - - let res = get_messages(&mut bob_store, namespace.id()); - assert_eq!(res.len(), all_messages.len()); - assert_eq!(res, all_messages); - } - } - - Ok(()) - } - - async fn run_sync( - alice_handle: SyncHandle, - alice_node_pubkey: PublicKey, - bob_handle: SyncHandle, - bob_node_pubkey: PublicKey, - namespace: NamespaceId, - ) -> Result<()> { - alice_handle - .open(namespace, OpenOpts::default().sync()) - .await?; - bob_handle - .open(namespace, OpenOpts::default().sync()) - .await?; - let (alice, bob) = tokio::io::duplex(1024); - - let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); - let alice_task = tokio::task::spawn(async move { - run_alice( - &mut alice_writer, - &mut alice_reader, - &alice_handle, - namespace, - bob_node_pubkey, - ) - .await - }); - - let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_task = tokio::task::spawn(async move { - run_bob( - &mut bob_writer, - &mut bob_reader, - bob_handle, - |_namespace, _peer| std::future::ready(AcceptOutcome::Allow), - alice_node_pubkey, - ) - .await - }); - - alice_task.await??; - bob_task.await??; - Ok(()) - } - - #[tokio::test] - async fn test_sync_timestamps_memory() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let alice_store = store::Store::memory(); - let bob_store = store::Store::memory(); - test_sync_timestamps(alice_store, bob_store).await - } - - #[tokio::test] - async fn test_sync_timestamps_fs() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let tmpdir = tempfile::tempdir()?; - let alice_store = store::fs::Store::persistent(tmpdir.path().join("a.db"))?; - let bob_store = store::fs::Store::persistent(tmpdir.path().join("b.db"))?; - test_sync_timestamps(alice_store, bob_store).await - } - - async fn test_sync_timestamps(mut alice_store: Store, mut bob_store: Store) -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - let alice_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); - let bob_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); - let namespace = NamespaceSecret::new(&mut rng); - - let author = alice_store.new_author(&mut rng)?; - bob_store.import_author(author.clone())?; - - let key = vec![1u8]; - let value_alice = vec![2u8]; - let value_bob = vec![3u8]; - let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); - let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); - // Insert into alice - let hash_alice = alice_replica - .hash_and_insert(&key, &author, &value_alice) - .unwrap(); - // Insert into bob - let hash_bob = bob_replica - .hash_and_insert(&key, &author, &value_bob) - .unwrap(); - - assert_eq!( - get_messages(&mut alice_store, namespace.id()), - vec![(author.id(), key.clone(), hash_alice)] - ); - - assert_eq!( - get_messages(&mut bob_store, namespace.id()), - vec![(author.id(), key.clone(), hash_bob)] - ); - - alice_store.close_replica(namespace.id()); - bob_store.close_replica(namespace.id()); - - let alice_handle = SyncHandle::spawn(alice_store, None, "alice".to_string()); - let bob_handle = SyncHandle::spawn(bob_store, None, "bob".to_string()); - - run_sync( - alice_handle.clone(), - alice_node_pubkey, - bob_handle.clone(), - bob_node_pubkey, - namespace.id(), - ) - .await?; - let mut alice_store = alice_handle.shutdown().await?; - let mut bob_store = bob_handle.shutdown().await?; - - assert_eq!( - get_messages(&mut alice_store, namespace.id()), - vec![(author.id(), key.clone(), hash_bob)] - ); - - assert_eq!( - get_messages(&mut bob_store, namespace.id()), - vec![(author.id(), key.clone(), hash_bob)] - ); - - Ok(()) - } -} diff --git a/iroh-docs/src/ranger.rs b/iroh-docs/src/ranger.rs deleted file mode 100644 index 95b696731d6..00000000000 --- a/iroh-docs/src/ranger.rs +++ /dev/null @@ -1,1637 +0,0 @@ -//! Implementation of Set Reconcilliation based on -//! "Range-Based Set Reconciliation" by Aljoscha Meyer. -//! - -use std::{cmp::Ordering, fmt::Debug}; - -use serde::{Deserialize, Serialize}; - -use crate::ContentStatus; - -/// Store entries that can be fingerprinted and put into ranges. -pub trait RangeEntry: Debug + Clone { - /// The key type for this entry. - /// - /// This type must implement [`Ord`] to define the range ordering used in the set - /// reconciliation algorithm. - /// - /// See [`RangeKey`] for details. - type Key: RangeKey; - - /// The value type for this entry. See - /// - /// The type must implement [`Ord`] to define the time ordering of entries used in the prefix - /// deletion algorithm. - /// - /// See [`RangeValue`] for details. - type Value: RangeValue; - - /// Get the key for this entry. - fn key(&self) -> &Self::Key; - - /// Get the value for this entry. - fn value(&self) -> &Self::Value; - - /// Get the fingerprint for this entry. - fn as_fingerprint(&self) -> Fingerprint; -} - -/// A trait constraining types that are valid entry keys. -pub trait RangeKey: Sized + Debug + Ord + PartialEq + Clone + 'static { - /// Returns `true` if `self` is a prefix of `other`. - #[cfg(test)] - fn is_prefix_of(&self, other: &Self) -> bool; - - /// Returns true if `other` is a prefix of `self`. - #[cfg(test)] - fn is_prefixed_by(&self, other: &Self) -> bool { - other.is_prefix_of(self) - } -} - -/// A trait constraining types that are valid entry values. -pub trait RangeValue: Sized + Debug + Ord + PartialEq + Clone + 'static {} - -/// Stores a range. -/// -/// There are three possibilities -/// -/// - x, x: All elements in a set, denoted with -/// - [x, y): x < y: Includes x, but not y -/// - S \ [y, x) y < x: Includes x, but not y. -/// -/// This means that ranges are "wrap around" conceptually. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Default)] -pub struct Range { - x: K, - y: K, -} - -impl Range { - pub fn x(&self) -> &K { - &self.x - } - - pub fn y(&self) -> &K { - &self.y - } - - pub fn new(x: K, y: K) -> Self { - Range { x, y } - } - - pub fn map(self, f: impl FnOnce(K, K) -> (X, X)) -> Range { - let (x, y) = f(self.x, self.y); - Range { x, y } - } -} - -impl Range { - pub fn is_all(&self) -> bool { - self.x() == self.y() - } - - pub fn contains(&self, t: &K) -> bool { - match self.x().cmp(self.y()) { - Ordering::Equal => true, - Ordering::Less => self.x() <= t && t < self.y(), - Ordering::Greater => self.x() <= t || t < self.y(), - } - } -} - -impl From<(K, K)> for Range { - fn from((x, y): (K, K)) -> Self { - Range { x, y } - } -} - -#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] -pub struct Fingerprint(pub [u8; 32]); - -impl Debug for Fingerprint { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Fp({})", blake3::Hash::from(self.0).to_hex()) - } -} - -impl Fingerprint { - /// The fingerprint of the empty set - pub fn empty() -> Self { - Fingerprint(*blake3::hash(&[]).as_bytes()) - } - - pub fn new(val: T) -> Self { - val.as_fingerprint() - } -} - -impl std::ops::BitXorAssign for Fingerprint { - fn bitxor_assign(&mut self, rhs: Self) { - for (a, b) in self.0.iter_mut().zip(rhs.0.iter()) { - *a ^= b; - } - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RangeFingerprint { - #[serde(bound( - serialize = "Range: Serialize", - deserialize = "Range: Deserialize<'de>" - ))] - pub range: Range, - /// The fingerprint of `range`. - pub fingerprint: Fingerprint, -} - -/// Transfers items inside a range to the other participant. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RangeItem { - /// The range out of which the elements are. - #[serde(bound( - serialize = "Range: Serialize", - deserialize = "Range: Deserialize<'de>" - ))] - pub range: Range, - #[serde(bound(serialize = "E: Serialize", deserialize = "E: Deserialize<'de>"))] - pub values: Vec<(E, ContentStatus)>, - /// If false, requests to send local items in the range. - /// Otherwise not. - pub have_local: bool, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum MessagePart { - #[serde(bound( - serialize = "RangeFingerprint: Serialize", - deserialize = "RangeFingerprint: Deserialize<'de>" - ))] - RangeFingerprint(RangeFingerprint), - #[serde(bound( - serialize = "RangeItem: Serialize", - deserialize = "RangeItem: Deserialize<'de>" - ))] - RangeItem(RangeItem), -} - -impl MessagePart { - pub fn is_range_fingerprint(&self) -> bool { - matches!(self, MessagePart::RangeFingerprint(_)) - } - - pub fn is_range_item(&self) -> bool { - matches!(self, MessagePart::RangeItem(_)) - } - - pub fn values(&self) -> Option<&[(E, ContentStatus)]> { - match self { - MessagePart::RangeFingerprint(_) => None, - MessagePart::RangeItem(RangeItem { values, .. }) => Some(values), - } - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct Message { - #[serde(bound( - serialize = "MessagePart: Serialize", - deserialize = "MessagePart: Deserialize<'de>" - ))] - parts: Vec>, -} - -impl Message { - /// Construct the initial message. - fn init>(store: &mut S) -> Result { - let x = store.get_first()?; - let range = Range::new(x.clone(), x); - let fingerprint = store.get_fingerprint(&range)?; - let part = MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }); - Ok(Message { parts: vec![part] }) - } - - pub fn parts(&self) -> &[MessagePart] { - &self.parts - } - - pub fn values(&self) -> impl Iterator { - self.parts().iter().filter_map(|p| p.values()).flatten() - } - - pub fn value_count(&self) -> usize { - self.values().count() - } -} - -pub trait Store: Sized { - type Error: Debug + Send + Sync + Into + 'static; - - type RangeIterator<'a>: Iterator> - where - Self: 'a, - E: 'a; - - type ParentIterator<'a>: Iterator> - where - Self: 'a, - E: 'a; - - /// Get a the first key (or the default if none is available). - fn get_first(&mut self) -> Result; - - /// Get a single entry. - fn get(&mut self, key: &E::Key) -> Result, Self::Error>; - - /// Get the number of entries in the store. - fn len(&mut self) -> Result; - - /// Returns `true` if the vector contains no elements. - fn is_empty(&mut self) -> Result; - - /// Calculate the fingerprint of the given range. - fn get_fingerprint(&mut self, range: &Range) -> Result; - - /// Insert just the given key value pair. - /// - /// This will replace just the existing entry, but will not perform prefix - /// deletion. - fn entry_put(&mut self, entry: E) -> Result<(), Self::Error>; - - /// Returns all entries in the given range. - fn get_range(&mut self, range: Range) -> Result, Self::Error>; - - /// Returns the number of entries in the range. - /// - /// Default impl is not optimized, but does avoid excessive memory usage. - fn get_range_len(&mut self, range: Range) -> Result { - let mut count = 0; - for el in self.get_range(range)? { - let _el = el?; - count += 1; - } - Ok(count) - } - - /// Returns all entries whose key starts with the given `prefix`. - fn prefixed_by(&mut self, prefix: &E::Key) -> Result, Self::Error>; - - /// Returns all entries that share a prefix with `key`, including the entry for `key` itself. - fn prefixes_of(&mut self, key: &E::Key) -> Result, Self::Error>; - - /// Get all entries in the store - fn all(&mut self) -> Result, Self::Error>; - - /// Remove an entry from the store. - /// - /// This will remove just the entry with the given key, but will not perform prefix deletion. - fn entry_remove(&mut self, key: &E::Key) -> Result, Self::Error>; - - /// Remove all entries whose key start with a prefix and for which the `predicate` callback - /// returns true. - /// - /// Returns the number of elements removed. - // TODO: We might want to return an iterator with the removed elements instead to emit as - // events to the application potentially. - fn remove_prefix_filtered( - &mut self, - prefix: &E::Key, - predicate: impl Fn(&E::Value) -> bool, - ) -> Result; - - /// Generates the initial message. - fn initial_message(&mut self) -> Result, Self::Error> { - Message::init(self) - } - - /// Processes an incoming message and produces a response. - /// If terminated, returns `None` - /// - /// `validate_cb` is called for each incoming entry received from the remote. - /// It must return true if the entry is valid and should be stored, and false otherwise - /// (which means the entry will be dropped and not stored). - /// - /// `on_insert_cb` is called for each entry that was actually inserted into the store (so not - /// for entries which validated, but are not inserted because they are older than one of their - /// prefixes). - /// - /// `content_status_cb` is called for each outgoing entry about to be sent to the remote. - /// It must return a [`ContentStatus`], which will be sent to the remote with the entry. - fn process_message( - &mut self, - config: &SyncConfig, - message: Message, - validate_cb: F, - mut on_insert_cb: F2, - content_status_cb: F3, - ) -> Result>, Self::Error> - where - F: Fn(&Self, &E, ContentStatus) -> bool, - F2: FnMut(&Self, E, ContentStatus), - F3: Fn(&Self, &E) -> ContentStatus, - { - let mut out = Vec::new(); - - // TODO: can these allocs be avoided? - let mut items = Vec::new(); - let mut fingerprints = Vec::new(); - for part in message.parts { - match part { - MessagePart::RangeItem(item) => { - items.push(item); - } - MessagePart::RangeFingerprint(fp) => { - fingerprints.push(fp); - } - } - } - - // Process item messages - for RangeItem { - range, - values, - have_local, - } in items - { - let diff: Option> = if have_local { - None - } else { - Some({ - // we get the range of the item form our store. from this set, we remove all - // entries that whose key is contained in the peer's set and where our value is - // lower than the peer entry's value. - let items = self - .get_range(range.clone())? - .filter_map(|our_entry| match our_entry { - Ok(our_entry) => { - if !values.iter().any(|(their_entry, _)| { - our_entry.key() == their_entry.key() - && their_entry.value() >= our_entry.value() - }) { - Some(Ok(our_entry)) - } else { - None - } - } - Err(err) => Some(Err(err)), - }) - .collect::, _>>()?; - // add the content status in a second pass - items - .into_iter() - .map(|entry| { - let content_status = content_status_cb(self, &entry); - (entry, content_status) - }) - .collect() - }) - }; - - // Store incoming values - for (entry, content_status) in values { - if validate_cb(self, &entry, content_status) { - // TODO: Get rid of the clone? - let outcome = self.put(entry.clone())?; - if let InsertOutcome::Inserted { .. } = outcome { - on_insert_cb(self, entry, content_status); - } - } - } - - if let Some(diff) = diff { - if !diff.is_empty() { - out.push(MessagePart::RangeItem(RangeItem { - range, - values: diff, - have_local: true, - })); - } - } - } - - // Process fingerprint messages - for RangeFingerprint { range, fingerprint } in fingerprints { - let local_fingerprint = self.get_fingerprint(&range)?; - // Case1 Match, nothing to do - if local_fingerprint == fingerprint { - continue; - } - - // Case2 Recursion Anchor - let num_local_values = self.get_range_len(range.clone())?; - if num_local_values <= 1 || fingerprint == Fingerprint::empty() { - let values = self - .get_range(range.clone())? - .collect::, _>>()?; - let values = values - .into_iter() - .map(|entry| { - let content_status = content_status_cb(self, &entry); - (entry, content_status) - }) - .collect(); - out.push(MessagePart::RangeItem(RangeItem { - range, - values, - have_local: false, - })); - } else { - // Case3 Recurse - // Create partition - // m0 = x < m1 < .. < mk = y, with k>= 2 - // such that [ml, ml+1) is nonempty - let mut ranges = Vec::with_capacity(config.split_factor); - - // Select the first index, for which the key is larger or equal than the x of the range. - let mut start_index = 0; - for el in self.get_range(range.clone())? { - let el = el?; - if el.key() >= range.x() { - break; - } - start_index += 1; - } - - // select a pivot value. pivots repeat every split_factor, so pivot(i) == pivot(i + self.split_factor * x) - // it is guaranteed that pivot(0) != x if local_values.len() >= 2 - let mut pivot = |i: usize| { - // ensure that pivots wrap around - let i = i % config.split_factor; - // choose an offset. this will be - // 1/2, 1 in case of split_factor == 2 - // 1/3, 2/3, 1 in case of split_factor == 3 - // etc. - let offset = (num_local_values * (i + 1)) / config.split_factor; - let offset = (start_index + offset) % num_local_values; - self.get_range(range.clone()) - .map(|mut i| i.nth(offset)) - .and_then(|e| e.expect("missing entry")) - .map(|e| e.key().clone()) - }; - if range.is_all() { - // the range is the whole set, so range.x and range.y should not matter - // just add all ranges as normal ranges. Exactly one of the ranges will - // wrap around, so we cover the entire set. - for i in 0..config.split_factor { - let (x, y) = (pivot(i)?, pivot(i + 1)?); - // don't push empty ranges - if x != y { - ranges.push(Range { x, y }) - } - } - } else { - // guaranteed to be non-empty because - // - pivot(0) is guaranteed to be != x for local_values.len() >= 2 - // - local_values.len() < 2 gets handled by the recursion anchor - // - x != y (regular range) - ranges.push(Range { - x: range.x().clone(), - y: pivot(0)?, - }); - // this will only be executed for split_factor > 2 - for i in 0..config.split_factor - 2 { - // don't push empty ranges - let (x, y) = (pivot(i)?, pivot(i + 1)?); - if x != y { - ranges.push(Range { x, y }) - } - } - // guaranteed to be non-empty because - // - pivot is a value in the range - // - y is the exclusive end of the range - // - x != y (regular range) - ranges.push(Range { - x: pivot(config.split_factor - 2)?, - y: range.y().clone(), - }); - } - - let mut non_empty = 0; - for range in ranges { - let chunk: Vec<_> = self.get_range(range.clone())?.collect(); - if !chunk.is_empty() { - non_empty += 1; - } - // Add either the fingerprint or the item set - let fingerprint = self.get_fingerprint(&range)?; - if chunk.len() > config.max_set_size { - out.push(MessagePart::RangeFingerprint(RangeFingerprint { - range: range.clone(), - fingerprint, - })); - } else { - let values = chunk - .into_iter() - .map(|entry| { - entry.map(|entry| { - let content_status = content_status_cb(self, &entry); - (entry, content_status) - }) - }) - .collect::>()?; - out.push(MessagePart::RangeItem(RangeItem { - range, - values, - have_local: false, - })); - } - } - debug_assert!(non_empty > 1); - } - } - - // If we have any parts, return a message - if !out.is_empty() { - Ok(Some(Message { parts: out })) - } else { - Ok(None) - } - } - - /// Insert a key value pair. - /// - /// Entries are inserted if they compare strictly greater than all entries in the set of - /// entries which have the same key as `entry` or have a key which is a prefix of `entry`. - /// - /// Additionally, entries that have a key which is a prefix of the entry's key and whose - /// timestamp is not strictly greater than that of the new entry are deleted - /// - /// Note: The deleted entries are simply dropped right now. We might want to make this return - /// an iterator, to potentially log or expose the deleted entries. - /// - /// Returns `true` if the entry was inserted. - /// Returns `false` if it was not inserted. - fn put(&mut self, entry: E) -> Result { - let prefix_entry = self.prefixes_of(entry.key())?; - // First we check if our entry is strictly greater than all parent elements. - // From the willow spec: - // "Remove all entries whose timestamp is strictly less than the timestamp of any other entry [..] - // whose path is a prefix of p." and then "remove all but those whose record has the greatest hash component". - // This is the contract of the `Ord` impl for `E::Value`. - for prefix_entry in prefix_entry { - let prefix_entry = prefix_entry?; - if entry.value() <= prefix_entry.value() { - return Ok(InsertOutcome::NotInserted); - } - } - - // Now we remove all entries that have our key as a prefix and are older than our entry. - let removed = self.remove_prefix_filtered(entry.key(), |value| entry.value() >= value)?; - - // Insert our new entry. - self.entry_put(entry)?; - Ok(InsertOutcome::Inserted { removed }) - } -} - -impl> Store for &mut S { - type Error = S::Error; - - type RangeIterator<'a> - = S::RangeIterator<'a> - where - Self: 'a, - E: 'a; - - type ParentIterator<'a> - = S::ParentIterator<'a> - where - Self: 'a, - E: 'a; - - fn get_first(&mut self) -> Result<::Key, Self::Error> { - (**self).get_first() - } - - fn get(&mut self, key: &::Key) -> Result, Self::Error> { - (**self).get(key) - } - - fn len(&mut self) -> Result { - (**self).len() - } - - fn is_empty(&mut self) -> Result { - (**self).is_empty() - } - - fn get_fingerprint( - &mut self, - range: &Range<::Key>, - ) -> Result { - (**self).get_fingerprint(range) - } - - fn entry_put(&mut self, entry: E) -> Result<(), Self::Error> { - (**self).entry_put(entry) - } - - fn get_range( - &mut self, - range: Range<::Key>, - ) -> Result, Self::Error> { - (**self).get_range(range) - } - - fn prefixed_by( - &mut self, - prefix: &::Key, - ) -> Result, Self::Error> { - (**self).prefixed_by(prefix) - } - - fn prefixes_of( - &mut self, - key: &::Key, - ) -> Result, Self::Error> { - (**self).prefixes_of(key) - } - - fn all(&mut self) -> Result, Self::Error> { - (**self).all() - } - - fn entry_remove(&mut self, key: &::Key) -> Result, Self::Error> { - (**self).entry_remove(key) - } - - fn remove_prefix_filtered( - &mut self, - prefix: &::Key, - predicate: impl Fn(&::Value) -> bool, - ) -> Result { - (**self).remove_prefix_filtered(prefix, predicate) - } -} - -#[derive(Debug, Clone, Copy)] -pub struct SyncConfig { - /// Up to how many values to send immediately, before sending only a fingerprint. - max_set_size: usize, - /// `k` in the protocol, how many splits to generate. at least 2 - split_factor: usize, -} - -impl Default for SyncConfig { - fn default() -> Self { - SyncConfig { - max_set_size: 1, - split_factor: 2, - } - } -} - -/// The outcome of a [`Store::put`] operation. -#[derive(Debug)] -pub enum InsertOutcome { - /// The entry was not inserted because a newer entry for its key or a - /// prefix of its key exists. - NotInserted, - /// The entry was inserted. - Inserted { - /// Number of entries that were removed as a consequence of this insert operation. - /// The removed entries had a key that starts with the new entry's key and a lower value. - removed: usize, - }, -} - -#[cfg(test)] -mod tests { - use std::{cell::RefCell, collections::BTreeMap, convert::Infallible, fmt::Debug, rc::Rc}; - - use proptest::prelude::*; - use test_strategy::proptest; - - use super::*; - - #[derive(Debug)] - struct SimpleStore { - data: BTreeMap, - } - - impl Default for SimpleStore { - fn default() -> Self { - SimpleStore { - data: BTreeMap::default(), - } - } - } - - impl RangeEntry for (K, V) - where - K: RangeKey, - V: RangeValue, - { - type Key = K; - type Value = V; - - fn key(&self) -> &Self::Key { - &self.0 - } - - fn value(&self) -> &Self::Value { - &self.1 - } - - fn as_fingerprint(&self) -> Fingerprint { - let mut hasher = blake3::Hasher::new(); - hasher.update(format!("{:?}", self.0).as_bytes()); - hasher.update(format!("{:?}", self.1).as_bytes()); - Fingerprint(hasher.finalize().into()) - } - } - - impl RangeKey for &'static str { - fn is_prefix_of(&self, other: &Self) -> bool { - other.starts_with(self) - } - } - impl RangeKey for String { - fn is_prefix_of(&self, other: &Self) -> bool { - other.starts_with(self) - } - } - - impl RangeValue for &'static [u8] {} - impl RangeValue for i32 {} - impl RangeValue for u8 {} - impl RangeValue for () {} - - impl Store<(K, V)> for SimpleStore - where - K: RangeKey + Default, - V: RangeValue, - { - type Error = Infallible; - type ParentIterator<'a> = std::vec::IntoIter>; - - fn get_first(&mut self) -> Result { - if let Some((k, _)) = self.data.first_key_value() { - Ok(k.clone()) - } else { - Ok(Default::default()) - } - } - - fn get(&mut self, key: &K) -> Result, Self::Error> { - Ok(self.data.get(key).cloned().map(|v| (key.clone(), v))) - } - - fn len(&mut self) -> Result { - Ok(self.data.len()) - } - - fn is_empty(&mut self) -> Result { - Ok(self.data.is_empty()) - } - - /// Calculate the fingerprint of the given range. - fn get_fingerprint(&mut self, range: &Range) -> Result { - let elements = self.get_range(range.clone())?; - let mut fp = Fingerprint::empty(); - for el in elements { - let el = el?; - fp ^= el.as_fingerprint(); - } - - Ok(fp) - } - - /// Insert the given key value pair. - fn entry_put(&mut self, e: (K, V)) -> Result<(), Self::Error> { - self.data.insert(e.0, e.1); - Ok(()) - } - - type RangeIterator<'a> - = SimpleRangeIterator<'a, K, V> - where - K: 'a, - V: 'a; - /// Returns all items in the given range - fn get_range(&mut self, range: Range) -> Result, Self::Error> { - // TODO: this is not very efficient, optimize depending on data structure - let iter = self.data.iter(); - - Ok(SimpleRangeIterator { - iter, - filter: SimpleFilter::Range(range), - }) - } - - fn entry_remove(&mut self, key: &K) -> Result, Self::Error> { - let res = self.data.remove(key).map(|v| (key.clone(), v)); - Ok(res) - } - - fn all(&mut self) -> Result, Self::Error> { - let iter = self.data.iter(); - Ok(SimpleRangeIterator { - iter, - filter: SimpleFilter::None, - }) - } - - // TODO: Not horrible. - fn prefixes_of(&mut self, key: &K) -> Result, Self::Error> { - let mut res = vec![]; - for (k, v) in self.data.iter() { - if k.is_prefix_of(key) { - res.push(Ok((k.clone(), v.clone()))); - } - } - Ok(res.into_iter()) - } - - fn prefixed_by(&mut self, prefix: &K) -> Result, Self::Error> { - let iter = self.data.iter(); - Ok(SimpleRangeIterator { - iter, - filter: SimpleFilter::Prefix(prefix.clone()), - }) - } - - fn remove_prefix_filtered( - &mut self, - prefix: &K, - predicate: impl Fn(&V) -> bool, - ) -> Result { - let old_len = self.data.len(); - self.data.retain(|k, v| { - let remove = prefix.is_prefix_of(k) && predicate(v); - !remove - }); - Ok(old_len - self.data.len()) - } - } - - #[derive(Debug)] - pub struct SimpleRangeIterator<'a, K, V> { - iter: std::collections::btree_map::Iter<'a, K, V>, - filter: SimpleFilter, - } - - #[derive(Debug)] - enum SimpleFilter { - None, - Range(Range), - Prefix(K), - } - - impl<'a, K, V> Iterator for SimpleRangeIterator<'a, K, V> - where - K: RangeKey + Default, - V: Clone, - { - type Item = Result<(K, V), Infallible>; - - fn next(&mut self) -> Option { - let mut next = self.iter.next()?; - - let filter = |x: &K| match &self.filter { - SimpleFilter::None => true, - SimpleFilter::Range(range) => range.contains(x), - SimpleFilter::Prefix(prefix) => prefix.is_prefix_of(x), - }; - - loop { - if filter(next.0) { - return Some(Ok((next.0.clone(), next.1.clone()))); - } - - next = self.iter.next()?; - } - } - } - - #[test] - fn test_paper_1() { - let alice_set = [("ape", 1), ("eel", 1), ("fox", 1), ("gnu", 1)]; - let bob_set = [ - ("bee", 1), - ("cat", 1), - ("doe", 1), - ("eel", 1), - ("fox", 1), - ("hog", 1), - ]; - - let res = sync(&alice_set, &bob_set); - res.print_messages(); - assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - - // Initial message - assert_eq!(res.alice_to_bob[0].parts.len(), 1); - assert!(res.alice_to_bob[0].parts[0].is_range_fingerprint()); - - // Response from Bob - recurse once - assert_eq!(res.bob_to_alice[0].parts.len(), 2); - assert!(res.bob_to_alice[0].parts[0].is_range_fingerprint()); - assert!(res.bob_to_alice[0].parts[1].is_range_fingerprint()); - // Last response from Alice - assert_eq!(res.alice_to_bob[1].parts.len(), 3); - assert!(res.alice_to_bob[1].parts[0].is_range_fingerprint()); - assert!(res.alice_to_bob[1].parts[1].is_range_fingerprint()); - assert!(res.alice_to_bob[1].parts[2].is_range_item()); - - // Last response from Bob - assert_eq!(res.bob_to_alice[1].parts.len(), 2); - assert!(res.bob_to_alice[1].parts[0].is_range_item()); - assert!(res.bob_to_alice[1].parts[1].is_range_item()); - } - - #[test] - fn test_paper_2() { - let alice_set = [ - ("ape", 1), - ("bee", 1), - ("cat", 1), - ("doe", 1), - ("eel", 1), - ("fox", 1), // the only value being sent - ("gnu", 1), - ("hog", 1), - ]; - let bob_set = [ - ("ape", 1), - ("bee", 1), - ("cat", 1), - ("doe", 1), - ("eel", 1), - ("gnu", 1), - ("hog", 1), - ]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - } - - #[test] - fn test_paper_3() { - let alice_set = [ - ("ape", 1), - ("bee", 1), - ("cat", 1), - ("doe", 1), - ("eel", 1), - ("fox", 1), - ("gnu", 1), - ("hog", 1), - ]; - let bob_set = [("ape", 1), ("cat", 1), ("eel", 1), ("gnu", 1)]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - } - - #[test] - fn test_limits() { - let alice_set = [("ape", 1), ("bee", 1), ("cat", 1)]; - let bob_set = [("ape", 1), ("cat", 1), ("doe", 1)]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - } - - #[test] - fn test_prefixes_simple() { - let alice_set = [("/foo/bar", 1), ("/foo/baz", 1), ("/foo/cat", 1)]; - let bob_set = [("/foo/bar", 1), ("/alice/bar", 1), ("/alice/baz", 1)]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - } - - #[test] - fn test_prefixes_empty_alice() { - let alice_set = []; - let bob_set = [("/foo/bar", 1), ("/alice/bar", 1), ("/alice/baz", 1)]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); - } - - #[test] - fn test_prefixes_empty_bob() { - let alice_set = [("/foo/bar", 1), ("/foo/baz", 1), ("/foo/cat", 1)]; - let bob_set = []; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); - } - - #[test] - fn test_equal_key_higher_value() { - let alice_set = [("foo", 2)]; - let bob_set = [("foo", 1)]; - - let res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); - } - - #[test] - fn test_multikey() { - /// Uses the blanket impl of [`RangeKey]` for `T: AsRef<[u8]>` in this module. - #[derive(Default, Clone, PartialEq, Eq, PartialOrd, Ord)] - struct Multikey { - author: [u8; 4], - key: Vec, - } - - impl RangeKey for Multikey { - fn is_prefix_of(&self, other: &Self) -> bool { - self.author == other.author && self.key.starts_with(&other.key) - } - } - - impl Debug for Multikey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let key = if let Ok(key) = std::str::from_utf8(&self.key) { - key.to_string() - } else { - hex::encode(&self.key) - }; - f.debug_struct("Multikey") - .field("author", &hex::encode(self.author)) - .field("key", &key) - .finish() - } - } - - impl Multikey { - fn new(author: [u8; 4], key: impl AsRef<[u8]>) -> Self { - Multikey { - author, - key: key.as_ref().to_vec(), - } - } - } - let author_a = [1u8; 4]; - let author_b = [2u8; 4]; - let alice_set = [ - (Multikey::new(author_a, "ape"), 1), - (Multikey::new(author_a, "bee"), 1), - (Multikey::new(author_b, "bee"), 1), - (Multikey::new(author_a, "doe"), 1), - ]; - let bob_set = [ - (Multikey::new(author_a, "ape"), 1), - (Multikey::new(author_a, "bee"), 1), - (Multikey::new(author_a, "cat"), 1), - (Multikey::new(author_b, "cat"), 1), - ]; - - // No limit - let mut res = sync(&alice_set, &bob_set); - assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); - assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); - res.assert_alice_set( - "no limit", - &[ - (Multikey::new(author_a, "ape"), 1), - (Multikey::new(author_a, "bee"), 1), - (Multikey::new(author_b, "bee"), 1), - (Multikey::new(author_a, "doe"), 1), - (Multikey::new(author_a, "cat"), 1), - (Multikey::new(author_b, "cat"), 1), - ], - ); - - res.assert_bob_set( - "no limit", - &[ - (Multikey::new(author_a, "ape"), 1), - (Multikey::new(author_a, "bee"), 1), - (Multikey::new(author_b, "bee"), 1), - (Multikey::new(author_a, "doe"), 1), - (Multikey::new(author_a, "cat"), 1), - (Multikey::new(author_b, "cat"), 1), - ], - ); - } - - // This tests two things: - // 1) validate cb returning false leads to no changes on both sides after sync - // 2) validate cb receives expected entries - #[test] - fn test_validate_cb() { - let alice_set = [("alice1", 1), ("alice2", 2)]; - let bob_set = [("bob1", 3), ("bob2", 4), ("bob3", 5)]; - let alice_validate_set = Rc::new(RefCell::new(vec![])); - let bob_validate_set = Rc::new(RefCell::new(vec![])); - - let validate_alice: ValidateCb<&str, i32> = Box::new({ - let alice_validate_set = alice_validate_set.clone(); - move |_, e, _| { - alice_validate_set.borrow_mut().push(*e); - false - } - }); - let validate_bob: ValidateCb<&str, i32> = Box::new({ - let bob_validate_set = bob_validate_set.clone(); - move |_, e, _| { - bob_validate_set.borrow_mut().push(*e); - false - } - }); - - let mut alice = SimpleStore::default(); - for (k, v) in alice_set { - alice.put((k, v)).unwrap(); - } - - let mut bob = SimpleStore::default(); - for (k, v) in bob_set { - bob.put((k, v)).unwrap(); - } - - // run sync with a validate callback returning false, so no new entries are stored on either side - let mut res = sync_exchange_messages(alice, bob, &validate_alice, &validate_bob, 100); - res.assert_alice_set("unchanged", &alice_set); - res.assert_bob_set("unchanged", &bob_set); - - // assert that the validate callbacks received all expected entries - assert_eq!(alice_validate_set.take(), bob_set); - assert_eq!(bob_validate_set.take(), alice_set); - } - - struct SyncResult - where - K: RangeKey + Default, - V: RangeValue, - { - alice: SimpleStore, - bob: SimpleStore, - alice_to_bob: Vec>, - bob_to_alice: Vec>, - } - - impl SyncResult - where - K: RangeKey + Default, - V: RangeValue, - { - fn print_messages(&self) { - let len = std::cmp::max(self.alice_to_bob.len(), self.bob_to_alice.len()); - for i in 0..len { - if let Some(msg) = self.alice_to_bob.get(i) { - println!("A -> B:"); - print_message(msg); - } - if let Some(msg) = self.bob_to_alice.get(i) { - println!("B -> A:"); - print_message(msg); - } - } - } - - fn assert_alice_set(&mut self, ctx: &str, expected: &[(K, V)]) { - dbg!(self.alice.all().unwrap().collect::>()); - for e in expected { - assert_eq!( - self.alice.get(e.key()).unwrap().as_ref(), - Some(e), - "{}: (alice) missing key {:?}", - ctx, - e.key() - ); - } - assert_eq!( - expected.len(), - self.alice.len().unwrap(), - "{}: (alice)", - ctx - ); - } - - fn assert_bob_set(&mut self, ctx: &str, expected: &[(K, V)]) { - dbg!(self.bob.all().unwrap().collect::>()); - - for e in expected { - assert_eq!( - self.bob.get(e.key()).unwrap().as_ref(), - Some(e), - "{}: (bob) missing key {:?}", - ctx, - e - ); - } - assert_eq!(expected.len(), self.bob.len().unwrap(), "{}: (bob)", ctx); - } - } - - fn print_message(msg: &Message) { - for part in &msg.parts { - match part { - MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }) => { - println!( - " RangeFingerprint({:?}, {:?}, {:?})", - range.x(), - range.y(), - fingerprint - ); - } - MessagePart::RangeItem(RangeItem { - range, - values, - have_local, - }) => { - println!( - " RangeItem({:?} | {:?}) (local?: {})\n {:?}", - range.x(), - range.y(), - have_local, - values, - ); - } - } - } - } - - type ValidateCb = Box, &(K, V), ContentStatus) -> bool>; - - fn sync(alice_set: &[(K, V)], bob_set: &[(K, V)]) -> SyncResult - where - K: RangeKey + Default, - V: RangeValue, - { - let alice_validate_cb: ValidateCb = Box::new(|_, _, _| true); - let bob_validate_cb: ValidateCb = Box::new(|_, _, _| true); - sync_with_validate_cb_and_assert(alice_set, bob_set, &alice_validate_cb, &bob_validate_cb) - } - - fn insert_if_larger(map: &mut BTreeMap, key: K, value: V) { - let mut insert = true; - for (k, v) in map.iter() { - if k.is_prefix_of(&key) && v >= &value { - insert = false; - } - } - if insert { - #[allow(clippy::needless_bool)] - map.retain(|k, v| { - if key.is_prefix_of(k) && value >= *v { - false - } else { - true - } - }); - map.insert(key, value); - } - } - - fn sync_with_validate_cb_and_assert( - alice_set: &[(K, V)], - bob_set: &[(K, V)], - alice_validate_cb: F1, - bob_validate_cb: F2, - ) -> SyncResult - where - K: RangeKey + Default, - V: RangeValue, - F1: Fn(&SimpleStore, &(K, V), ContentStatus) -> bool, - F2: Fn(&SimpleStore, &(K, V), ContentStatus) -> bool, - { - let mut alice = SimpleStore::::default(); - let mut bob = SimpleStore::::default(); - - let expected_set = { - let mut expected_set = BTreeMap::new(); - let mut alice_expected = BTreeMap::new(); - for e in alice_set { - alice.put(e.clone()).unwrap(); - insert_if_larger(&mut expected_set, e.0.clone(), e.1.clone()); - insert_if_larger(&mut alice_expected, e.0.clone(), e.1.clone()); - } - let alice_expected = alice_expected.into_iter().collect::>(); - let alice_now: Vec<_> = alice.all().unwrap().collect::>().unwrap(); - assert_eq!( - alice_expected, alice_now, - "alice initial set does not match" - ); - - let mut bob_expected = BTreeMap::new(); - for e in bob_set { - bob.put(e.clone()).unwrap(); - insert_if_larger(&mut expected_set, e.0.clone(), e.1.clone()); - insert_if_larger(&mut bob_expected, e.0.clone(), e.1.clone()); - } - let bob_expected = bob_expected.into_iter().collect::>(); - let bob_now: Vec<_> = bob.all().unwrap().collect::>().unwrap(); - assert_eq!(bob_expected, bob_now, "bob initial set does not match"); - - expected_set.into_iter().collect::>() - }; - - let mut res = sync_exchange_messages(alice, bob, alice_validate_cb, bob_validate_cb, 100); - - let alice_now: Vec<_> = res.alice.all().unwrap().collect::>().unwrap(); - if alice_now != expected_set { - res.print_messages(); - println!("alice_init: {alice_set:?}"); - println!("bob_init: {bob_set:?}"); - println!("expected: {expected_set:?}"); - println!("alice_now: {alice_now:?}"); - panic!("alice_now does not match expected"); - } - - let bob_now: Vec<_> = res.bob.all().unwrap().collect::>().unwrap(); - if bob_now != expected_set { - res.print_messages(); - println!("alice_init: {alice_set:?}"); - println!("bob_init: {bob_set:?}"); - println!("expected: {expected_set:?}"); - println!("bob_now: {bob_now:?}"); - panic!("bob_now does not match expected"); - } - - // Check that values were never sent twice - let mut alice_sent = BTreeMap::new(); - for msg in &res.alice_to_bob { - for part in &msg.parts { - if let Some(values) = part.values() { - for (e, _) in values { - assert!( - alice_sent.insert(e.key(), e).is_none(), - "alice: duplicate {:?}", - e - ); - } - } - } - } - - let mut bob_sent = BTreeMap::new(); - for msg in &res.bob_to_alice { - for part in &msg.parts { - if let Some(values) = part.values() { - for (e, _) in values { - assert!( - bob_sent.insert(e.key(), e).is_none(), - "bob: duplicate {:?}", - e - ); - } - } - } - } - - res - } - - fn sync_exchange_messages( - mut alice: SimpleStore, - mut bob: SimpleStore, - alice_validate_cb: F1, - bob_validate_cb: F2, - max_rounds: usize, - ) -> SyncResult - where - K: RangeKey + Default, - V: RangeValue, - F1: Fn(&SimpleStore, &(K, V), ContentStatus) -> bool, - F2: Fn(&SimpleStore, &(K, V), ContentStatus) -> bool, - { - let mut alice_to_bob = Vec::new(); - let mut bob_to_alice = Vec::new(); - let initial_message = alice.initial_message().unwrap(); - - let mut next_to_bob = Some(initial_message); - let mut rounds = 0; - while let Some(msg) = next_to_bob.take() { - assert!(rounds < max_rounds, "too many rounds"); - rounds += 1; - alice_to_bob.push(msg.clone()); - - if let Some(msg) = bob - .process_message( - &Default::default(), - msg, - &bob_validate_cb, - |_, _, _| (), - |_, _| ContentStatus::Complete, - ) - .unwrap() - { - bob_to_alice.push(msg.clone()); - next_to_bob = alice - .process_message( - &Default::default(), - msg, - &alice_validate_cb, - |_, _, _| (), - |_, _| ContentStatus::Complete, - ) - .unwrap(); - } - } - SyncResult { - alice, - bob, - alice_to_bob, - bob_to_alice, - } - } - - #[test] - fn store_get_range() { - let mut store = SimpleStore::<&'static str, i32>::default(); - let set = [ - ("bee", 1), - ("cat", 1), - ("doe", 1), - ("eel", 1), - ("fox", 1), - ("hog", 1), - ]; - for (k, v) in &set { - store.entry_put((*k, *v)).unwrap(); - } - - let all: Vec<_> = store - .get_range(Range::new("", "")) - .unwrap() - .collect::>() - .unwrap(); - assert_eq!(&all, &set[..]); - - let regular: Vec<_> = store - .get_range(("bee", "eel").into()) - .unwrap() - .collect::>() - .unwrap(); - assert_eq!(®ular, &set[..3]); - - // empty start - let regular: Vec<_> = store - .get_range(("", "eel").into()) - .unwrap() - .collect::>() - .unwrap(); - assert_eq!(®ular, &set[..3]); - - let regular: Vec<_> = store - .get_range(("cat", "hog").into()) - .unwrap() - .collect::>() - .unwrap(); - - assert_eq!(®ular, &set[1..5]); - - let excluded: Vec<_> = store - .get_range(("fox", "bee").into()) - .unwrap() - .collect::>() - .unwrap(); - - assert_eq!(excluded[0].0, "fox"); - assert_eq!(excluded[1].0, "hog"); - assert_eq!(excluded.len(), 2); - - let excluded: Vec<_> = store - .get_range(("fox", "doe").into()) - .unwrap() - .collect::>() - .unwrap(); - - assert_eq!(excluded.len(), 4); - assert_eq!(excluded[0].0, "bee"); - assert_eq!(excluded[1].0, "cat"); - assert_eq!(excluded[2].0, "fox"); - assert_eq!(excluded[3].0, "hog"); - } - - type TestSetStringUnit = BTreeMap; - type TestSetStringU8 = BTreeMap; - - fn test_key() -> impl Strategy { - "[a-z0-9]{0,5}" - } - - fn test_set_string_unit() -> impl Strategy { - prop::collection::btree_map(test_key(), Just(()), 0..10) - } - - fn test_set_string_u8() -> impl Strategy { - prop::collection::btree_map(test_key(), test_value_u8(), 0..10) - } - - fn test_value_u8() -> impl Strategy { - 0u8..u8::MAX - } - - fn test_vec_string_unit() -> impl Strategy> { - test_set_string_unit().prop_map(|m| m.into_iter().collect::>()) - } - fn test_vec_string_u8() -> impl Strategy> { - test_set_string_u8().prop_map(|m| m.into_iter().collect::>()) - } - - fn test_range() -> impl Strategy> { - // ranges with x > y are explicitly allowed - they wrap around - (test_key(), test_key()).prop_map(|(x, y)| Range::new(x, y)) - } - - fn mk_test_set(values: impl IntoIterator>) -> TestSetStringUnit { - values - .into_iter() - .map(|v| v.as_ref().to_string()) - .map(|k| (k, ())) - .collect() - } - - fn mk_test_vec(values: impl IntoIterator>) -> Vec<(String, ())> { - mk_test_set(values).into_iter().collect() - } - - #[test] - fn simple_store_sync_1() { - let alice = mk_test_vec(["3"]); - let bob = mk_test_vec(["2", "3", "4", "5", "6", "7", "8"]); - let _res = sync(&alice, &bob); - } - - #[test] - fn simple_store_sync_x() { - let alice = mk_test_vec(["1", "3"]); - let bob = mk_test_vec(["2"]); - let _res = sync(&alice, &bob); - } - - #[test] - fn simple_store_sync_2() { - let alice = mk_test_vec(["1", "3"]); - let bob = mk_test_vec(["0", "2", "3"]); - let _res = sync(&alice, &bob); - } - - #[test] - fn simple_store_sync_3() { - let alice = mk_test_vec(["8", "9"]); - let bob = mk_test_vec(["1", "2", "3"]); - let _res = sync(&alice, &bob); - } - - #[proptest] - fn simple_store_sync( - #[strategy(test_vec_string_unit())] alice: Vec<(String, ())>, - #[strategy(test_vec_string_unit())] bob: Vec<(String, ())>, - ) { - let _res = sync(&alice, &bob); - } - - #[proptest] - fn simple_store_sync_u8( - #[strategy(test_vec_string_u8())] alice: Vec<(String, u8)>, - #[strategy(test_vec_string_u8())] bob: Vec<(String, u8)>, - ) { - let _res = sync(&alice, &bob); - } - - /// A generic fn to make a test for the get_range fn of a store. - #[allow(clippy::type_complexity)] - fn store_get_ranges_test( - elems: impl IntoIterator, - range: Range, - ) -> (Vec, Vec) - where - S: Store + Default, - E: RangeEntry, - { - let mut store = S::default(); - let elems = elems.into_iter().collect::>(); - for e in elems.iter().cloned() { - store.entry_put(e).unwrap(); - } - let mut actual = store - .get_range(range.clone()) - .unwrap() - .collect::, S::Error>>() - .unwrap(); - let mut expected = elems - .into_iter() - .filter(|e| range.contains(e.key())) - .collect::>(); - - actual.sort_by(|a, b| a.key().cmp(b.key())); - expected.sort_by(|a, b| a.key().cmp(b.key())); - (expected, actual) - } - - #[proptest] - fn simple_store_get_ranges( - #[strategy(test_set_string_unit())] contents: BTreeMap, - #[strategy(test_range())] range: Range, - ) { - let (expected, actual) = store_get_ranges_test::, _>(contents, range); - prop_assert_eq!(expected, actual); - } -} diff --git a/iroh-docs/src/store.rs b/iroh-docs/src/store.rs deleted file mode 100644 index e5946a08ee4..00000000000 --- a/iroh-docs/src/store.rs +++ /dev/null @@ -1,426 +0,0 @@ -//! Storage trait and implementation for iroh-docs documents -use std::num::NonZeroUsize; - -use anyhow::Result; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; - -use crate::{AuthorId, Entry, NamespaceId}; - -pub mod fs; -mod pubkeys; -mod util; -pub use fs::Store; -pub use pubkeys::*; - -/// Number of peers to cache per document. -pub(crate) const PEERS_PER_DOC_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(5) { - Some(val) => val, - None => panic!("this is clearly non zero"), -}; - -/// Error return from [`Store::open_replica`] -#[derive(Debug, thiserror::Error)] -pub enum OpenError { - /// The replica does not exist. - #[error("Replica not found")] - NotFound, - /// Other error while opening the replica. - #[error("{0}")] - Other(#[from] anyhow::Error), -} - -/// Store that gives read access to download policies for a document. -pub trait DownloadPolicyStore { - /// Get the download policy for a document. - fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result; -} - -impl DownloadPolicyStore for &mut T { - fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result { - DownloadPolicyStore::get_download_policy(*self, namespace) - } -} - -impl DownloadPolicyStore for crate::store::Store { - fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result { - self.get_download_policy(namespace) - } -} - -/// Outcome of [`Store::import_namespace`] -#[derive(Debug, Clone, Copy)] -pub enum ImportNamespaceOutcome { - /// The namespace did not exist before and is now inserted. - Inserted, - /// The namespace existed and now has an upgraded capability. - Upgraded, - /// The namespace existed and its capability remains unchanged. - NoChange, -} - -/// Download policy to decide which content blobs shall be downloaded. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum DownloadPolicy { - /// Do not download any key unless it matches one of the filters. - NothingExcept(Vec), - /// Download every key unless it matches one of the filters. - EverythingExcept(Vec), -} - -impl Default for DownloadPolicy { - fn default() -> Self { - DownloadPolicy::EverythingExcept(Vec::default()) - } -} - -/// Filter strategy used in download policies. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum FilterKind { - /// Matches if the contained bytes are a prefix of the key. - Prefix(Bytes), - /// Matches if the contained bytes and the key are the same. - Exact(Bytes), -} - -impl std::fmt::Display for FilterKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // hardly usable but good enough as a poc - let (kind, bytes) = match self { - FilterKind::Prefix(bytes) => ("prefix", bytes), - FilterKind::Exact(bytes) => ("exact", bytes), - }; - let (encoding, repr) = match String::from_utf8(bytes.to_vec()) { - Ok(repr) => ("utf8", repr), - Err(_) => ("hex", hex::encode(bytes)), - }; - write!(f, "{kind}:{encoding}:{repr}") - } -} - -impl std::str::FromStr for FilterKind { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - let Some((kind, rest)) = s.split_once(':') else { - anyhow::bail!("missing filter kind, either \"prefix:\" or \"exact:\"") - }; - let Some((encoding, rest)) = rest.split_once(':') else { - anyhow::bail!("missing encoding: either \"hex:\" or \"utf8:\"") - }; - - let is_exact = match kind { - "exact" => true, - "prefix" => false, - other => { - anyhow::bail!("expected filter kind \"prefix:\" or \"exact:\", found {other}") - } - }; - - let decoded = match encoding { - "utf8" => Bytes::from(rest.to_owned()), - "hex" => match hex::decode(rest) { - Ok(bytes) => Bytes::from(bytes), - Err(_) => anyhow::bail!("failed to decode hex"), - }, - other => { - anyhow::bail!("expected encoding: either \"hex:\" or \"utf8:\", found {other}") - } - }; - - if is_exact { - Ok(FilterKind::Exact(decoded)) - } else { - Ok(FilterKind::Prefix(decoded)) - } - } -} - -impl FilterKind { - /// Verifies whether this filter matches a given key - pub fn matches(&self, key: impl AsRef<[u8]>) -> bool { - match self { - FilterKind::Prefix(prefix) => key.as_ref().starts_with(prefix), - FilterKind::Exact(expected) => expected == key.as_ref(), - } - } -} - -impl DownloadPolicy { - /// Check if an entry should be downloaded according to this policy. - pub fn matches(&self, entry: &Entry) -> bool { - let key = entry.key(); - match self { - DownloadPolicy::NothingExcept(patterns) => { - patterns.iter().any(|pattern| pattern.matches(key)) - } - DownloadPolicy::EverythingExcept(patterns) => { - patterns.iter().all(|pattern| !pattern.matches(key)) - } - } - } -} - -/// A query builder for document queries. -#[derive(Debug, Default)] -pub struct QueryBuilder { - kind: K, - filter_author: AuthorFilter, - filter_key: KeyFilter, - limit: Option, - offset: u64, - include_empty: bool, - sort_direction: SortDirection, -} - -impl QueryBuilder { - /// Call to include empty entries (deletion markers). - pub fn include_empty(mut self) -> Self { - self.include_empty = true; - self - } - /// Filter by exact key match. - pub fn key_exact(mut self, key: impl AsRef<[u8]>) -> Self { - self.filter_key = KeyFilter::Exact(key.as_ref().to_vec().into()); - self - } - /// Filter by key prefix. - pub fn key_prefix(mut self, key: impl AsRef<[u8]>) -> Self { - self.filter_key = KeyFilter::Prefix(key.as_ref().to_vec().into()); - self - } - /// Filter by author. - pub fn author(mut self, author: AuthorId) -> Self { - self.filter_author = AuthorFilter::Exact(author); - self - } - /// Set the maximum number of entries to be returned. - pub fn limit(mut self, limit: u64) -> Self { - self.limit = Some(limit); - self - } - /// Set the offset within the result set from where to start returning results. - pub fn offset(mut self, offset: u64) -> Self { - self.offset = offset; - self - } -} - -/// Query on all entries without aggregation. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct FlatQuery { - sort_by: SortBy, -} - -/// Query that only returns the latest entry for a key which has entries from multiple authors. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct SingleLatestPerKeyQuery {} - -impl QueryBuilder { - /// Set the sort for the query. - /// - /// The default is to sort by author, then by key, in ascending order. - pub fn sort_by(mut self, sort_by: SortBy, direction: SortDirection) -> Self { - self.kind.sort_by = sort_by; - self.sort_direction = direction; - self - } - - /// Build the query. - pub fn build(self) -> Query { - Query::from(self) - } -} - -impl QueryBuilder { - /// Set the order direction for the query. - /// - /// Ordering is always by key for this query type. - /// Default direction is ascending. - pub fn sort_direction(mut self, direction: SortDirection) -> Self { - self.sort_direction = direction; - self - } - - /// Build the query. - pub fn build(self) -> Query { - Query::from(self) - } -} - -impl From> for Query { - fn from(builder: QueryBuilder) -> Query { - Query { - kind: QueryKind::SingleLatestPerKey(builder.kind), - filter_author: builder.filter_author, - filter_key: builder.filter_key, - limit: builder.limit, - offset: builder.offset, - include_empty: builder.include_empty, - sort_direction: builder.sort_direction, - } - } -} - -impl From> for Query { - fn from(builder: QueryBuilder) -> Query { - Query { - kind: QueryKind::Flat(builder.kind), - filter_author: builder.filter_author, - filter_key: builder.filter_key, - limit: builder.limit, - offset: builder.offset, - include_empty: builder.include_empty, - sort_direction: builder.sort_direction, - } - } -} - -/// Note: When using the `SingleLatestPerKey` query kind, the key filter is applied *before* the -/// grouping, the author filter is applied *after* the grouping. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Query { - kind: QueryKind, - filter_author: AuthorFilter, - filter_key: KeyFilter, - limit: Option, - offset: u64, - include_empty: bool, - sort_direction: SortDirection, -} - -impl Query { - /// Query all records. - pub fn all() -> QueryBuilder { - Default::default() - } - /// Query only the latest entry for each key, omitting older entries if the entry was written - /// to by multiple authors. - pub fn single_latest_per_key() -> QueryBuilder { - Default::default() - } - - /// Create a [`Query::all`] query filtered by a single author. - pub fn author(author: AuthorId) -> QueryBuilder { - Self::all().author(author) - } - - /// Create a [`Query::all`] query filtered by a single key. - pub fn key_exact(key: impl AsRef<[u8]>) -> QueryBuilder { - Self::all().key_exact(key) - } - - /// Create a [`Query::all`] query filtered by a key prefix. - pub fn key_prefix(prefix: impl AsRef<[u8]>) -> QueryBuilder { - Self::all().key_prefix(prefix) - } - - /// Get the limit for this query (max. number of entries to emit). - pub fn limit(&self) -> Option { - self.limit - } - - /// Get the offset for this query (number of entries to skip at the beginning). - pub fn offset(&self) -> u64 { - self.offset - } -} - -/// Sort direction -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] -pub enum SortDirection { - /// Sort ascending - #[default] - Asc, - /// Sort descending - Desc, -} - -#[derive(derive_more::Debug, Clone, Serialize, Deserialize)] -enum QueryKind { - #[debug("Flat {{ sort_by: {:?}}}", _0)] - Flat(FlatQuery), - #[debug("SingleLatestPerKey")] - SingleLatestPerKey(SingleLatestPerKeyQuery), -} - -/// Fields by which the query can be sorted -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] -pub enum SortBy { - /// Sort by key, then author. - KeyAuthor, - /// Sort by author, then key. - #[default] - AuthorKey, -} - -/// Key matching. -#[derive(Debug, Serialize, Deserialize, Clone, Default, Eq, PartialEq)] -pub enum KeyFilter { - /// Matches any key. - #[default] - Any, - /// Only keys that are exactly the provided value. - Exact(Bytes), - /// All keys that start with the provided value. - Prefix(Bytes), -} - -impl> From for KeyFilter { - fn from(value: T) -> Self { - KeyFilter::Exact(Bytes::copy_from_slice(value.as_ref())) - } -} - -impl KeyFilter { - /// Test if a key is matched by this [`KeyFilter`]. - pub fn matches(&self, key: &[u8]) -> bool { - match self { - Self::Any => true, - Self::Exact(k) => &k[..] == key, - Self::Prefix(p) => key.starts_with(p), - } - } -} - -/// Author matching. -#[derive(Debug, Serialize, Deserialize, Clone, Default, Eq, PartialEq)] -pub enum AuthorFilter { - /// Matches any author. - #[default] - Any, - /// Matches exactly the provided author. - Exact(AuthorId), -} - -impl AuthorFilter { - /// Test if an author is matched by this [`AuthorFilter`]. - pub fn matches(&self, author: &AuthorId) -> bool { - match self { - Self::Any => true, - Self::Exact(a) => a == author, - } - } -} - -impl From for AuthorFilter { - fn from(value: AuthorId) -> Self { - AuthorFilter::Exact(value) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_filter_kind_encode_decode() { - const REPR: &str = "prefix:utf8:memes/futurama"; - let filter: FilterKind = REPR.parse().expect("should decode"); - assert_eq!( - filter, - FilterKind::Prefix(Bytes::from(String::from("memes/futurama"))) - ); - assert_eq!(filter.to_string(), REPR) - } -} diff --git a/iroh-docs/src/store/fs.rs b/iroh-docs/src/store/fs.rs deleted file mode 100644 index 45723aa6745..00000000000 --- a/iroh-docs/src/store/fs.rs +++ /dev/null @@ -1,1144 +0,0 @@ -//! On disk storage for replicas. - -use std::{ - cmp::Ordering, - collections::HashSet, - iter::{Chain, Flatten}, - num::NonZeroU64, - ops::Bound, - path::Path, -}; - -use anyhow::{anyhow, Result}; -use ed25519_dalek::{SignatureError, VerifyingKey}; -use iroh_base::hash::Hash; -use rand_core::CryptoRngCore; -use redb::{Database, DatabaseError, ReadableMultimapTable, ReadableTable, ReadableTableMetadata}; - -use super::{ - pubkeys::MemPublicKeyStore, DownloadPolicy, ImportNamespaceOutcome, OpenError, PublicKeyStore, - Query, -}; -use crate::{ - actor::MAX_COMMIT_DELAY, - keys::Author, - ranger::{Fingerprint, Range, RangeEntry}, - sync::{Entry, EntrySignature, Record, RecordIdentifier, Replica, SignedEntry}, - AuthorHeads, AuthorId, Capability, CapabilityKind, NamespaceId, NamespaceSecret, PeerIdBytes, - ReplicaInfo, -}; - -mod bounds; -mod migrate_v1_v2; -mod migrations; -mod query; -mod ranges; -pub(crate) mod tables; - -pub use self::ranges::RecordsRange; -use self::{ - bounds::{ByKeyBounds, RecordsBounds}, - query::QueryIterator, - ranges::RangeExt, - tables::{ - LatestPerAuthorKey, LatestPerAuthorValue, ReadOnlyTables, RecordsId, RecordsTable, - RecordsValue, Tables, TransactionAndTables, - }, -}; - -/// Manages the replicas and authors for an instance. -#[derive(Debug)] -pub struct Store { - db: Database, - transaction: CurrentTransaction, - open_replicas: HashSet, - pubkeys: MemPublicKeyStore, -} - -impl AsRef for Store { - fn as_ref(&self) -> &Store { - self - } -} - -impl AsMut for Store { - fn as_mut(&mut self) -> &mut Store { - self - } -} - -#[derive(derive_more::Debug, Default)] -enum CurrentTransaction { - #[default] - None, - Read(ReadOnlyTables), - Write(TransactionAndTables), -} - -impl Store { - /// Create a new store in memory. - pub fn memory() -> Self { - Self::memory_impl().expect("failed to create memory store") - } - - fn memory_impl() -> Result { - let db = Database::builder().create_with_backend(redb::backends::InMemoryBackend::new())?; - Self::new_impl(db) - } - - /// Create or open a store from a `path` to a database file. - /// - /// The file will be created if it does not exist, otherwise it will be opened. - pub fn persistent(path: impl AsRef) -> Result { - let db = match Database::create(&path) { - Ok(db) => db, - Err(DatabaseError::UpgradeRequired(1)) => migrate_v1_v2::run(&path)?, - Err(err) => return Err(err.into()), - }; - Self::new_impl(db) - } - - fn new_impl(db: redb::Database) -> Result { - // Setup all tables - let write_tx = db.begin_write()?; - let _ = Tables::new(&write_tx)?; - write_tx.commit()?; - - // Run database migrations - migrations::run_migrations(&db)?; - - Ok(Store { - db, - transaction: Default::default(), - open_replicas: Default::default(), - pubkeys: Default::default(), - }) - } - - /// Flush the current transaction, if any. - /// - /// This is the cheapest way to ensure that the data is persisted. - pub fn flush(&mut self) -> Result<()> { - if let CurrentTransaction::Write(w) = std::mem::take(&mut self.transaction) { - w.commit()?; - } - Ok(()) - } - - /// Get a read-only snapshot of the database. - /// - /// This has the side effect of committing any open write transaction, - /// so it can be used as a way to ensure that the data is persisted. - pub fn snapshot(&mut self) -> Result<&ReadOnlyTables> { - let guard = &mut self.transaction; - let tables = match std::mem::take(guard) { - CurrentTransaction::None => { - let tx = self.db.begin_read()?; - ReadOnlyTables::new(tx)? - } - CurrentTransaction::Write(w) => { - w.commit()?; - let tx = self.db.begin_read()?; - ReadOnlyTables::new(tx)? - } - CurrentTransaction::Read(tables) => tables, - }; - *guard = CurrentTransaction::Read(tables); - match &*guard { - CurrentTransaction::Read(ref tables) => Ok(tables), - _ => unreachable!(), - } - } - - /// Get an owned read-only snapshot of the database. - /// - /// This will open a new read transaction. The read transaction won't be reused for other - /// reads. - /// - /// This has the side effect of committing any open write transaction, - /// so it can be used as a way to ensure that the data is persisted. - pub fn snapshot_owned(&mut self) -> Result { - // make sure the current transaction is committed - self.flush()?; - assert!(matches!(self.transaction, CurrentTransaction::None)); - let tx = self.db.begin_read()?; - let tables = ReadOnlyTables::new(tx)?; - Ok(tables) - } - - /// Get access to the tables to read from them. - /// - /// The underlying transaction is a write transaction, but with a non-mut - /// reference to the tables you can not write. - /// - /// There is no guarantee that this will be an independent transaction. - /// You just get readonly access to the current state of the database. - /// - /// As such, there is also no guarantee that the data you see is - /// already persisted. - fn tables(&mut self) -> Result<&Tables> { - let guard = &mut self.transaction; - let tables = match std::mem::take(guard) { - CurrentTransaction::None => { - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } - CurrentTransaction::Write(w) => { - if w.since.elapsed() > MAX_COMMIT_DELAY { - tracing::debug!("committing transaction because it's too old"); - w.commit()?; - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } else { - w - } - } - CurrentTransaction::Read(_) => { - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } - }; - *guard = CurrentTransaction::Write(tables); - match guard { - CurrentTransaction::Write(ref mut tables) => Ok(tables.tables()), - _ => unreachable!(), - } - } - - /// Get exclusive write access to the tables in the current transaction. - /// - /// There is no guarantee that this will be an independent transaction. - /// As such, there is also no guarantee that the data you see or write - /// will be persisted. - /// - /// To ensure that the data is persisted, acquire a snapshot of the database - /// or call flush. - fn modify(&mut self, f: impl FnOnce(&mut Tables) -> Result) -> Result { - let guard = &mut self.transaction; - let tables = match std::mem::take(guard) { - CurrentTransaction::None => { - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } - CurrentTransaction::Write(w) => { - if w.since.elapsed() > MAX_COMMIT_DELAY { - tracing::debug!("committing transaction because it's too old"); - w.commit()?; - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } else { - w - } - } - CurrentTransaction::Read(_) => { - let tx = self.db.begin_write()?; - TransactionAndTables::new(tx)? - } - }; - *guard = CurrentTransaction::Write(tables); - let res = match &mut *guard { - CurrentTransaction::Write(ref mut tables) => tables.with_tables_mut(f)?, - _ => unreachable!(), - }; - Ok(res) - } -} - -type PeersIter = std::vec::IntoIter; - -impl Store { - /// Create a new replica for `namespace` and persist in this store. - pub fn new_replica(&mut self, namespace: NamespaceSecret) -> Result { - let id = namespace.id(); - self.import_namespace(namespace.into())?; - self.open_replica(&id).map_err(Into::into) - } - - /// Create a new author key and persist it in the store. - pub fn new_author(&mut self, rng: &mut R) -> Result { - let author = Author::new(rng); - self.import_author(author.clone())?; - Ok(author) - } - - /// Check if a [`AuthorHeads`] contains entry timestamps that we do not have locally. - /// - /// Returns the number of authors that the other peer has updates for. - pub fn has_news_for_us( - &mut self, - namespace: NamespaceId, - heads: &AuthorHeads, - ) -> Result> { - let our_heads = { - let latest = self.get_latest_for_each_author(namespace)?; - let mut heads = AuthorHeads::default(); - for e in latest { - let (author, timestamp, _key) = e?; - heads.insert(author, timestamp); - } - heads - }; - let has_news_for_us = heads.has_news_for(&our_heads); - Ok(has_news_for_us) - } - - /// Open a replica from this store. - /// - /// This just calls load_replica_info and then creates a new replica with the info. - pub fn open_replica(&mut self, namespace_id: &NamespaceId) -> Result { - let info = self.load_replica_info(namespace_id)?; - let instance = StoreInstance::new(*namespace_id, self); - Ok(Replica::new(instance, Box::new(info))) - } - - /// Load the replica info from the store. - pub fn load_replica_info( - &mut self, - namespace_id: &NamespaceId, - ) -> Result { - let tables = self.tables()?; - let info = match tables.namespaces.get(namespace_id.as_bytes()) { - Ok(Some(db_value)) => { - let (raw_kind, raw_bytes) = db_value.value(); - let namespace = Capability::from_raw(raw_kind, raw_bytes)?; - ReplicaInfo::new(namespace) - } - Ok(None) => return Err(OpenError::NotFound), - Err(err) => return Err(OpenError::Other(err.into())), - }; - self.open_replicas.insert(info.capability.id()); - Ok(info) - } - - /// Close a replica. - pub fn close_replica(&mut self, id: NamespaceId) { - self.open_replicas.remove(&id); - } - - /// List all replica namespaces in this store. - pub fn list_namespaces( - &mut self, - ) -> Result>> { - let snapshot = self.snapshot()?; - let iter = snapshot.namespaces.range::<&'static [u8; 32]>(..)?; - let iter = iter.map(|res| { - let capability = parse_capability(res?.1.value())?; - Ok((capability.id(), capability.kind())) - }); - Ok(iter) - } - - /// Get an author key from the store. - pub fn get_author(&mut self, author_id: &AuthorId) -> Result> { - let tables = self.tables()?; - let Some(author) = tables.authors.get(author_id.as_bytes())? else { - return Ok(None); - }; - let author = Author::from_bytes(author.value()); - Ok(Some(author)) - } - - /// Import an author key pair. - pub fn import_author(&mut self, author: Author) -> Result<()> { - self.modify(|tables| { - tables - .authors - .insert(author.id().as_bytes(), &author.to_bytes())?; - Ok(()) - }) - } - - /// Delete an author. - pub fn delete_author(&mut self, author: AuthorId) -> Result<()> { - self.modify(|tables| { - tables.authors.remove(author.as_bytes())?; - Ok(()) - }) - } - - /// List all author keys in this store. - pub fn list_authors(&mut self) -> Result>> { - let tables = self.snapshot()?; - let iter = tables - .authors - .range::<&'static [u8; 32]>(..)? - .map(|res| match res { - Ok((_key, value)) => Ok(Author::from_bytes(value.value())), - Err(err) => Err(err.into()), - }); - Ok(iter) - } - - /// Import a new replica namespace. - pub fn import_namespace(&mut self, capability: Capability) -> Result { - self.modify(|tables| { - let outcome = { - let (capability, outcome) = { - let existing = tables.namespaces.get(capability.id().as_bytes())?; - if let Some(existing) = existing { - let mut existing = parse_capability(existing.value())?; - let outcome = if existing.merge(capability)? { - ImportNamespaceOutcome::Upgraded - } else { - ImportNamespaceOutcome::NoChange - }; - (existing, outcome) - } else { - (capability, ImportNamespaceOutcome::Inserted) - } - }; - let id = capability.id().to_bytes(); - let (kind, bytes) = capability.raw(); - tables.namespaces.insert(&id, (kind, &bytes))?; - outcome - }; - Ok(outcome) - }) - } - - /// Remove a replica. - /// - /// Completely removes a replica and deletes both the namespace private key and all document - /// entries. - /// - /// Note that a replica has to be closed before it can be removed. The store has to enforce - /// that a replica cannot be removed while it is still open. - pub fn remove_replica(&mut self, namespace: &NamespaceId) -> Result<()> { - if self.open_replicas.contains(namespace) { - return Err(anyhow!("replica is not closed")); - } - self.modify(|tables| { - let bounds = RecordsBounds::namespace(*namespace); - tables.records.retain_in(bounds.as_ref(), |_k, _v| false)?; - let bounds = ByKeyBounds::namespace(*namespace); - let _ = tables - .records_by_key - .retain_in(bounds.as_ref(), |_k, _v| false); - tables.namespaces.remove(namespace.as_bytes())?; - tables.namespace_peers.remove_all(namespace.as_bytes())?; - tables.download_policy.remove(namespace.as_bytes())?; - Ok(()) - }) - } - - /// Get an iterator over entries of a replica. - pub fn get_many( - &mut self, - namespace: NamespaceId, - query: impl Into, - ) -> Result { - let tables = self.snapshot_owned()?; - QueryIterator::new(tables, namespace, query.into()) - } - - /// Get an entry by key and author. - pub fn get_exact( - &mut self, - namespace: NamespaceId, - author: AuthorId, - key: impl AsRef<[u8]>, - include_empty: bool, - ) -> Result> { - get_exact( - &self.tables()?.records, - namespace, - author, - key, - include_empty, - ) - } - - /// Get all content hashes of all replicas in the store. - pub fn content_hashes(&mut self) -> Result { - let tables = self.snapshot_owned()?; - ContentHashesIterator::all(&tables.records) - } - - /// Get the latest entry for each author in a namespace. - pub fn get_latest_for_each_author(&mut self, namespace: NamespaceId) -> Result { - LatestIterator::new(&self.tables()?.latest_per_author, namespace) - } - - /// Register a peer that has been useful to sync a document. - pub fn register_useful_peer( - &mut self, - namespace: NamespaceId, - peer: crate::PeerIdBytes, - ) -> Result<()> { - let peer = &peer; - let namespace = namespace.as_bytes(); - // calculate nanos since UNIX_EPOCH for a time measurement - let nanos = std::time::UNIX_EPOCH - .elapsed() - .map(|duration| duration.as_nanos() as u64)?; - self.modify(|tables| { - // ensure the document exists - anyhow::ensure!( - tables.namespaces.get(namespace)?.is_some(), - "document not created" - ); - - let mut namespace_peers = tables.namespace_peers.get(namespace)?; - - // get the oldest entry since it's candidate for removal - let maybe_oldest = namespace_peers.next().transpose()?.map(|guard| { - let (oldest_nanos, &oldest_peer) = guard.value(); - (oldest_nanos, oldest_peer) - }); - match maybe_oldest { - None => { - // the table is empty so the peer can be inserted without further checks since - // super::PEERS_PER_DOC_CACHE_SIZE is non zero - drop(namespace_peers); - tables.namespace_peers.insert(namespace, (nanos, peer))?; - } - Some((oldest_nanos, oldest_peer)) => { - let oldest_peer = &oldest_peer; - - if oldest_peer == peer { - // oldest peer is the current one, so replacing the entry for the peer will - // maintain the size - drop(namespace_peers); - tables - .namespace_peers - .remove(namespace, (oldest_nanos, oldest_peer))?; - tables.namespace_peers.insert(namespace, (nanos, peer))?; - } else { - // calculate the len in the same loop since calling `len` is another fallible operation - let mut len = 1; - // find any previous entry for the same peer to remove it - let mut prev_peer_nanos = None; - - for result in namespace_peers { - len += 1; - let guard = result?; - let (peer_nanos, peer_bytes) = guard.value(); - if prev_peer_nanos.is_none() && peer_bytes == peer { - prev_peer_nanos = Some(peer_nanos) - } - } - - match prev_peer_nanos { - Some(prev_nanos) => { - // the peer was already present, so we can remove the old entry and - // insert the new one without checking the size - tables - .namespace_peers - .remove(namespace, (prev_nanos, peer))?; - tables.namespace_peers.insert(namespace, (nanos, peer))?; - } - None => { - // the peer is new and the table is non empty, add it and check the - // size to decide if the oldest peer should be evicted - tables.namespace_peers.insert(namespace, (nanos, peer))?; - len += 1; - if len > super::PEERS_PER_DOC_CACHE_SIZE.get() { - tables - .namespace_peers - .remove(namespace, (oldest_nanos, oldest_peer))?; - } - } - } - } - } - } - Ok(()) - }) - } - - /// Get the peers that have been useful for a document. - pub fn get_sync_peers(&mut self, namespace: &NamespaceId) -> Result> { - let tables = self.tables()?; - let mut peers = Vec::with_capacity(super::PEERS_PER_DOC_CACHE_SIZE.get()); - for result in tables.namespace_peers.get(namespace.as_bytes())?.rev() { - let (_nanos, &peer) = result?.value(); - peers.push(peer); - } - if peers.is_empty() { - Ok(None) - } else { - Ok(Some(peers.into_iter())) - } - } - - /// Set the download policy for a namespace. - pub fn set_download_policy( - &mut self, - namespace: &NamespaceId, - policy: DownloadPolicy, - ) -> Result<()> { - self.modify(|tables| { - let namespace = namespace.as_bytes(); - - // ensure the document exists - anyhow::ensure!( - tables.namespaces.get(&namespace)?.is_some(), - "document not created" - ); - - let value = postcard::to_stdvec(&policy)?; - tables.download_policy.insert(namespace, value.as_slice())?; - Ok(()) - }) - } - - /// Get the download policy for a namespace. - pub fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result { - let tables = self.tables()?; - let value = tables.download_policy.get(namespace.as_bytes())?; - Ok(match value { - None => DownloadPolicy::default(), - Some(value) => postcard::from_bytes(value.value())?, - }) - } -} - -impl PublicKeyStore for Store { - fn public_key(&self, id: &[u8; 32]) -> Result { - self.pubkeys.public_key(id) - } -} - -fn parse_capability((raw_kind, raw_bytes): (u8, &[u8; 32])) -> Result { - Capability::from_raw(raw_kind, raw_bytes) -} - -fn get_exact( - record_table: &impl ReadableTable, RecordsValue<'static>>, - namespace: NamespaceId, - author: AuthorId, - key: impl AsRef<[u8]>, - include_empty: bool, -) -> Result> { - let id = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); - let record = record_table.get(id)?; - Ok(record - .map(|r| into_entry(id, r.value())) - .filter(|entry| include_empty || !entry.is_empty())) -} - -/// A wrapper around [`Store`] for a specific [`NamespaceId`] -#[derive(Debug)] -pub struct StoreInstance<'a> { - namespace: NamespaceId, - pub(crate) store: &'a mut Store, -} - -impl<'a> StoreInstance<'a> { - pub(crate) fn new(namespace: NamespaceId, store: &'a mut Store) -> Self { - StoreInstance { namespace, store } - } -} - -impl<'a> PublicKeyStore for StoreInstance<'a> { - fn public_key(&self, id: &[u8; 32]) -> std::result::Result { - self.store.public_key(id) - } -} - -impl<'a> super::DownloadPolicyStore for StoreInstance<'a> { - fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result { - self.store.get_download_policy(namespace) - } -} - -impl<'a> crate::ranger::Store for StoreInstance<'a> { - type Error = anyhow::Error; - type RangeIterator<'x> - = Chain, Flatten>>> - where - 'a: 'x; - type ParentIterator<'x> - = ParentIterator - where - 'a: 'x; - - /// Get a the first key (or the default if none is available). - fn get_first(&mut self) -> Result { - let tables = self.store.as_mut().tables()?; - // TODO: verify this fetches all keys with this namespace - let bounds = RecordsBounds::namespace(self.namespace); - let mut records = tables.records.range(bounds.as_ref())?; - - let Some(record) = records.next() else { - return Ok(RecordIdentifier::default()); - }; - let (compound_key, _value) = record?; - let (namespace_id, author_id, key) = compound_key.value(); - let id = RecordIdentifier::new(namespace_id, author_id, key); - Ok(id) - } - - fn get(&mut self, id: &RecordIdentifier) -> Result> { - self.store - .as_mut() - .get_exact(id.namespace(), id.author(), id.key(), true) - } - - fn len(&mut self) -> Result { - let tables = self.store.as_mut().tables()?; - let bounds = RecordsBounds::namespace(self.namespace); - let records = tables.records.range(bounds.as_ref())?; - Ok(records.count()) - } - - fn is_empty(&mut self) -> Result { - let tables = self.store.as_mut().tables()?; - Ok(tables.records.is_empty()?) - } - - fn get_fingerprint(&mut self, range: &Range) -> Result { - // TODO: optimize - let elements = self.get_range(range.clone())?; - - let mut fp = Fingerprint::empty(); - for el in elements { - let el = el?; - fp ^= el.as_fingerprint(); - } - - Ok(fp) - } - - fn entry_put(&mut self, e: SignedEntry) -> Result<()> { - let id = e.id(); - self.store.as_mut().modify(|tables| { - // insert into record table - let key = ( - &id.namespace().to_bytes(), - &id.author().to_bytes(), - id.key(), - ); - let hash = e.content_hash(); // let binding is needed - let value = ( - e.timestamp(), - &e.signature().namespace().to_bytes(), - &e.signature().author().to_bytes(), - e.content_len(), - hash.as_bytes(), - ); - tables.records.insert(key, value)?; - - // insert into by key index table - let key = ( - &id.namespace().to_bytes(), - id.key(), - &id.author().to_bytes(), - ); - tables.records_by_key.insert(key, ())?; - - // insert into latest table - let key = (&e.id().namespace().to_bytes(), &e.id().author().to_bytes()); - let value = (e.timestamp(), e.id().key()); - tables.latest_per_author.insert(key, value)?; - Ok(()) - }) - } - - fn get_range(&mut self, range: Range) -> Result> { - let tables = self.store.as_mut().tables()?; - let iter = match range.x().cmp(range.y()) { - // identity range: iter1 = all, iter2 = none - Ordering::Equal => { - // iterator for all entries in replica - let bounds = RecordsBounds::namespace(self.namespace); - let iter = RecordsRange::with_bounds(&tables.records, bounds)?; - chain_none(iter) - } - // regular range: iter1 = x <= t < y, iter2 = none - Ordering::Less => { - // iterator for entries from range.x to range.y - let start = Bound::Included(range.x().to_byte_tuple()); - let end = Bound::Excluded(range.y().to_byte_tuple()); - let bounds = RecordsBounds::new(start, end); - let iter = RecordsRange::with_bounds(&tables.records, bounds)?; - chain_none(iter) - } - // split range: iter1 = start <= t < y, iter2 = x <= t <= end - Ordering::Greater => { - // iterator for entries from start to range.y - let end = Bound::Excluded(range.y().to_byte_tuple()); - let bounds = RecordsBounds::from_start(&self.namespace, end); - let iter = RecordsRange::with_bounds(&tables.records, bounds)?; - - // iterator for entries from range.x to end - let start = Bound::Included(range.x().to_byte_tuple()); - let bounds = RecordsBounds::to_end(&self.namespace, start); - let iter2 = RecordsRange::with_bounds(&tables.records, bounds)?; - - iter.chain(Some(iter2).into_iter().flatten()) - } - }; - Ok(iter) - } - - fn entry_remove(&mut self, id: &RecordIdentifier) -> Result> { - self.store.as_mut().modify(|tables| { - let entry = { - let (namespace, author, key) = id.as_byte_tuple(); - let id = (namespace, key, author); - tables.records_by_key.remove(id)?; - let id = (namespace, author, key); - let value = tables.records.remove(id)?; - value.map(|value| into_entry(id, value.value())) - }; - Ok(entry) - }) - } - - fn all(&mut self) -> Result> { - let tables = self.store.as_mut().tables()?; - let bounds = RecordsBounds::namespace(self.namespace); - let iter = RecordsRange::with_bounds(&tables.records, bounds)?; - Ok(chain_none(iter)) - } - - fn prefixes_of( - &mut self, - id: &RecordIdentifier, - ) -> Result, Self::Error> { - let tables = self.store.as_mut().tables()?; - ParentIterator::new(tables, id.namespace(), id.author(), id.key().to_vec()) - } - - fn prefixed_by(&mut self, id: &RecordIdentifier) -> Result> { - let tables = self.store.as_mut().tables()?; - let bounds = RecordsBounds::author_prefix(id.namespace(), id.author(), id.key_bytes()); - let iter = RecordsRange::with_bounds(&tables.records, bounds)?; - Ok(chain_none(iter)) - } - - fn remove_prefix_filtered( - &mut self, - id: &RecordIdentifier, - predicate: impl Fn(&Record) -> bool, - ) -> Result { - let bounds = RecordsBounds::author_prefix(id.namespace(), id.author(), id.key_bytes()); - self.store.as_mut().modify(|tables| { - let cb = |_k: RecordsId, v: RecordsValue| { - let (timestamp, _namespace_sig, _author_sig, len, hash) = v; - let record = Record::new(hash.into(), len, timestamp); - - predicate(&record) - }; - let iter = tables.records.extract_from_if(bounds.as_ref(), cb)?; - let count = iter.count(); - Ok(count) - }) - } -} - -fn chain_none<'a, I: Iterator + 'a, T>( - iter: I, -) -> Chain>> { - iter.chain(None.into_iter().flatten()) -} - -/// Iterator over parent entries, i.e. entries with the same namespace and author, and a key which -/// is a prefix of the key passed to the iterator. -#[derive(Debug)] -pub struct ParentIterator { - inner: std::vec::IntoIter>, -} - -impl ParentIterator { - fn new( - tables: &Tables, - namespace: NamespaceId, - author: AuthorId, - key: Vec, - ) -> anyhow::Result { - let parents = parents(&tables.records, namespace, author, key.clone()); - Ok(Self { - inner: parents.into_iter(), - }) - } -} - -fn parents( - table: &impl ReadableTable, RecordsValue<'static>>, - namespace: NamespaceId, - author: AuthorId, - mut key: Vec, -) -> Vec> { - let mut res = Vec::new(); - - while !key.is_empty() { - let entry = get_exact(table, namespace, author, &key, false); - key.pop(); - match entry { - Err(err) => res.push(Err(err)), - Ok(Some(entry)) => res.push(Ok(entry)), - Ok(None) => continue, - } - } - res.reverse(); - res -} - -impl Iterator for ParentIterator { - type Item = Result; - - fn next(&mut self) -> Option { - self.inner.next() - } -} - -/// Iterator for all content hashes -/// -/// Note that you might get duplicate hashes. Also, the iterator will keep -/// a database snapshot open until it is dropped. -/// -/// Also, this represents a snapshot of the database at the time of creation. -/// It needs a copy of a redb::ReadOnlyTable to be self-contained. -#[derive(derive_more::Debug)] -pub struct ContentHashesIterator { - #[debug(skip)] - range: RecordsRange<'static>, -} - -impl ContentHashesIterator { - /// Create a new iterator over all content hashes. - pub fn all(table: &RecordsTable) -> anyhow::Result { - let range = RecordsRange::all_static(table)?; - Ok(Self { range }) - } -} - -impl Iterator for ContentHashesIterator { - type Item = Result; - - fn next(&mut self) -> Option { - let v = self.range.next()?; - Some(v.map(|e| e.content_hash())) - } -} - -/// Iterator over the latest entry per author. -#[derive(derive_more::Debug)] -#[debug("LatestIterator")] -pub struct LatestIterator<'a>( - redb::Range<'a, LatestPerAuthorKey<'static>, LatestPerAuthorValue<'static>>, -); - -impl<'a> LatestIterator<'a> { - fn new( - latest_per_author: &'a impl ReadableTable< - LatestPerAuthorKey<'static>, - LatestPerAuthorValue<'static>, - >, - namespace: NamespaceId, - ) -> anyhow::Result { - let start = (namespace.as_bytes(), &[u8::MIN; 32]); - let end = (namespace.as_bytes(), &[u8::MAX; 32]); - let range = latest_per_author.range(start..=end)?; - Ok(Self(range)) - } -} - -impl<'a> Iterator for LatestIterator<'a> { - type Item = Result<(AuthorId, u64, Vec)>; - - fn next(&mut self) -> Option { - self.0.next_map(|key, value| { - let (_namespace, author) = key; - let (timestamp, key) = value; - (author.into(), timestamp, key.to_vec()) - }) - } -} - -fn into_entry(key: RecordsId, value: RecordsValue) -> SignedEntry { - let (namespace, author, key) = key; - let (timestamp, namespace_sig, author_sig, len, hash) = value; - let id = RecordIdentifier::new(namespace, author, key); - let record = Record::new(hash.into(), len, timestamp); - let entry = Entry::new(id, record); - let entry_signature = EntrySignature::from_parts(namespace_sig, author_sig); - SignedEntry::new(entry_signature, entry) -} - -#[cfg(test)] -mod tests { - use super::{tables::LATEST_PER_AUTHOR_TABLE, *}; - use crate::ranger::Store as _; - - #[test] - fn test_ranges() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let mut store = Store::persistent(dbfile.path())?; - - let author = store.new_author(&mut rand::thread_rng())?; - let namespace = NamespaceSecret::new(&mut rand::thread_rng()); - let mut replica = store.new_replica(namespace.clone())?; - - // test author prefix relation for all-255 keys - let key1 = vec![255, 255]; - let key2 = vec![255, 255, 255]; - replica.hash_and_insert(&key1, &author, b"v1")?; - replica.hash_and_insert(&key2, &author, b"v2")?; - let res = store - .get_many(namespace.id(), Query::author(author.id()).key_prefix([255]))? - .collect::>>()?; - assert_eq!(res.len(), 2); - assert_eq!( - res.into_iter() - .map(|entry| entry.key().to_vec()) - .collect::>(), - vec![key1, key2] - ); - Ok(()) - } - - #[test] - fn test_basics() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let mut store = Store::persistent(dbfile.path())?; - - let authors: Vec<_> = store.list_authors()?.collect::>()?; - assert!(authors.is_empty()); - - let author = store.new_author(&mut rand::thread_rng())?; - let namespace = NamespaceSecret::new(&mut rand::thread_rng()); - let _replica = store.new_replica(namespace.clone())?; - store.close_replica(namespace.id()); - let replica = store.load_replica_info(&namespace.id())?; - assert_eq!(replica.capability.id(), namespace.id()); - - let author_back = store.get_author(&author.id())?.unwrap(); - assert_eq!(author.to_bytes(), author_back.to_bytes(),); - - let mut wrapper = StoreInstance::new(namespace.id(), &mut store); - for i in 0..5 { - let id = RecordIdentifier::new(namespace.id(), author.id(), format!("hello-{i}")); - let entry = Entry::new(id, Record::current_from_data(format!("world-{i}"))); - let entry = SignedEntry::from_entry(entry, &namespace, &author); - wrapper.entry_put(entry)?; - } - - // all - let all: Vec<_> = wrapper.all()?.collect(); - assert_eq!(all.len(), 5); - - // add a second version - let mut ids = Vec::new(); - for i in 0..5 { - let id = RecordIdentifier::new(namespace.id(), author.id(), format!("hello-{i}")); - let entry = Entry::new( - id.clone(), - Record::current_from_data(format!("world-{i}-2")), - ); - let entry = SignedEntry::from_entry(entry, &namespace, &author); - wrapper.entry_put(entry)?; - ids.push(id); - } - - // get all - let entries = wrapper - .store - .get_many(namespace.id(), Query::all())? - .collect::>>()?; - assert_eq!(entries.len(), 5); - - // get all prefix - let entries = wrapper - .store - .get_many(namespace.id(), Query::key_prefix("hello-"))? - .collect::>>()?; - assert_eq!(entries.len(), 5); - - // delete and get - for id in ids { - let res = wrapper.get(&id)?; - assert!(res.is_some()); - let out = wrapper.entry_remove(&id)?.unwrap(); - assert_eq!(out.entry().id(), &id); - let res = wrapper.get(&id)?; - assert!(res.is_none()); - } - - // get latest - let entries = wrapper - .store - .get_many(namespace.id(), Query::all())? - .collect::>>()?; - assert_eq!(entries.len(), 0); - - Ok(()) - } - - fn copy_and_modify( - source: &Path, - modify: impl Fn(&redb::WriteTransaction) -> Result<()>, - ) -> Result { - let dbfile = tempfile::NamedTempFile::new()?; - std::fs::copy(source, dbfile.path())?; - let db = Database::create(dbfile.path())?; - let write_tx = db.begin_write()?; - modify(&write_tx)?; - write_tx.commit()?; - drop(db); - Ok(dbfile) - } - - #[test] - fn test_migration_001_populate_latest_table() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let namespace = NamespaceSecret::new(&mut rand::thread_rng()); - - // create a store and add some data - let expected = { - let mut store = Store::persistent(dbfile.path())?; - let author1 = store.new_author(&mut rand::thread_rng())?; - let author2 = store.new_author(&mut rand::thread_rng())?; - let mut replica = store.new_replica(namespace.clone())?; - replica.hash_and_insert(b"k1", &author1, b"v1")?; - replica.hash_and_insert(b"k2", &author2, b"v1")?; - replica.hash_and_insert(b"k3", &author1, b"v1")?; - - let expected = store - .get_latest_for_each_author(namespace.id())? - .collect::>>()?; - // drop everything to clear file locks. - store.close_replica(namespace.id()); - // flush the store to disk - store.flush()?; - drop(store); - expected - }; - assert_eq!(expected.len(), 2); - - // create a copy of our db file with the latest table deleted. - let dbfile_before_migration = copy_and_modify(dbfile.path(), |tx| { - tx.delete_table(LATEST_PER_AUTHOR_TABLE)?; - Ok(()) - })?; - - // open the copied db file, which will run the migration. - let mut store = Store::persistent(dbfile_before_migration.path())?; - let actual = store - .get_latest_for_each_author(namespace.id())? - .collect::>>()?; - - assert_eq!(expected, actual); - - Ok(()) - } - - #[test] - fn test_migration_004_populate_by_key_index() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - - let mut store = Store::persistent(dbfile.path())?; - - // check that the new table is there, even if empty - { - let tables = store.tables()?; - assert_eq!(tables.records_by_key.len()?, 0); - } - - // TODO: write test checking that the indexing is done correctly - - Ok(()) - } -} diff --git a/iroh-docs/src/store/fs/bounds.rs b/iroh-docs/src/store/fs/bounds.rs deleted file mode 100644 index fd35a081669..00000000000 --- a/iroh-docs/src/store/fs/bounds.rs +++ /dev/null @@ -1,295 +0,0 @@ -use std::ops::{Bound, RangeBounds}; - -use bytes::Bytes; - -use super::tables::{RecordsByKeyId, RecordsByKeyIdOwned, RecordsId, RecordsIdOwned}; -use crate::{store::KeyFilter, AuthorId, NamespaceId}; - -/// Bounds on the records table. -/// -/// Supports bounds by author, key -pub struct RecordsBounds(Bound, Bound); - -impl RecordsBounds { - pub fn new(start: Bound, end: Bound) -> Self { - Self(start, end) - } - - pub fn author_key(ns: NamespaceId, author: AuthorId, key_matcher: KeyFilter) -> Self { - let key_is_exact = matches!(key_matcher, KeyFilter::Exact(_)); - let key = match key_matcher { - KeyFilter::Any => Bytes::new(), - KeyFilter::Exact(key) => key, - KeyFilter::Prefix(prefix) => prefix, - }; - let author = author.to_bytes(); - let ns = ns.to_bytes(); - let mut author_end = author; - let mut ns_end = ns; - let mut key_end = key.to_vec(); - - let start = (ns, author, key); - - let end = if key_is_exact { - Bound::Included(start.clone()) - } else if increment_by_one(&mut key_end) { - Bound::Excluded((ns, author, key_end.into())) - } else if increment_by_one(&mut author_end) { - Bound::Excluded((ns, author_end, Bytes::new())) - } else if increment_by_one(&mut ns_end) { - Bound::Excluded((ns_end, [0u8; 32], Bytes::new())) - } else { - Bound::Unbounded - }; - - Self(Bound::Included(start), end) - } - - pub fn author_prefix(ns: NamespaceId, author: AuthorId, prefix: Bytes) -> Self { - RecordsBounds::author_key(ns, author, KeyFilter::Prefix(prefix)) - } - - pub fn namespace(ns: NamespaceId) -> Self { - Self::new(Self::namespace_start(&ns), Self::namespace_end(&ns)) - } - - pub fn from_start(ns: &NamespaceId, end: Bound) -> Self { - Self::new(Self::namespace_start(ns), end) - } - - pub fn to_end(ns: &NamespaceId, start: Bound) -> Self { - Self::new(start, Self::namespace_end(ns)) - } - - pub fn as_ref(&self) -> (Bound, Bound) { - fn map(id: &RecordsIdOwned) -> RecordsId { - (&id.0, &id.1, &id.2[..]) - } - (map_bound(&self.0, map), map_bound(&self.1, map)) - } - - fn namespace_start(namespace: &NamespaceId) -> Bound { - Bound::Included((namespace.to_bytes(), [0u8; 32], Bytes::new())) - } - - fn namespace_end(namespace: &NamespaceId) -> Bound { - let mut ns_end = namespace.to_bytes(); - if increment_by_one(&mut ns_end) { - Bound::Excluded((ns_end, [0u8; 32], Bytes::new())) - } else { - Bound::Unbounded - } - } -} - -impl RangeBounds for RecordsBounds { - fn start_bound(&self) -> Bound<&RecordsIdOwned> { - map_bound(&self.0, |s| s) - } - - fn end_bound(&self) -> Bound<&RecordsIdOwned> { - map_bound(&self.1, |s| s) - } -} - -impl From<(Bound, Bound)> for RecordsBounds { - fn from(value: (Bound, Bound)) -> Self { - Self::new(value.0, value.1) - } -} - -/// Bounds for the by-key index table. -/// -/// Supports bounds by key. -pub struct ByKeyBounds(Bound, Bound); -impl ByKeyBounds { - pub fn new(ns: NamespaceId, matcher: &KeyFilter) -> Self { - match matcher { - KeyFilter::Any => Self::namespace(ns), - KeyFilter::Exact(key) => { - let start = (ns.to_bytes(), key.clone(), [0u8; 32]); - let end = (ns.to_bytes(), key.clone(), [255u8; 32]); - Self(Bound::Included(start), Bound::Included(end)) - } - KeyFilter::Prefix(ref prefix) => { - let start = Bound::Included((ns.to_bytes(), prefix.clone(), [0u8; 32])); - - let mut ns_end = ns.to_bytes(); - let mut key_end = prefix.to_vec(); - let end = if increment_by_one(&mut key_end) { - Bound::Excluded((ns.to_bytes(), key_end.into(), [0u8; 32])) - } else if increment_by_one(&mut ns_end) { - Bound::Excluded((ns_end, Bytes::new(), [0u8; 32])) - } else { - Bound::Unbounded - }; - Self(start, end) - } - } - } - - pub fn namespace(ns: NamespaceId) -> Self { - let start = Bound::Included((ns.to_bytes(), Bytes::new(), [0u8; 32])); - let mut ns_end = ns.to_bytes(); - let end = if increment_by_one(&mut ns_end) { - Bound::Excluded((ns_end, Bytes::new(), [0u8; 32])) - } else { - Bound::Unbounded - }; - Self(start, end) - } - - pub fn as_ref(&self) -> (Bound, Bound) { - fn map(id: &RecordsByKeyIdOwned) -> RecordsByKeyId { - (&id.0, &id.1[..], &id.2) - } - (map_bound(&self.0, map), map_bound(&self.1, map)) - } -} - -impl RangeBounds for ByKeyBounds { - fn start_bound(&self) -> Bound<&RecordsByKeyIdOwned> { - map_bound(&self.0, |s| s) - } - - fn end_bound(&self) -> Bound<&RecordsByKeyIdOwned> { - map_bound(&self.1, |s| s) - } -} - -/// Increment a byte string by one, by incrementing the last byte that is not 255 by one. -/// -/// Returns false if all bytes are 255. -fn increment_by_one(value: &mut [u8]) -> bool { - for char in value.iter_mut().rev() { - if *char != 255 { - *char += 1; - return true; - } else { - *char = 0; - } - } - false -} - -fn map_bound<'a, T, U: 'a>(bound: &'a Bound, f: impl Fn(&'a T) -> U) -> Bound { - match bound { - Bound::Unbounded => Bound::Unbounded, - Bound::Included(t) => Bound::Included(f(t)), - Bound::Excluded(t) => Bound::Excluded(f(t)), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn records_bounds() { - let ns = NamespaceId::from(&[255u8; 32]); - - let bounds = RecordsBounds::namespace(ns); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), [0u8; 32], Bytes::new())) - ); - assert_eq!(bounds.end_bound(), Bound::Unbounded); - - let a = AuthorId::from(&[255u8; 32]); - - let bounds = RecordsBounds::author_key(ns, a, KeyFilter::Any); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), a.to_bytes(), Bytes::new())) - ); - assert_eq!(bounds.end_bound(), Bound::Unbounded); - - let a = AuthorId::from(&[0u8; 32]); - let mut a_end = a.to_bytes(); - a_end[31] = 1; - let bounds = RecordsBounds::author_key(ns, a, KeyFilter::Any); - assert_eq!( - bounds.end_bound(), - Bound::Excluded(&(ns.to_bytes(), a_end, Default::default())) - ); - - let bounds = RecordsBounds::author_key(ns, a, KeyFilter::Prefix(vec![1u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), a.to_bytes(), vec![1u8].into())) - ); - assert_eq!( - bounds.end_bound(), - Bound::Excluded(&(ns.to_bytes(), a.to_bytes(), vec![2u8].into())) - ); - - let bounds = RecordsBounds::author_key(ns, a, KeyFilter::Exact(vec![1u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), a.to_bytes(), vec![1u8].into())) - ); - assert_eq!( - bounds.end_bound(), - Bound::Included(&(ns.to_bytes(), a.to_bytes(), vec![1u8].into())) - ); - } - - #[test] - fn by_key_bounds() { - let ns = NamespaceId::from(&[255u8; 32]); - - let bounds = ByKeyBounds::namespace(ns); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), Bytes::new(), [0u8; 32])) - ); - assert_eq!(bounds.end_bound(), Bound::Unbounded); - - let bounds = ByKeyBounds::new(ns, &KeyFilter::Any); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), Bytes::new(), [0u8; 32])) - ); - assert_eq!(bounds.end_bound(), Bound::Unbounded); - - let bounds = ByKeyBounds::new(ns, &KeyFilter::Prefix(vec![1u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), vec![1u8].into(), [0u8; 32])) - ); - assert_eq!( - bounds.end_bound(), - Bound::Excluded(&(ns.to_bytes(), vec![2u8].into(), [0u8; 32])) - ); - - let bounds = ByKeyBounds::new(ns, &KeyFilter::Prefix(vec![255u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), vec![255u8].into(), [0u8; 32])) - ); - assert_eq!(bounds.end_bound(), Bound::Unbounded); - - let ns = NamespaceId::from(&[2u8; 32]); - let mut ns_end = ns.to_bytes(); - ns_end[31] = 3u8; - let bounds = ByKeyBounds::new(ns, &KeyFilter::Prefix(vec![255u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), vec![255u8].into(), [0u8; 32])) - ); - assert_eq!( - bounds.end_bound(), - Bound::Excluded(&(ns_end, Bytes::new(), [0u8; 32])) - ); - - let bounds = ByKeyBounds::new(ns, &KeyFilter::Exact(vec![1u8].into())); - assert_eq!( - bounds.start_bound(), - Bound::Included(&(ns.to_bytes(), vec![1u8].into(), [0u8; 32])) - ); - assert_eq!( - bounds.end_bound(), - Bound::Included(&(ns.to_bytes(), vec![1u8].into(), [255u8; 32])) - ); - } -} diff --git a/iroh-docs/src/store/fs/migrate_v1_v2.rs b/iroh-docs/src/store/fs/migrate_v1_v2.rs deleted file mode 100644 index b064ad025d7..00000000000 --- a/iroh-docs/src/store/fs/migrate_v1_v2.rs +++ /dev/null @@ -1,144 +0,0 @@ -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use redb::{MultimapTableHandle, TableHandle}; -use redb_v1::{ReadableMultimapTable, ReadableTable}; -use tempfile::NamedTempFile; -use tracing::info; - -macro_rules! migrate_table { - ($rtx:expr, $wtx:expr, $old:expr, $new:expr) => {{ - let old_table = $rtx.open_table($old)?; - let mut new_table = $wtx.open_table($new)?; - let name = $new.name(); - let len = old_table.len()?; - info!("migrate {name} ({len} rows).."); - let ind = (len as usize / 1000) + 1; - for (i, entry) in old_table.iter()?.enumerate() { - let (key, value) = entry?; - let key = key.value(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" {name} {i:>ind$}/{len}"); - } - new_table.insert(key, value)?; - } - info!("migrate {name} done"); - }}; -} - -macro_rules! migrate_multimap_table { - ($rtx:expr, $wtx:expr, $old:expr, $new:expr) => {{ - let old_table = $rtx.open_multimap_table($old)?; - let mut new_table = $wtx.open_multimap_table($new)?; - let name = $new.name(); - let len = old_table.len()?; - info!("migrate {name} ({len} rows)"); - let ind = (len as usize / 1000) + 1; - for (i, entry) in old_table.iter()?.enumerate() { - let (key, values) = entry?; - let key = key.value(); - if i > 0 && i % 1000 == 0 { - info!(" {name} {i:>ind$}/{len}"); - } - for value in values { - let value = value?; - new_table.insert(key, value.value())?; - } - } - info!("migrate {name} done"); - }}; -} - -pub fn run(source: impl AsRef) -> Result { - let source = source.as_ref(); - let dir = source.parent().expect("database is not in root"); - // create the new database in a tempfile in the same directory as the old db - let target = NamedTempFile::with_prefix_in("docs.db.migrate", dir)?; - let target = target.into_temp_path(); - info!("migrate {} to {}", source.display(), target.display()); - let old_db = redb_v1::Database::open(source)?; - let new_db = redb::Database::create(&target)?; - - let rtx = old_db.begin_read()?; - let wtx = new_db.begin_write()?; - - migrate_table!(rtx, wtx, old::AUTHORS_TABLE, new::tables::AUTHORS_TABLE); - migrate_table!( - rtx, - wtx, - old::NAMESPACES_TABLE, - new::tables::NAMESPACES_TABLE - ); - migrate_table!(rtx, wtx, old::RECORDS_TABLE, new::tables::RECORDS_TABLE); - migrate_table!( - rtx, - wtx, - old::LATEST_PER_AUTHOR_TABLE, - new::tables::LATEST_PER_AUTHOR_TABLE - ); - migrate_table!( - rtx, - wtx, - old::RECORDS_BY_KEY_TABLE, - new::tables::RECORDS_BY_KEY_TABLE - ); - migrate_multimap_table!( - rtx, - wtx, - old::NAMESPACE_PEERS_TABLE, - new::tables::NAMESPACE_PEERS_TABLE - ); - migrate_table!( - rtx, - wtx, - old::DOWNLOAD_POLICY_TABLE, - new::tables::DOWNLOAD_POLICY_TABLE - ); - - wtx.commit()?; - drop(rtx); - drop(old_db); - drop(new_db); - - let backup_path: PathBuf = { - let mut p = source.to_owned().into_os_string(); - p.push(".backup-redb-v1"); - p.into() - }; - info!("rename {} to {}", source.display(), backup_path.display()); - std::fs::rename(source, &backup_path)?; - info!("rename {} to {}", target.display(), source.display()); - target.persist_noclobber(source)?; - info!("opening migrated database from {}", source.display()); - let db = redb::Database::open(source)?; - Ok(db) -} - -mod new { - pub use super::super::*; -} - -mod old { - use redb_v1::{MultimapTableDefinition, TableDefinition}; - - use super::new::tables::{ - LatestPerAuthorKey, LatestPerAuthorValue, Nanos, RecordsByKeyId, RecordsId, RecordsValue, - }; - use crate::PeerIdBytes; - - pub const AUTHORS_TABLE: TableDefinition<&[u8; 32], &[u8; 32]> = - TableDefinition::new("authors-1"); - pub const NAMESPACES_TABLE: TableDefinition<&[u8; 32], (u8, &[u8; 32])> = - TableDefinition::new("namespaces-2"); - pub const RECORDS_TABLE: TableDefinition = - TableDefinition::new("records-1"); - pub const LATEST_PER_AUTHOR_TABLE: TableDefinition = - TableDefinition::new("latest-by-author-1"); - pub const RECORDS_BY_KEY_TABLE: TableDefinition = - TableDefinition::new("records-by-key-1"); - pub const NAMESPACE_PEERS_TABLE: MultimapTableDefinition<&[u8; 32], (Nanos, &PeerIdBytes)> = - MultimapTableDefinition::new("sync-peers-1"); - pub const DOWNLOAD_POLICY_TABLE: TableDefinition<&[u8; 32], &[u8]> = - TableDefinition::new("download-policy-1"); -} diff --git a/iroh-docs/src/store/fs/migrations.rs b/iroh-docs/src/store/fs/migrations.rs deleted file mode 100644 index e7cef40f0f4..00000000000 --- a/iroh-docs/src/store/fs/migrations.rs +++ /dev/null @@ -1,138 +0,0 @@ -use std::collections::HashMap; - -use anyhow::Result; -use redb::{Database, ReadableTable, ReadableTableMetadata, TableHandle, WriteTransaction}; -use tracing::{debug, info}; - -use super::tables::{ - LATEST_PER_AUTHOR_TABLE, NAMESPACES_TABLE, NAMESPACES_TABLE_V1, RECORDS_BY_KEY_TABLE, - RECORDS_TABLE, -}; -use crate::{Capability, NamespaceSecret}; - -/// Run all database migrations, if needed. -pub fn run_migrations(db: &Database) -> Result<()> { - run_migration(db, migration_001_populate_latest_table)?; - run_migration(db, migration_002_namespaces_populate_v2)?; - run_migration(db, migration_003_namespaces_delete_v1)?; - run_migration(db, migration_004_populate_by_key_index)?; - Ok(()) -} - -fn run_migration(db: &Database, f: F) -> Result<()> -where - F: Fn(&WriteTransaction) -> Result, -{ - let name = std::any::type_name::(); - let name = name.split("::").last().unwrap(); - let tx = db.begin_write()?; - debug!("Start migration {name}"); - match f(&tx)? { - MigrateOutcome::Execute(len) => { - tx.commit()?; - info!("Executed migration {name} ({len} rows affected)"); - } - MigrateOutcome::Skip => debug!("Skip migration {name}: Not needed"), - } - Ok(()) -} - -enum MigrateOutcome { - Skip, - Execute(usize), -} - -/// migration 001: populate the latest table (which did not exist before) -fn migration_001_populate_latest_table(tx: &WriteTransaction) -> Result { - let mut latest_table = tx.open_table(LATEST_PER_AUTHOR_TABLE)?; - let records_table = tx.open_table(RECORDS_TABLE)?; - if !latest_table.is_empty()? || records_table.is_empty()? { - return Ok(MigrateOutcome::Skip); - } - - #[allow(clippy::type_complexity)] - let mut heads: HashMap<([u8; 32], [u8; 32]), (u64, Vec)> = HashMap::new(); - let iter = records_table.iter()?; - - for next in iter { - let next = next?; - let (namespace, author, key) = next.0.value(); - let (timestamp, _namespace_sig, _author_sig, _len, _hash) = next.1.value(); - heads - .entry((*namespace, *author)) - .and_modify(|e| { - if timestamp >= e.0 { - *e = (timestamp, key.to_vec()); - } - }) - .or_insert_with(|| (timestamp, key.to_vec())); - } - let len = heads.len(); - for ((namespace, author), (timestamp, key)) in heads { - latest_table.insert((&namespace, &author), (timestamp, key.as_slice()))?; - } - Ok(MigrateOutcome::Execute(len)) -} - -/// Copy the namespaces data from V1 to V2. -fn migration_002_namespaces_populate_v2(tx: &WriteTransaction) -> Result { - let namespaces_v1_exists = tx - .list_tables()? - .any(|handle| handle.name() == NAMESPACES_TABLE_V1.name()); - if !namespaces_v1_exists { - return Ok(MigrateOutcome::Skip); - } - let namespaces_v1 = tx.open_table(NAMESPACES_TABLE_V1)?; - let mut namespaces_v2 = tx.open_table(NAMESPACES_TABLE)?; - let mut entries = 0; - for res in namespaces_v1.iter()? { - let db_value = res?.1; - let secret_bytes = db_value.value(); - let capability = Capability::Write(NamespaceSecret::from_bytes(secret_bytes)); - let id = capability.id().to_bytes(); - let (raw_kind, raw_bytes) = capability.raw(); - namespaces_v2.insert(&id, (raw_kind, &raw_bytes))?; - entries += 1; - } - Ok(MigrateOutcome::Execute(entries)) -} - -/// Delete the v1 namespaces table. -/// -/// This should be part of [`migration_002_namespaces_populate_v2`] but due to a limitation in -/// [`redb`] up to v1.3.0 a table cannot be deleted in a transaction that also opens this table. -/// Therefore the table deletion has to be in a separate transaction. -/// -/// This limitation was removed in so this can be merged -/// back into [`migration_002_namespaces_populate_v2`] once we upgrade to the next redb version -/// after 1.3. -fn migration_003_namespaces_delete_v1(tx: &WriteTransaction) -> Result { - let namespaces_v1_exists = tx - .list_tables()? - .any(|handle| handle.name() == NAMESPACES_TABLE_V1.name()); - if !namespaces_v1_exists { - return Ok(MigrateOutcome::Skip); - } - tx.delete_table(NAMESPACES_TABLE_V1)?; - Ok(MigrateOutcome::Execute(1)) -} - -/// migration 004: populate the by_key index table(which did not exist before) -fn migration_004_populate_by_key_index(tx: &WriteTransaction) -> Result { - let mut by_key_table = tx.open_table(RECORDS_BY_KEY_TABLE)?; - let records_table = tx.open_table(RECORDS_TABLE)?; - if !by_key_table.is_empty()? { - return Ok(MigrateOutcome::Skip); - } - - let iter = records_table.iter()?; - let mut len = 0; - for next in iter { - let next = next?; - let (namespace, author, key) = next.0.value(); - let id = (namespace, key, author); - by_key_table.insert(id, ())?; - len += 1; - } - Ok(MigrateOutcome::Execute(len)) -} diff --git a/iroh-docs/src/store/fs/query.rs b/iroh-docs/src/store/fs/query.rs deleted file mode 100644 index 72113f2e8d6..00000000000 --- a/iroh-docs/src/store/fs/query.rs +++ /dev/null @@ -1,159 +0,0 @@ -use anyhow::Result; -use iroh_base::hash::Hash; - -use super::{ - bounds::{ByKeyBounds, RecordsBounds}, - ranges::{RecordsByKeyRange, RecordsRange}, - RecordsValue, -}; -use crate::{ - store::{ - fs::tables::ReadOnlyTables, - util::{IndexKind, LatestPerKeySelector, SelectorRes}, - AuthorFilter, KeyFilter, Query, - }, - AuthorId, NamespaceId, SignedEntry, -}; - -/// A query iterator for entry queries. -#[derive(Debug)] -pub struct QueryIterator { - range: QueryRange, - query: Query, - offset: u64, - count: u64, -} - -#[derive(Debug)] -enum QueryRange { - AuthorKey { - range: RecordsRange<'static>, - key_filter: KeyFilter, - }, - KeyAuthor { - range: RecordsByKeyRange, - author_filter: AuthorFilter, - selector: Option, - }, -} - -impl QueryIterator { - pub fn new(tables: ReadOnlyTables, namespace: NamespaceId, query: Query) -> Result { - let index_kind = IndexKind::from(&query); - let range = match index_kind { - IndexKind::AuthorKey { range, key_filter } => { - let (bounds, filter) = match range { - // single author: both author and key are selected via the range. therefore - // set `filter` to `Any`. - AuthorFilter::Exact(author) => ( - RecordsBounds::author_key(namespace, author, key_filter), - KeyFilter::Any, - ), - // no author set => full table scan with the provided key filter - AuthorFilter::Any => (RecordsBounds::namespace(namespace), key_filter), - }; - let range = RecordsRange::with_bounds_static(&tables.records, bounds)?; - QueryRange::AuthorKey { - range, - key_filter: filter, - } - } - IndexKind::KeyAuthor { - range, - author_filter, - latest_per_key, - } => { - let bounds = ByKeyBounds::new(namespace, &range); - let range = - RecordsByKeyRange::with_bounds(tables.records_by_key, tables.records, bounds)?; - let selector = latest_per_key.then(LatestPerKeySelector::default); - QueryRange::KeyAuthor { - author_filter, - range, - selector, - } - } - }; - - Ok(Self { - range, - query, - offset: 0, - count: 0, - }) - } -} - -impl Iterator for QueryIterator { - type Item = Result; - - fn next(&mut self) -> Option> { - // early-return if we reached the query limit. - if let Some(limit) = self.query.limit() { - if self.count >= limit { - return None; - } - } - loop { - let next = match &mut self.range { - QueryRange::AuthorKey { range, key_filter } => { - // get the next entry from the query range, filtered by the key and empty filters - range.next_filtered(&self.query.sort_direction, |(_ns, _author, key), value| { - key_filter.matches(key) - && (self.query.include_empty || !value_is_empty(&value)) - }) - } - - QueryRange::KeyAuthor { - range, - author_filter, - selector, - } => loop { - // get the next entry from the query range, filtered by the author filter - let next = range - .next_filtered(&self.query.sort_direction, |(_ns, _key, author)| { - author_filter.matches(&(AuthorId::from(author))) - }); - - // early-break if next contains Err - let next = match next.transpose() { - Err(err) => break Some(Err(err)), - Ok(next) => next, - }; - - // push the entry into the selector. if active, only the latest entry - // for each key will be emitted. - let next = match selector { - None => next, - Some(selector) => match selector.push(next) { - SelectorRes::Continue => continue, - SelectorRes::Finished => None, - SelectorRes::Some(res) => Some(res), - }, - }; - - // skip the entry if empty and no empty entries requested - if !self.query.include_empty && matches!(&next, Some(e) if e.is_empty()) { - continue; - } - - break next.map(Result::Ok); - }, - }; - - // skip the entry if we didn't get past the requested offset yet. - if self.offset < self.query.offset() && matches!(next, Some(Ok(_))) { - self.offset += 1; - continue; - } - - self.count += 1; - return next; - } - } -} - -fn value_is_empty(value: &RecordsValue) -> bool { - let (_timestamp, _namespace_sig, _author_sig, _len, hash) = value; - *hash == Hash::EMPTY.as_bytes() -} diff --git a/iroh-docs/src/store/fs/ranges.rs b/iroh-docs/src/store/fs/ranges.rs deleted file mode 100644 index 34aaa80f3c7..00000000000 --- a/iroh-docs/src/store/fs/ranges.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Ranges and helpers for working with [`redb`] tables - -use redb::{Key, Range, ReadOnlyTable, ReadableTable, Value}; - -use super::{ - bounds::{ByKeyBounds, RecordsBounds}, - into_entry, - tables::{RecordsByKeyId, RecordsId, RecordsValue}, -}; -use crate::{store::SortDirection, SignedEntry}; - -/// An extension trait for [`Range`] that provides methods for mapped retrieval. -pub trait RangeExt { - /// Get the next entry and map the item with a callback function. - fn next_map( - &mut self, - map: impl for<'x> Fn(K::SelfType<'x>, V::SelfType<'x>) -> T, - ) -> Option>; - - /// Get the next entry, but only if the callback function returns Some, otherwise continue. - /// - /// With `direction` the range can be either process in forward or backward direction. - fn next_filter_map( - &mut self, - direction: &SortDirection, - filter_map: impl for<'x> Fn(K::SelfType<'x>, V::SelfType<'x>) -> Option, - ) -> Option>; - - /// Like [`Self::next_filter_map`], but the callback returns a `Result`, and the result is - /// flattened with the result from the range operation. - fn next_try_filter_map( - &mut self, - direction: &SortDirection, - filter_map: impl for<'x> Fn(K::SelfType<'x>, V::SelfType<'x>) -> Option>, - ) -> Option> { - Some(self.next_filter_map(direction, filter_map)?.and_then(|r| r)) - } -} - -impl<'a, K: Key + 'static, V: Value + 'static> RangeExt for Range<'a, K, V> { - fn next_map( - &mut self, - map: impl for<'x> Fn(K::SelfType<'x>, V::SelfType<'x>) -> T, - ) -> Option> { - self.next() - .map(|r| r.map_err(Into::into).map(|r| map(r.0.value(), r.1.value()))) - } - - fn next_filter_map( - &mut self, - direction: &SortDirection, - filter_map: impl for<'x> Fn(K::SelfType<'x>, V::SelfType<'x>) -> Option, - ) -> Option> { - loop { - let next = match direction { - SortDirection::Asc => self.next(), - SortDirection::Desc => self.next_back(), - }; - match next { - None => break None, - Some(Err(err)) => break Some(Err(err.into())), - Some(Ok(res)) => match filter_map(res.0.value(), res.1.value()) { - None => continue, - Some(item) => break Some(Ok(item)), - }, - } - } - } -} - -/// An iterator over a range of entries from the records table. -#[derive(derive_more::Debug)] -#[debug("RecordsRange")] -pub struct RecordsRange<'a>(Range<'a, RecordsId<'static>, RecordsValue<'static>>); - -// pub type RecordsRange<'a> = Range<'a, RecordsId<'static>, RecordsValue<'static>>; - -impl<'a> RecordsRange<'a> { - pub(super) fn with_bounds( - records: &'a impl ReadableTable, RecordsValue<'static>>, - bounds: RecordsBounds, - ) -> anyhow::Result { - let range = records.range(bounds.as_ref())?; - Ok(Self(range)) - } - - // - /// Get the next item in the range. - /// - /// Omit items for which the `matcher` function returns false. - pub(super) fn next_filtered( - &mut self, - direction: &SortDirection, - filter: impl for<'x> Fn(RecordsId<'x>, RecordsValue<'x>) -> bool, - ) -> Option> { - self.0 - .next_filter_map(direction, |k, v| filter(k, v).then(|| into_entry(k, v))) - } -} - -impl RecordsRange<'static> { - pub(super) fn all_static( - records: &ReadOnlyTable, RecordsValue<'static>>, - ) -> anyhow::Result { - let range = records.range::>(..)?; - Ok(Self(range)) - } - pub(super) fn with_bounds_static( - records: &ReadOnlyTable, RecordsValue<'static>>, - bounds: RecordsBounds, - ) -> anyhow::Result { - let range = records.range(bounds.as_ref())?; - Ok(Self(range)) - } -} - -impl<'a> Iterator for RecordsRange<'a> { - type Item = anyhow::Result; - fn next(&mut self) -> Option { - self.0.next_map(into_entry) - } -} - -#[derive(derive_more::Debug)] -#[debug("RecordsByKeyRange")] -pub struct RecordsByKeyRange { - records_table: ReadOnlyTable, RecordsValue<'static>>, - by_key_range: Range<'static, RecordsByKeyId<'static>, ()>, -} - -impl RecordsByKeyRange { - pub fn with_bounds( - records_by_key_table: ReadOnlyTable, ()>, - records_table: ReadOnlyTable, RecordsValue<'static>>, - bounds: ByKeyBounds, - ) -> anyhow::Result { - let by_key_range = records_by_key_table.range(bounds.as_ref())?; - Ok(Self { - records_table, - by_key_range, - }) - } - - /// Get the next item in the range. - /// - /// Omit items for which the `filter` function returns false. - pub fn next_filtered( - &mut self, - direction: &SortDirection, - filter: impl for<'x> Fn(RecordsByKeyId<'x>) -> bool, - ) -> Option> { - let entry = self.by_key_range.next_try_filter_map(direction, |k, _v| { - if !filter(k) { - return None; - }; - let (namespace, key, author) = k; - let records_id = (namespace, author, key); - let entry = self.records_table.get(&records_id).transpose()?; - let entry = entry - .map(|value| into_entry(records_id, value.value())) - .map_err(anyhow::Error::from); - Some(entry) - }); - entry - } -} diff --git a/iroh-docs/src/store/fs/tables.rs b/iroh-docs/src/store/fs/tables.rs deleted file mode 100644 index 898fffca4d1..00000000000 --- a/iroh-docs/src/store/fs/tables.rs +++ /dev/null @@ -1,186 +0,0 @@ -#![allow(missing_docs)] -// Table Definitions - -use std::time::Instant; - -use bytes::Bytes; -use redb::{ - MultimapTable, MultimapTableDefinition, ReadOnlyMultimapTable, ReadOnlyTable, ReadTransaction, - Table, TableDefinition, WriteTransaction, -}; - -use crate::PeerIdBytes; - -/// Table: Authors -/// Key: `[u8; 32]` # AuthorId -/// Value: `[u8; 32]` # Author -pub const AUTHORS_TABLE: TableDefinition<&[u8; 32], &[u8; 32]> = TableDefinition::new("authors-1"); - -/// Table: Namespaces v1 (replaced by Namespaces v2 in migration ) -/// Key: `[u8; 32]` # NamespaceId -/// Value: `[u8; 32]` # NamespaceSecret -pub const NAMESPACES_TABLE_V1: TableDefinition<&[u8; 32], &[u8; 32]> = - TableDefinition::new("namespaces-1"); - -/// Table: Namespaces v2 -/// Key: `[u8; 32]` # NamespaceId -/// Value: `(u8, [u8; 32])` # (CapabilityKind, Capability) -pub const NAMESPACES_TABLE: TableDefinition<&[u8; 32], (u8, &[u8; 32])> = - TableDefinition::new("namespaces-2"); - -/// Table: Records -/// Key: `([u8; 32], [u8; 32], &[u8])` -/// # (NamespaceId, AuthorId, Key) -/// Value: `(u64, [u8; 32], [u8; 32], u64, [u8; 32])` -/// # (timestamp, signature_namespace, signature_author, len, hash) -pub const RECORDS_TABLE: TableDefinition = - TableDefinition::new("records-1"); -pub type RecordsId<'a> = (&'a [u8; 32], &'a [u8; 32], &'a [u8]); -pub type RecordsIdOwned = ([u8; 32], [u8; 32], Bytes); -pub type RecordsValue<'a> = (u64, &'a [u8; 64], &'a [u8; 64], u64, &'a [u8; 32]); -pub type RecordsTable = ReadOnlyTable, RecordsValue<'static>>; - -/// Table: Latest per author -/// Key: `([u8; 32], [u8; 32])` # (NamespaceId, AuthorId) -/// Value: `(u64, Vec)` # (Timestamp, Key) -pub const LATEST_PER_AUTHOR_TABLE: TableDefinition = - TableDefinition::new("latest-by-author-1"); -pub type LatestPerAuthorKey<'a> = (&'a [u8; 32], &'a [u8; 32]); -pub type LatestPerAuthorValue<'a> = (u64, &'a [u8]); - -/// Table: Records by key -/// Key: `([u8; 32], Vec, [u8; 32]])` # (NamespaceId, Key, AuthorId) -/// Value: `()` -pub const RECORDS_BY_KEY_TABLE: TableDefinition = - TableDefinition::new("records-by-key-1"); -pub type RecordsByKeyId<'a> = (&'a [u8; 32], &'a [u8], &'a [u8; 32]); -pub type RecordsByKeyIdOwned = ([u8; 32], Bytes, [u8; 32]); - -/// Table: Peers per document. -/// Key: `[u8; 32]` # NamespaceId -/// Value: `(u64, [u8; 32])` # ([`Nanos`], &[`PeerIdBytes`]) representing the last time a peer was used. -pub const NAMESPACE_PEERS_TABLE: MultimapTableDefinition<&[u8; 32], (Nanos, &PeerIdBytes)> = - MultimapTableDefinition::new("sync-peers-1"); -/// Number of seconds elapsed since [`std::time::SystemTime::UNIX_EPOCH`]. Used to register the -/// last time a peer was useful in a document. -// NOTE: resolution is nanoseconds, stored as a u64 since this covers ~500years from unix epoch, -// which should be more than enough -pub type Nanos = u64; - -/// Table: Download policy -/// Key: `[u8; 32]` # NamespaceId -/// Value: `Vec` # Postcard encoded download policy -pub const DOWNLOAD_POLICY_TABLE: TableDefinition<&[u8; 32], &[u8]> = - TableDefinition::new("download-policy-1"); - -self_cell::self_cell! { - struct TransactionAndTablesInner { - owner: WriteTransaction, - #[covariant] - dependent: Tables, - } -} - -#[derive(derive_more::Debug)] -pub struct TransactionAndTables { - #[debug("TransactionAndTablesInner")] - inner: TransactionAndTablesInner, - pub(crate) since: Instant, -} - -impl TransactionAndTables { - pub fn new(tx: WriteTransaction) -> std::result::Result { - Ok(Self { - inner: TransactionAndTablesInner::try_new(tx, |tx| Tables::new(tx))?, - since: Instant::now(), - }) - } - - pub fn tables(&self) -> &Tables { - self.inner.borrow_dependent() - } - - pub fn with_tables_mut( - &mut self, - f: impl FnOnce(&mut Tables) -> anyhow::Result, - ) -> anyhow::Result { - self.inner.with_dependent_mut(|_, t| f(t)) - } - - pub fn commit(self) -> std::result::Result<(), redb::CommitError> { - self.inner.into_owner().commit() - } -} - -#[derive(derive_more::Debug)] -pub struct Tables<'tx> { - pub records: Table<'tx, RecordsId<'static>, RecordsValue<'static>>, - pub records_by_key: Table<'tx, RecordsByKeyId<'static>, ()>, - pub namespaces: Table<'tx, &'static [u8; 32], (u8, &'static [u8; 32])>, - pub latest_per_author: Table<'tx, LatestPerAuthorKey<'static>, LatestPerAuthorValue<'static>>, - #[debug("MultimapTable")] - pub namespace_peers: MultimapTable<'tx, &'static [u8; 32], (Nanos, &'static PeerIdBytes)>, - pub download_policy: Table<'tx, &'static [u8; 32], &'static [u8]>, - pub authors: Table<'tx, &'static [u8; 32], &'static [u8; 32]>, -} - -impl<'tx> Tables<'tx> { - pub fn new(tx: &'tx WriteTransaction) -> Result { - let records = tx.open_table(RECORDS_TABLE)?; - let records_by_key = tx.open_table(RECORDS_BY_KEY_TABLE)?; - let namespaces = tx.open_table(NAMESPACES_TABLE)?; - let latest_per_author = tx.open_table(LATEST_PER_AUTHOR_TABLE)?; - let namespace_peers = tx.open_multimap_table(NAMESPACE_PEERS_TABLE)?; - let download_policy = tx.open_table(DOWNLOAD_POLICY_TABLE)?; - let authors = tx.open_table(AUTHORS_TABLE)?; - Ok(Self { - records, - records_by_key, - namespaces, - latest_per_author, - namespace_peers, - download_policy, - authors, - }) - } -} -#[derive(derive_more::Debug)] -pub struct ReadOnlyTables { - pub records: ReadOnlyTable, RecordsValue<'static>>, - pub records_by_key: ReadOnlyTable, ()>, - pub namespaces: ReadOnlyTable<&'static [u8; 32], (u8, &'static [u8; 32])>, - pub latest_per_author: - ReadOnlyTable, LatestPerAuthorValue<'static>>, - #[debug("namespace_peers")] - pub namespace_peers: ReadOnlyMultimapTable<&'static [u8; 32], (Nanos, &'static PeerIdBytes)>, - pub download_policy: ReadOnlyTable<&'static [u8; 32], &'static [u8]>, - pub authors: ReadOnlyTable<&'static [u8; 32], &'static [u8; 32]>, - tx: ReadTransaction, -} - -impl ReadOnlyTables { - pub fn new(tx: ReadTransaction) -> Result { - let records = tx.open_table(RECORDS_TABLE)?; - let records_by_key = tx.open_table(RECORDS_BY_KEY_TABLE)?; - let namespaces = tx.open_table(NAMESPACES_TABLE)?; - let latest_per_author = tx.open_table(LATEST_PER_AUTHOR_TABLE)?; - let namespace_peers = tx.open_multimap_table(NAMESPACE_PEERS_TABLE)?; - let download_policy = tx.open_table(DOWNLOAD_POLICY_TABLE)?; - let authors = tx.open_table(AUTHORS_TABLE)?; - Ok(Self { - records, - records_by_key, - namespaces, - latest_per_author, - namespace_peers, - download_policy, - authors, - tx, - }) - } - - /// Create a clone of the records table for use in iterators. - pub fn records_clone(&self) -> Result { - self.tx.open_table(RECORDS_TABLE) - } -} diff --git a/iroh-docs/src/store/pubkeys.rs b/iroh-docs/src/store/pubkeys.rs deleted file mode 100644 index 77209348dfd..00000000000 --- a/iroh-docs/src/store/pubkeys.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use ed25519_dalek::{SignatureError, VerifyingKey}; - -use crate::{AuthorId, AuthorPublicKey, NamespaceId, NamespacePublicKey}; - -/// Store trait for expanded public keys for authors and namespaces. -/// -/// Used to cache [`ed25519_dalek::VerifyingKey`]. -/// -/// This trait is implemented for the unit type [`()`], where no caching is used. -pub trait PublicKeyStore { - /// Convert a byte array into a [`VerifyingKey`]. - /// - /// New keys are inserted into the [`PublicKeyStore ] and reused on subsequent calls. - fn public_key(&self, id: &[u8; 32]) -> Result; - - /// Convert a [`NamespaceId`] into a [`NamespacePublicKey`]. - /// - /// New keys are inserted into the [`PublicKeyStore ] and reused on subsequent calls. - fn namespace_key(&self, bytes: &NamespaceId) -> Result { - self.public_key(bytes.as_bytes()).map(Into::into) - } - - /// Convert a [`AuthorId`] into a [`AuthorPublicKey`]. - /// - /// New keys are inserted into the [`PublicKeyStore ] and reused on subsequent calls. - fn author_key(&self, bytes: &AuthorId) -> Result { - self.public_key(bytes.as_bytes()).map(Into::into) - } -} - -impl PublicKeyStore for &T { - fn public_key(&self, id: &[u8; 32]) -> Result { - (*self).public_key(id) - } -} - -impl PublicKeyStore for &mut T { - fn public_key(&self, id: &[u8; 32]) -> Result { - PublicKeyStore::public_key(*self, id) - } -} - -impl PublicKeyStore for () { - fn public_key(&self, id: &[u8; 32]) -> Result { - VerifyingKey::from_bytes(id) - } -} - -/// In-memory key storage -// TODO: Make max number of keys stored configurable. -#[derive(Debug, Clone, Default)] -pub struct MemPublicKeyStore { - keys: Arc>>, -} - -impl PublicKeyStore for MemPublicKeyStore { - fn public_key(&self, bytes: &[u8; 32]) -> Result { - if let Some(id) = self.keys.read().unwrap().get(bytes) { - return Ok(*id); - } - let id = VerifyingKey::from_bytes(bytes)?; - self.keys.write().unwrap().insert(*bytes, id); - Ok(id) - } -} diff --git a/iroh-docs/src/store/util.rs b/iroh-docs/src/store/util.rs deleted file mode 100644 index 04cad47e5d0..00000000000 --- a/iroh-docs/src/store/util.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Utilities useful across different store impls. - -use super::{AuthorFilter, KeyFilter, Query, QueryKind, SortBy}; -use crate::SignedEntry; - -/// A helper for stores that have by-author and by-key indexes for records. -#[derive(Debug)] -pub enum IndexKind { - AuthorKey { - range: AuthorFilter, - key_filter: KeyFilter, - }, - KeyAuthor { - range: KeyFilter, - author_filter: AuthorFilter, - latest_per_key: bool, - }, -} - -impl From<&Query> for IndexKind { - fn from(query: &Query) -> Self { - match &query.kind { - QueryKind::Flat(details) => match (&query.filter_author, details.sort_by) { - (AuthorFilter::Any, SortBy::KeyAuthor) => IndexKind::KeyAuthor { - range: query.filter_key.clone(), - author_filter: AuthorFilter::Any, - latest_per_key: false, - }, - _ => IndexKind::AuthorKey { - range: query.filter_author.clone(), - key_filter: query.filter_key.clone(), - }, - }, - QueryKind::SingleLatestPerKey(_) => IndexKind::KeyAuthor { - range: query.filter_key.clone(), - author_filter: query.filter_author.clone(), - latest_per_key: true, - }, - } - } -} - -/// Helper to extract the latest entry per key from an iterator that yields [`SignedEntry`] items. -/// -/// Items must be pushed in key-sorted order. -#[derive(Debug, Default)] -pub struct LatestPerKeySelector(Option); - -pub enum SelectorRes { - /// The iterator is finished. - Finished, - /// The selection is not yet finished, keep pushing more items. - Continue, - /// The selection yielded an entry. - Some(SignedEntry), -} - -impl LatestPerKeySelector { - /// Push an entry into the selector. - /// - /// Entries must be sorted by key beforehand. - pub fn push(&mut self, entry: Option) -> SelectorRes { - let Some(entry) = entry else { - return match self.0.take() { - Some(entry) => SelectorRes::Some(entry), - None => SelectorRes::Finished, - }; - }; - match self.0.take() { - None => { - self.0 = Some(entry); - SelectorRes::Continue - } - Some(last) if last.key() == entry.key() => { - if entry.timestamp() > last.timestamp() { - self.0 = Some(entry); - } else { - self.0 = Some(last); - } - SelectorRes::Continue - } - Some(last) => { - self.0 = Some(entry); - SelectorRes::Some(last) - } - } - } -} diff --git a/iroh-docs/src/sync.rs b/iroh-docs/src/sync.rs deleted file mode 100644 index c11c6d3aba4..00000000000 --- a/iroh-docs/src/sync.rs +++ /dev/null @@ -1,2533 +0,0 @@ -//! API for iroh-docs replicas - -// Names and concepts are roughly based on Willows design at the moment: -// -// https://hackmd.io/DTtck8QOQm6tZaQBBtTf7w -// -// This is going to change! - -use std::{ - cmp::Ordering, - fmt::Debug, - ops::{Deref, DerefMut}, - sync::Arc, - time::{Duration, SystemTime}, -}; - -use bytes::{Bytes, BytesMut}; -use ed25519_dalek::{Signature, SignatureError}; -use iroh_base::{base32, hash::Hash}; -#[cfg(feature = "metrics")] -use iroh_metrics::{inc, inc_by}; -use serde::{Deserialize, Serialize}; - -pub use crate::heads::AuthorHeads; -#[cfg(feature = "metrics")] -use crate::metrics::Metrics; -use crate::{ - keys::{Author, AuthorId, AuthorPublicKey, NamespaceId, NamespacePublicKey, NamespaceSecret}, - ranger::{self, Fingerprint, InsertOutcome, RangeEntry, RangeKey, RangeValue, Store}, - store::{self, fs::StoreInstance, DownloadPolicyStore, PublicKeyStore}, -}; - -/// Protocol message for the set reconciliation protocol. -/// -/// Can be serialized to bytes with [serde] to transfer between peers. -pub type ProtocolMessage = crate::ranger::Message; - -/// Byte representation of a `PeerId` from `iroh-net`. -// TODO: PeerId is in iroh-net which iroh-docs doesn't depend on. Add iroh-base crate with `PeerId`. -pub type PeerIdBytes = [u8; 32]; - -/// Max time in the future from our wall clock time that we accept entries for. -/// Value is 10 minutes. -pub const MAX_TIMESTAMP_FUTURE_SHIFT: u64 = 10 * 60 * Duration::from_secs(1).as_millis() as u64; - -/// Callback that may be set on a replica to determine the availability status for a content hash. -pub type ContentStatusCallback = Arc ContentStatus + Send + Sync + 'static>; - -/// Event emitted by sync when entries are added. -#[derive(Debug, Clone)] -pub enum Event { - /// A local entry has been added. - LocalInsert { - /// Document in which the entry was inserted. - namespace: NamespaceId, - /// Inserted entry. - entry: SignedEntry, - }, - /// A remote entry has been added. - RemoteInsert { - /// Document in which the entry was inserted. - namespace: NamespaceId, - /// Inserted entry. - entry: SignedEntry, - /// Peer that provided the inserted entry. - from: PeerIdBytes, - /// Whether download policies require the content to be downloaded. - should_download: bool, - /// [`ContentStatus`] for this entry in the remote's replica. - remote_content_status: ContentStatus, - }, -} - -/// Whether an entry was inserted locally or by a remote peer. -#[derive(Debug, Clone)] -pub enum InsertOrigin { - /// The entry was inserted locally. - Local, - /// The entry was received from the remote node identified by [`PeerIdBytes`]. - Sync { - /// The peer from which we received this entry. - from: PeerIdBytes, - /// Whether the peer claims to have the content blob for this entry. - remote_content_status: ContentStatus, - }, -} - -/// Whether the content status is available on a node. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] -pub enum ContentStatus { - /// The content is completely available. - Complete, - /// The content is partially available. - Incomplete, - /// The content is missing. - Missing, -} - -/// Outcome of a sync operation. -#[derive(Debug, Clone, Default)] -pub struct SyncOutcome { - /// Timestamp of the latest entry for each author in the set we received. - pub heads_received: AuthorHeads, - /// Number of entries we received. - pub num_recv: usize, - /// Number of entries we sent. - pub num_sent: usize, -} - -fn get_as_ptr(value: &T) -> Option { - use std::mem; - if mem::size_of::() == std::mem::size_of::() - && mem::align_of::() == mem::align_of::() - { - // Safe only if size and alignment requirements are met - unsafe { Some(mem::transmute_copy(value)) } - } else { - None - } -} - -fn same_channel(a: &async_channel::Sender, b: &async_channel::Sender) -> bool { - get_as_ptr(a).unwrap() == get_as_ptr(b).unwrap() -} - -#[derive(Debug, Default)] -struct Subscribers(Vec>); -impl Subscribers { - pub fn subscribe(&mut self, sender: async_channel::Sender) { - self.0.push(sender) - } - pub fn unsubscribe(&mut self, sender: &async_channel::Sender) { - self.0.retain(|s| !same_channel(s, sender)); - } - pub fn send(&mut self, event: Event) { - self.0 - .retain(|sender| sender.send_blocking(event.clone()).is_ok()) - } - pub fn len(&self) -> usize { - self.0.len() - } - pub fn send_with(&mut self, f: impl FnOnce() -> Event) { - if !self.0.is_empty() { - self.send(f()) - } - } -} - -/// Kind of capability of the namespace. -#[derive( - Debug, - Clone, - Copy, - Serialize, - Deserialize, - num_enum::IntoPrimitive, - num_enum::TryFromPrimitive, - strum::Display, -)] -#[repr(u8)] -#[strum(serialize_all = "snake_case")] -pub enum CapabilityKind { - /// A writable replica. - Write = 1, - /// A readable replica. - Read = 2, -} - -/// The capability of the namespace. -#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From)] -pub enum Capability { - /// Write access to the namespace. - Write(NamespaceSecret), - /// Read only access to the namespace. - Read(NamespaceId), -} - -impl Capability { - /// Get the [`NamespaceId`] for this [`Capability`]. - pub fn id(&self) -> NamespaceId { - match self { - Capability::Write(secret) => secret.id(), - Capability::Read(id) => *id, - } - } - - /// Get the [`NamespaceSecret`] of this [`Capability`]. - /// Will fail if the [`Capability`] is read only. - pub fn secret_key(&self) -> Result<&NamespaceSecret, ReadOnly> { - match self { - Capability::Write(secret) => Ok(secret), - Capability::Read(_) => Err(ReadOnly), - } - } - - /// Get the kind of capability. - pub fn kind(&self) -> CapabilityKind { - match self { - Capability::Write(_) => CapabilityKind::Write, - Capability::Read(_) => CapabilityKind::Read, - } - } - - /// Get the raw representation of this namespace capability. - pub fn raw(&self) -> (u8, [u8; 32]) { - let capability_repr: u8 = self.kind().into(); - let bytes = match self { - Capability::Write(secret) => secret.to_bytes(), - Capability::Read(id) => id.to_bytes(), - }; - (capability_repr, bytes) - } - - /// Create a [`Capability`] from its raw representation. - pub fn from_raw(kind: u8, bytes: &[u8; 32]) -> anyhow::Result { - let kind: CapabilityKind = kind.try_into()?; - let capability = match kind { - CapabilityKind::Write => { - let secret = NamespaceSecret::from_bytes(bytes); - Capability::Write(secret) - } - CapabilityKind::Read => { - let id = NamespaceId::from(bytes); - Capability::Read(id) - } - }; - Ok(capability) - } - - /// Merge this capability with another capability. - /// - /// Will return an error if `other` is not a capability for the same namespace. - /// - /// Returns `true` if the capability was changed, `false` otherwise. - pub fn merge(&mut self, other: Capability) -> Result { - if other.id() != self.id() { - return Err(CapabilityError::NamespaceMismatch); - } - - // the only capability upgrade is from read-only (self) to writable (other) - if matches!(self, Capability::Read(_)) && matches!(other, Capability::Write(_)) { - let _ = std::mem::replace(self, other); - Ok(true) - } else { - Ok(false) - } - } -} - -/// Errors for capability operations -#[derive(Debug, thiserror::Error)] -pub enum CapabilityError { - /// Namespaces are not the same - #[error("Namespaces are not the same")] - NamespaceMismatch, -} - -/// In memory information about an open replica. -#[derive(derive_more::Debug)] -pub struct ReplicaInfo { - pub(crate) capability: Capability, - subscribers: Subscribers, - #[debug("ContentStatusCallback")] - content_status_cb: Option, - closed: bool, -} - -impl ReplicaInfo { - /// Create a new replica. - pub fn new(capability: Capability) -> Self { - Self { - capability, - subscribers: Default::default(), - // on_insert_sender: RwLock::new(None), - content_status_cb: None, - closed: false, - } - } - - /// Subscribe to insert events. - /// - /// When subscribing to a replica, you must ensure that the corresponding [`async_channel::Receiver`] is - /// received from in a loop. If not receiving, local and remote inserts will hang waiting for - /// the receiver to be received from. - pub fn subscribe(&mut self, sender: async_channel::Sender) { - self.subscribers.subscribe(sender) - } - - /// Explicitly unsubscribe a sender. - /// - /// Simply dropping the receiver is fine too. If you cloned a single sender to subscribe to - /// multiple replicas, you can use this method to explicitly unsubscribe the sender from - /// this replica without having to drop the receiver. - pub fn unsubscribe(&mut self, sender: &async_channel::Sender) { - self.subscribers.unsubscribe(sender) - } - - /// Get the number of current event subscribers. - pub fn subscribers_count(&self) -> usize { - self.subscribers.len() - } - - /// Set the content status callback. - /// - /// Only one callback can be active at a time. If a previous callback was registered, this - /// will return `false`. - pub fn set_content_status_callback(&mut self, cb: ContentStatusCallback) -> bool { - if self.content_status_cb.is_some() { - false - } else { - self.content_status_cb = Some(cb); - true - } - } - - fn ensure_open(&self) -> Result<(), InsertError> { - if self.closed() { - Err(InsertError::Closed) - } else { - Ok(()) - } - } - - /// Returns true if the replica is closed. - /// - /// If a replica is closed, no further operations can be performed. A replica cannot be closed - /// manually, it must be closed via [`store::Store::close_replica`] or - /// [`store::Store::remove_replica`] - pub fn closed(&self) -> bool { - self.closed - } - - /// Merge a capability. - /// - /// The capability must refer to the the same namespace, otherwise an error will be returned. - /// - /// This will upgrade the replica's capability when passing a `Capability::Write`. - /// It is a no-op if `capability` is a Capability::Read`. - pub fn merge_capability(&mut self, capability: Capability) -> Result { - self.capability.merge(capability) - } -} - -/// Local representation of a mutable, synchronizable key-value store. -#[derive(derive_more::Debug)] -pub struct Replica<'a, I = Box> { - pub(crate) store: StoreInstance<'a>, - pub(crate) info: I, -} - -impl<'a, I> Replica<'a, I> -where - I: Deref + DerefMut, -{ - /// Create a new replica. - pub fn new(store: StoreInstance<'a>, info: I) -> Self { - Replica { info, store } - } - - /// Insert a new record at the given key. - /// - /// The entry will by signed by the provided `author`. - /// The `len` must be the byte length of the data identified by `hash`. - /// - /// Returns the number of entries removed as a consequence of this insertion, - /// or an error either if the entry failed to validate or if a store operation failed. - pub fn insert( - &mut self, - key: impl AsRef<[u8]>, - author: &Author, - hash: Hash, - len: u64, - ) -> Result { - if len == 0 || hash == Hash::EMPTY { - return Err(InsertError::EntryIsEmpty); - } - self.info.ensure_open()?; - let id = RecordIdentifier::new(self.id(), author.id(), key); - let record = Record::new_current(hash, len); - let entry = Entry::new(id, record); - let secret = self.secret_key()?; - let signed_entry = entry.sign(secret, author); - self.insert_entry(signed_entry, InsertOrigin::Local) - } - - /// Delete entries that match the given `author` and key `prefix`. - /// - /// This inserts an empty entry with the key set to `prefix`, effectively clearing all other - /// entries whose key starts with or is equal to the given `prefix`. - /// - /// Returns the number of entries deleted. - pub fn delete_prefix( - &mut self, - prefix: impl AsRef<[u8]>, - author: &Author, - ) -> Result { - self.info.ensure_open()?; - let id = RecordIdentifier::new(self.id(), author.id(), prefix); - let entry = Entry::new_empty(id); - let signed_entry = entry.sign(self.secret_key()?, author); - self.insert_entry(signed_entry, InsertOrigin::Local) - } - - /// Insert an entry into this replica which was received from a remote peer. - /// - /// This will verify both the namespace and author signatures of the entry, emit an `on_insert` - /// event, and insert the entry into the replica store. - /// - /// Returns the number of entries removed as a consequence of this insertion, - /// or an error if the entry failed to validate or if a store operation failed. - pub fn insert_remote_entry( - &mut self, - entry: SignedEntry, - received_from: PeerIdBytes, - content_status: ContentStatus, - ) -> Result { - self.info.ensure_open()?; - entry.validate_empty()?; - let origin = InsertOrigin::Sync { - from: received_from, - remote_content_status: content_status, - }; - self.insert_entry(entry, origin) - } - - /// Insert a signed entry into the database. - /// - /// Returns the number of entries removed as a consequence of this insertion. - fn insert_entry( - &mut self, - entry: SignedEntry, - origin: InsertOrigin, - ) -> Result { - let namespace = self.id(); - - #[cfg(feature = "metrics")] - let len = entry.content_len(); - - let store = &self.store; - validate_entry(system_time_now(), store, namespace, &entry, &origin)?; - - let outcome = self.store.put(entry.clone()).map_err(InsertError::Store)?; - tracing::debug!(?origin, hash = %entry.content_hash(), ?outcome, "insert"); - - let removed_count = match outcome { - InsertOutcome::Inserted { removed } => removed, - InsertOutcome::NotInserted => return Err(InsertError::NewerEntryExists), - }; - - let insert_event = match origin { - InsertOrigin::Local => { - #[cfg(feature = "metrics")] - { - inc!(Metrics, new_entries_local); - inc_by!(Metrics, new_entries_local_size, len); - } - Event::LocalInsert { namespace, entry } - } - InsertOrigin::Sync { - from, - remote_content_status, - } => { - #[cfg(feature = "metrics")] - { - inc!(Metrics, new_entries_remote); - inc_by!(Metrics, new_entries_remote_size, len); - } - - let download_policy = self - .store - .get_download_policy(&self.id()) - .unwrap_or_default(); - let should_download = download_policy.matches(entry.entry()); - Event::RemoteInsert { - namespace, - entry, - from, - should_download, - remote_content_status, - } - } - }; - - self.info.subscribers.send(insert_event); - - Ok(removed_count) - } - - /// Hashes the given data and inserts it. - /// - /// This does not store the content, just the record of it. - /// Returns the calculated hash. - pub fn hash_and_insert( - &mut self, - key: impl AsRef<[u8]>, - author: &Author, - data: impl AsRef<[u8]>, - ) -> Result { - self.info.ensure_open()?; - let len = data.as_ref().len() as u64; - let hash = Hash::new(data); - self.insert(key, author, hash, len)?; - Ok(hash) - } - - /// Get the identifier for an entry in this replica. - pub fn record_id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { - RecordIdentifier::new(self.info.capability.id(), author.id(), key) - } - - /// Create the initial message for the set reconciliation flow with a remote peer. - pub fn sync_initial_message(&mut self) -> anyhow::Result> { - self.info.ensure_open().map_err(anyhow::Error::from)?; - self.store.initial_message().map_err(Into::into) - } - - /// Process a set reconciliation message from a remote peer. - /// - /// Returns the next message to be sent to the peer, if any. - pub fn sync_process_message( - &mut self, - message: crate::ranger::Message, - from_peer: PeerIdBytes, - state: &mut SyncOutcome, - ) -> Result>, anyhow::Error> { - self.info.ensure_open()?; - let my_namespace = self.id(); - let now = system_time_now(); - - // update state with incoming data. - state.num_recv += message.value_count(); - for (entry, _content_status) in message.values() { - state - .heads_received - .insert(entry.author(), entry.timestamp()); - } - - // let subscribers = std::rc::Rc::new(&mut self.subscribers); - // l - let cb = self.info.content_status_cb.clone(); - let download_policy = self - .store - .get_download_policy(&my_namespace) - .unwrap_or_default(); - let reply = self.store.process_message( - &Default::default(), - message, - // validate callback: validate incoming entries, and send to on_insert channel - |store, entry, content_status| { - let origin = InsertOrigin::Sync { - from: from_peer, - remote_content_status: content_status, - }; - validate_entry(now, store, my_namespace, entry, &origin).is_ok() - }, - // on_insert callback: is called when an entry was actually inserted in the store - |_store, entry, content_status| { - // We use `send_with` to only clone the entry if we have active subscriptions. - self.info.subscribers.send_with(|| { - let should_download = download_policy.matches(entry.entry()); - Event::RemoteInsert { - from: from_peer, - namespace: my_namespace, - entry: entry.clone(), - should_download, - remote_content_status: content_status, - } - }) - }, - // content_status callback: get content status for outgoing entries - |_store, entry| { - if let Some(cb) = cb.as_ref() { - cb(entry.content_hash()) - } else { - ContentStatus::Missing - } - }, - )?; - - // update state with outgoing data. - if let Some(ref reply) = reply { - state.num_sent += reply.value_count(); - } - - Ok(reply) - } - - /// Get the namespace identifier for this [`Replica`]. - pub fn id(&self) -> NamespaceId { - self.info.capability.id() - } - - /// Get the [`Capability`] of this [`Replica`]. - pub fn capability(&self) -> &Capability { - &self.info.capability - } - - /// Get the byte representation of the [`NamespaceSecret`] key for this replica. Will fail if - /// the replica is read only - pub fn secret_key(&self) -> Result<&NamespaceSecret, ReadOnly> { - self.info.capability.secret_key() - } -} - -/// Error that occurs trying to access the [`NamespaceSecret`] of a read-only [`Capability`]. -#[derive(Debug, thiserror::Error)] -#[error("Replica allows read access only.")] -pub struct ReadOnly; - -/// Validate a [`SignedEntry`] if it's fit to be inserted. -/// -/// This validates that -/// * the entry's author and namespace signatures are correct -/// * the entry's namespace matches the current replica -/// * the entry's timestamp is not more than 10 minutes in the future of our system time -/// * the entry is newer than an existing entry for the same key and author, if such exists. -fn validate_entry + PublicKeyStore>( - now: u64, - store: &S, - expected_namespace: NamespaceId, - entry: &SignedEntry, - origin: &InsertOrigin, -) -> Result<(), ValidationFailure> { - // Verify the namespace - if entry.namespace() != expected_namespace { - return Err(ValidationFailure::InvalidNamespace); - } - - // Verify signature for non-local entries. - if !matches!(origin, InsertOrigin::Local) && entry.verify(store).is_err() { - return Err(ValidationFailure::BadSignature); - } - - // Verify that the timestamp of the entry is not too far in the future. - if entry.timestamp() > now + MAX_TIMESTAMP_FUTURE_SHIFT { - return Err(ValidationFailure::TooFarInTheFuture); - } - Ok(()) -} - -/// Error emitted when inserting entries into a [`Replica`] failed -#[derive(thiserror::Error, derive_more::Debug, derive_more::From)] -pub enum InsertError { - /// Storage error - #[error("storage error")] - Store(anyhow::Error), - /// Validation failure - #[error("validation failure")] - Validation(#[from] ValidationFailure), - /// A newer entry exists for either this entry's key or a prefix of the key. - #[error("A newer entry exists for either this entry's key or a prefix of the key.")] - NewerEntryExists, - /// Attempted to insert an empty entry. - #[error("Attempted to insert an empty entry")] - EntryIsEmpty, - /// Replica is read only. - #[error("Attempted to insert to read only replica")] - #[from(ReadOnly)] - ReadOnly, - /// The replica is closed, no operations may be performed. - #[error("replica is closed")] - Closed, -} - -/// Reason why entry validation failed -#[derive(thiserror::Error, Debug)] -pub enum ValidationFailure { - /// Entry namespace does not match the current replica. - #[error("Entry namespace does not match the current replica")] - InvalidNamespace, - /// Entry signature is invalid. - #[error("Entry signature is invalid")] - BadSignature, - /// Entry timestamp is too far in the future. - #[error("Entry timestamp is too far in the future.")] - TooFarInTheFuture, - /// Entry has length 0 but not the empty hash, or the empty hash but not length 0. - #[error("Entry has length 0 but not the empty hash, or the empty hash but not length 0")] - InvalidEmptyEntry, -} - -/// A signed entry. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct SignedEntry { - signature: EntrySignature, - entry: Entry, -} - -impl From for Entry { - fn from(value: SignedEntry) -> Self { - value.entry - } -} - -impl PartialOrd for SignedEntry { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for SignedEntry { - fn cmp(&self, other: &Self) -> Ordering { - self.entry.cmp(&other.entry) - } -} - -impl PartialOrd for Entry { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Entry { - fn cmp(&self, other: &Self) -> Ordering { - self.id - .cmp(&other.id) - .then_with(|| self.record.cmp(&other.record)) - } -} - -impl SignedEntry { - pub(crate) fn new(signature: EntrySignature, entry: Entry) -> Self { - SignedEntry { signature, entry } - } - - /// Create a new signed entry by signing an entry with the `namespace` and `author`. - pub fn from_entry(entry: Entry, namespace: &NamespaceSecret, author: &Author) -> Self { - let signature = EntrySignature::from_entry(&entry, namespace, author); - SignedEntry { signature, entry } - } - - /// Create a new signed entries from its parts. - pub fn from_parts( - namespace: &NamespaceSecret, - author: &Author, - key: impl AsRef<[u8]>, - record: Record, - ) -> Self { - let id = RecordIdentifier::new(namespace.id(), author.id(), key); - let entry = Entry::new(id, record); - Self::from_entry(entry, namespace, author) - } - - /// Verify the signatures on this entry. - pub fn verify(&self, store: &S) -> Result<(), SignatureError> { - self.signature.verify( - &self.entry, - &self.entry.namespace().public_key(store)?, - &self.entry.author().public_key(store)?, - ) - } - - /// Get the signature. - pub fn signature(&self) -> &EntrySignature { - &self.signature - } - - /// Validate that the entry has the empty hash if the length is 0, or a non-zero length. - pub fn validate_empty(&self) -> Result<(), ValidationFailure> { - self.entry().validate_empty() - } - - /// Get the [`Entry`]. - pub fn entry(&self) -> &Entry { - &self.entry - } - - /// Get the content [`struct@Hash`] of the entry. - pub fn content_hash(&self) -> Hash { - self.entry().content_hash() - } - - /// Get the content length of the entry. - pub fn content_len(&self) -> u64 { - self.entry().content_len() - } - - /// Get the author bytes of this entry. - pub fn author_bytes(&self) -> AuthorId { - self.entry().id().author() - } - - /// Get the key of the entry. - pub fn key(&self) -> &[u8] { - self.entry().id().key() - } - - /// Get the timestamp of the entry. - pub fn timestamp(&self) -> u64 { - self.entry().timestamp() - } -} - -impl RangeEntry for SignedEntry { - type Key = RecordIdentifier; - type Value = Record; - - fn key(&self) -> &Self::Key { - &self.entry.id - } - - fn value(&self) -> &Self::Value { - &self.entry.record - } - - fn as_fingerprint(&self) -> crate::ranger::Fingerprint { - let mut hasher = blake3::Hasher::new(); - hasher.update(self.namespace().as_ref()); - hasher.update(self.author_bytes().as_ref()); - hasher.update(self.key()); - hasher.update(&self.timestamp().to_be_bytes()); - hasher.update(self.content_hash().as_bytes()); - Fingerprint(hasher.finalize().into()) - } -} - -/// Signature over an entry. -#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct EntrySignature { - author_signature: Signature, - namespace_signature: Signature, -} - -impl Debug for EntrySignature { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("EntrySignature") - .field( - "namespace_signature", - &base32::fmt(self.namespace_signature.to_bytes()), - ) - .field( - "author_signature", - &base32::fmt(self.author_signature.to_bytes()), - ) - .finish() - } -} - -impl EntrySignature { - /// Create a new signature by signing an entry with the `namespace` and `author`. - pub fn from_entry(entry: &Entry, namespace: &NamespaceSecret, author: &Author) -> Self { - // TODO: this should probably include a namespace prefix - // namespace in the cryptographic sense. - let bytes = entry.to_vec(); - let namespace_signature = namespace.sign(&bytes); - let author_signature = author.sign(&bytes); - - EntrySignature { - author_signature, - namespace_signature, - } - } - - /// Verify that this signature was created by signing the `entry` with the - /// secret keys of the specified `author` and `namespace`. - pub fn verify( - &self, - entry: &Entry, - namespace: &NamespacePublicKey, - author: &AuthorPublicKey, - ) -> Result<(), SignatureError> { - let bytes = entry.to_vec(); - namespace.verify(&bytes, &self.namespace_signature)?; - author.verify(&bytes, &self.author_signature)?; - - Ok(()) - } - - pub(crate) fn from_parts(namespace_sig: &[u8; 64], author_sig: &[u8; 64]) -> Self { - let namespace_signature = Signature::from_bytes(namespace_sig); - let author_signature = Signature::from_bytes(author_sig); - - EntrySignature { - author_signature, - namespace_signature, - } - } - - pub(crate) fn author(&self) -> &Signature { - &self.author_signature - } - - pub(crate) fn namespace(&self) -> &Signature { - &self.namespace_signature - } -} - -/// A single entry in a [`Replica`] -/// -/// An entry is identified by a key, its [`Author`], and the [`Replica`]'s -/// [`NamespaceSecret`]. Its value is the [32-byte BLAKE3 hash](iroh_base::hash::Hash) -/// of the entry's content data, the size of this content data, and a timestamp. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Entry { - id: RecordIdentifier, - record: Record, -} - -impl Entry { - /// Create a new entry - pub fn new(id: RecordIdentifier, record: Record) -> Self { - Entry { id, record } - } - - /// Create a new empty entry with the current timestamp. - pub fn new_empty(id: RecordIdentifier) -> Self { - Entry { - id, - record: Record::empty_current(), - } - } - - /// Validate that the entry has the empty hash if the length is 0, or a non-zero length. - pub fn validate_empty(&self) -> Result<(), ValidationFailure> { - match (self.content_hash() == Hash::EMPTY, self.content_len() == 0) { - (true, true) => Ok(()), - (false, false) => Ok(()), - (true, false) => Err(ValidationFailure::InvalidEmptyEntry), - (false, true) => Err(ValidationFailure::InvalidEmptyEntry), - } - } - - /// Get the [`RecordIdentifier`] for this entry. - pub fn id(&self) -> &RecordIdentifier { - &self.id - } - - /// Get the [`NamespaceId`] of this entry. - pub fn namespace(&self) -> NamespaceId { - self.id.namespace() - } - - /// Get the [`AuthorId`] of this entry. - pub fn author(&self) -> AuthorId { - self.id.author() - } - - /// Get the key of this entry. - pub fn key(&self) -> &[u8] { - self.id.key() - } - - /// Get the [`Record`] contained in this entry. - pub fn record(&self) -> &Record { - &self.record - } - - /// Serialize this entry into its canonical byte representation used for signing. - pub fn encode(&self, out: &mut Vec) { - self.id.encode(out); - self.record.encode(out); - } - - /// Serialize this entry into a new vector with its canonical byte representation. - pub fn to_vec(&self) -> Vec { - let mut out = Vec::new(); - self.encode(&mut out); - out - } - - /// Sign this entry with a [`NamespaceSecret`] and [`Author`]. - pub fn sign(self, namespace: &NamespaceSecret, author: &Author) -> SignedEntry { - SignedEntry::from_entry(self, namespace, author) - } -} - -const NAMESPACE_BYTES: std::ops::Range = 0..32; -const AUTHOR_BYTES: std::ops::Range = 32..64; -const KEY_BYTES: std::ops::RangeFrom = 64..; - -/// The identifier of a record. -#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct RecordIdentifier(Bytes); - -impl Default for RecordIdentifier { - fn default() -> Self { - Self::new(NamespaceId::default(), AuthorId::default(), b"") - } -} - -impl Debug for RecordIdentifier { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RecordIdentifier") - .field("namespace", &self.namespace()) - .field("author", &self.author()) - .field("key", &std::string::String::from_utf8_lossy(self.key())) - .finish() - } -} - -impl RangeKey for RecordIdentifier { - #[cfg(test)] - fn is_prefix_of(&self, other: &Self) -> bool { - other.as_ref().starts_with(self.as_ref()) - } -} - -fn system_time_now() -> u64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("time drift") - .as_micros() as u64 -} - -impl RecordIdentifier { - /// Create a new [`RecordIdentifier`]. - pub fn new( - namespace: impl Into, - author: impl Into, - key: impl AsRef<[u8]>, - ) -> Self { - let mut bytes = BytesMut::with_capacity(32 + 32 + key.as_ref().len()); - bytes.extend_from_slice(namespace.into().as_bytes()); - bytes.extend_from_slice(author.into().as_bytes()); - bytes.extend_from_slice(key.as_ref()); - Self(bytes.freeze()) - } - - /// Serialize this [`RecordIdentifier`] into a mutable byte array. - pub(crate) fn encode(&self, out: &mut Vec) { - out.extend_from_slice(&self.0); - } - - /// Get this [`RecordIdentifier`] as [Bytes]. - pub fn as_bytes(&self) -> Bytes { - self.0.clone() - } - - /// Get this [`RecordIdentifier`] as a tuple of byte slices. - pub fn as_byte_tuple(&self) -> (&[u8; 32], &[u8; 32], &[u8]) { - ( - self.0[NAMESPACE_BYTES].try_into().unwrap(), - self.0[AUTHOR_BYTES].try_into().unwrap(), - &self.0[KEY_BYTES], - ) - } - - /// Get this [`RecordIdentifier`] as a tuple of bytes. - pub fn to_byte_tuple(&self) -> ([u8; 32], [u8; 32], Bytes) { - ( - self.0[NAMESPACE_BYTES].try_into().unwrap(), - self.0[AUTHOR_BYTES].try_into().unwrap(), - self.0.slice(KEY_BYTES), - ) - } - - /// Get the key of this record. - pub fn key(&self) -> &[u8] { - &self.0[KEY_BYTES] - } - - /// Get the key of this record as [`Bytes`]. - pub fn key_bytes(&self) -> Bytes { - self.0.slice(KEY_BYTES) - } - - /// Get the [`NamespaceId`] of this record as byte array. - pub fn namespace(&self) -> NamespaceId { - let value: &[u8; 32] = &self.0[NAMESPACE_BYTES].try_into().unwrap(); - value.into() - } - - /// Get the [`AuthorId`] of this record as byte array. - pub fn author(&self) -> AuthorId { - let value: &[u8; 32] = &self.0[AUTHOR_BYTES].try_into().unwrap(); - value.into() - } -} - -impl AsRef<[u8]> for RecordIdentifier { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl Deref for SignedEntry { - type Target = Entry; - fn deref(&self) -> &Self::Target { - &self.entry - } -} - -impl Deref for Entry { - type Target = Record; - fn deref(&self) -> &Self::Target { - &self.record - } -} - -/// The data part of an entry in a [`Replica`]. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Record { - /// Length of the data referenced by `hash`. - len: u64, - /// Hash of the content data. - hash: Hash, - /// Record creation timestamp. Counted as micros since the Unix epoch. - timestamp: u64, -} - -impl RangeValue for Record {} - -/// Ordering for entry values. -/// -/// Compares first the timestamp, then the content hash. -impl Ord for Record { - fn cmp(&self, other: &Self) -> Ordering { - self.timestamp - .cmp(&other.timestamp) - .then_with(|| self.hash.cmp(&other.hash)) - } -} - -impl PartialOrd for Record { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Record { - /// Create a new record. - pub fn new(hash: Hash, len: u64, timestamp: u64) -> Self { - debug_assert!( - len != 0 || hash == Hash::EMPTY, - "if `len` is 0 then `hash` must be the hash of the empty byte range" - ); - Record { - hash, - len, - timestamp, - } - } - - /// Create a tombstone record (empty content) - pub fn empty(timestamp: u64) -> Self { - Self::new(Hash::EMPTY, 0, timestamp) - } - - /// Create a tombstone record with the timestamp set to now. - pub fn empty_current() -> Self { - Self::new_current(Hash::EMPTY, 0) - } - - /// Return `true` if the entry is empty. - pub fn is_empty(&self) -> bool { - self.hash == Hash::EMPTY - } - - /// Create a new [`Record`] with the timestamp set to now. - pub fn new_current(hash: Hash, len: u64) -> Self { - let timestamp = system_time_now(); - Self::new(hash, len, timestamp) - } - - /// Get the length of the data addressed by this record's content hash. - pub fn content_len(&self) -> u64 { - self.len - } - - /// Get the [`struct@Hash`] of the content data of this record. - pub fn content_hash(&self) -> Hash { - self.hash - } - - /// Get the timestamp of this record. - pub fn timestamp(&self) -> u64 { - self.timestamp - } - - #[cfg(test)] - pub(crate) fn current_from_data(data: impl AsRef<[u8]>) -> Self { - let len = data.as_ref().len() as u64; - let hash = Hash::new(data); - Self::new_current(hash, len) - } - - #[cfg(test)] - pub(crate) fn from_data(data: impl AsRef<[u8]>, timestamp: u64) -> Self { - let len = data.as_ref().len() as u64; - let hash = Hash::new(data); - Self::new(hash, len, timestamp) - } - - /// Serialize this record into a mutable byte array. - pub(crate) fn encode(&self, out: &mut Vec) { - out.extend_from_slice(&self.len.to_be_bytes()); - out.extend_from_slice(self.hash.as_ref()); - out.extend_from_slice(&self.timestamp.to_be_bytes()) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use anyhow::Result; - use rand_core::SeedableRng; - - use super::*; - use crate::{ - actor::SyncHandle, - ranger::{Range, Store as _}, - store::{OpenError, Query, SortBy, SortDirection, Store}, - }; - - #[test] - fn test_basics_memory() -> Result<()> { - let store = store::Store::memory(); - test_basics(store)?; - - Ok(()) - } - - #[test] - fn test_basics_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_basics(store)?; - Ok(()) - } - - fn test_basics(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let alice = Author::new(&mut rng); - let bob = Author::new(&mut rng); - let myspace = NamespaceSecret::new(&mut rng); - - let record_id = RecordIdentifier::new(myspace.id(), alice.id(), "/my/key"); - let record = Record::current_from_data(b"this is my cool data"); - let entry = Entry::new(record_id, record); - let signed_entry = entry.sign(&myspace, &alice); - signed_entry.verify(&()).expect("failed to verify"); - - let mut my_replica = store.new_replica(myspace.clone())?; - for i in 0..10 { - my_replica.hash_and_insert( - format!("/{i}"), - &alice, - format!("{i}: hello from alice"), - )?; - } - - for i in 0..10 { - let res = store - .get_exact(myspace.id(), alice.id(), format!("/{i}"), false)? - .unwrap(); - let len = format!("{i}: hello from alice").as_bytes().len() as u64; - assert_eq!(res.entry().record().content_len(), len); - res.verify(&())?; - } - - // Test multiple records for the same key - let mut my_replica = store.new_replica(myspace.clone())?; - my_replica.hash_and_insert("/cool/path", &alice, "round 1")?; - let _entry = store - .get_exact(myspace.id(), alice.id(), "/cool/path", false)? - .unwrap(); - // Second - let mut my_replica = store.new_replica(myspace.clone())?; - my_replica.hash_and_insert("/cool/path", &alice, "round 2")?; - let _entry = store - .get_exact(myspace.id(), alice.id(), "/cool/path", false)? - .unwrap(); - - // Get All by author - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(alice.id()))? - .collect::>()?; - assert_eq!(entries.len(), 11); - - // Get All by author - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(bob.id()))? - .collect::>()?; - assert_eq!(entries.len(), 0); - - // Get All by key - let entries: Vec<_> = store - .get_many(myspace.id(), Query::key_exact(b"/cool/path"))? - .collect::>()?; - assert_eq!(entries.len(), 1); - - // Get All - let entries: Vec<_> = store - .get_many(myspace.id(), Query::all())? - .collect::>()?; - assert_eq!(entries.len(), 11); - - // insert record from different author - let mut my_replica = store.new_replica(myspace.clone())?; - let _entry = my_replica.hash_and_insert("/cool/path", &bob, "bob round 1")?; - - // Get All by author - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(alice.id()))? - .collect::>()?; - assert_eq!(entries.len(), 11); - - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(bob.id()))? - .collect::>()?; - assert_eq!(entries.len(), 1); - - // Get All by key - let entries: Vec<_> = store - .get_many(myspace.id(), Query::key_exact(b"/cool/path"))? - .collect::>()?; - assert_eq!(entries.len(), 2); - - // Get all by prefix - let entries: Vec<_> = store - .get_many(myspace.id(), Query::key_prefix(b"/cool"))? - .collect::>()?; - assert_eq!(entries.len(), 2); - - // Get All by author and prefix - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(alice.id()).key_prefix(b"/cool"))? - .collect::>()?; - assert_eq!(entries.len(), 1); - - let entries: Vec<_> = store - .get_many(myspace.id(), Query::author(bob.id()).key_prefix(b"/cool"))? - .collect::>()?; - assert_eq!(entries.len(), 1); - - // Get All - let entries: Vec<_> = store - .get_many(myspace.id(), Query::all())? - .collect::>()?; - assert_eq!(entries.len(), 12); - - // Get Range of all should return all latest - let mut my_replica = store.new_replica(myspace.clone())?; - let entries_second: Vec<_> = my_replica - .store - .get_range(Range::new( - RecordIdentifier::default(), - RecordIdentifier::default(), - ))? - .collect::>()?; - - assert_eq!(entries_second.len(), 12); - assert_eq!(entries, entries_second.into_iter().collect::>()); - - test_lru_cache_like_behaviour(&mut store, myspace.id()) - } - - /// Test that [`Store::register_useful_peer`] behaves like a LRUCache of size - /// [`super::store::PEERS_PER_DOC_CACHE_SIZE`]. - fn test_lru_cache_like_behaviour(store: &mut Store, namespace: NamespaceId) -> Result<()> { - /// Helper to verify the store returns the expected peers for the namespace. - #[track_caller] - fn verify_peers(store: &mut Store, namespace: NamespaceId, expected_peers: &Vec<[u8; 32]>) { - assert_eq!( - expected_peers, - &store - .get_sync_peers(&namespace) - .unwrap() - .unwrap() - .collect::>(), - "sync peers differ" - ); - } - - let count = super::store::PEERS_PER_DOC_CACHE_SIZE.get(); - // expected peers: newest peers are to the front, oldest to the back - let mut expected_peers = Vec::with_capacity(count); - for i in 0..count as u8 { - let peer = [i; 32]; - expected_peers.insert(0, peer); - store.register_useful_peer(namespace, peer)?; - } - verify_peers(store, namespace, &expected_peers); - - // one more peer should evict the last peer - expected_peers.pop(); - let newer_peer = [count as u8; 32]; - expected_peers.insert(0, newer_peer); - store.register_useful_peer(namespace, newer_peer)?; - verify_peers(store, namespace, &expected_peers); - - // move one existing peer up - let refreshed_peer = expected_peers.remove(2); - expected_peers.insert(0, refreshed_peer); - store.register_useful_peer(namespace, refreshed_peer)?; - verify_peers(store, namespace, &expected_peers); - Ok(()) - } - - #[test] - fn test_content_hashes_iterator_memory() -> Result<()> { - let store = store::Store::memory(); - test_content_hashes_iterator(store) - } - - #[test] - fn test_content_hashes_iterator_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_content_hashes_iterator(store) - } - - fn test_content_hashes_iterator(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let mut expected = HashSet::new(); - let n_replicas = 3; - let n_entries = 4; - for i in 0..n_replicas { - let namespace = NamespaceSecret::new(&mut rng); - let author = store.new_author(&mut rng)?; - let mut replica = store.new_replica(namespace)?; - for j in 0..n_entries { - let key = format!("{j}"); - let data = format!("{i}:{j}"); - let hash = replica.hash_and_insert(key, &author, data)?; - expected.insert(hash); - } - } - assert_eq!(expected.len(), n_replicas * n_entries); - let actual = store.content_hashes()?.collect::>>()?; - assert_eq!(actual, expected); - Ok(()) - } - - #[test] - fn test_multikey() { - let mut rng = rand::thread_rng(); - - let k = ["a", "c", "z"]; - - let mut n: Vec<_> = (0..3).map(|_| NamespaceSecret::new(&mut rng)).collect(); - n.sort_by_key(|n| n.id()); - - let mut a: Vec<_> = (0..3).map(|_| Author::new(&mut rng)).collect(); - a.sort_by_key(|a| a.id()); - - // Just key - { - let ri0 = RecordIdentifier::new(n[0].id(), a[0].id(), k[0]); - let ri1 = RecordIdentifier::new(n[0].id(), a[0].id(), k[1]); - let ri2 = RecordIdentifier::new(n[0].id(), a[0].id(), k[2]); - - let range = Range::new(ri0.clone(), ri2.clone()); - assert!(range.contains(&ri0), "start"); - assert!(range.contains(&ri1), "inside"); - assert!(!range.contains(&ri2), "end"); - - assert!(ri0 < ri1); - assert!(ri1 < ri2); - } - - // Just namespace - { - let ri0 = RecordIdentifier::new(n[0].id(), a[0].id(), k[0]); - let ri1 = RecordIdentifier::new(n[1].id(), a[0].id(), k[1]); - let ri2 = RecordIdentifier::new(n[2].id(), a[0].id(), k[2]); - - let range = Range::new(ri0.clone(), ri2.clone()); - assert!(range.contains(&ri0), "start"); - assert!(range.contains(&ri1), "inside"); - assert!(!range.contains(&ri2), "end"); - - assert!(ri0 < ri1); - assert!(ri1 < ri2); - } - - // Just author - { - let ri0 = RecordIdentifier::new(n[0].id(), a[0].id(), k[0]); - let ri1 = RecordIdentifier::new(n[0].id(), a[1].id(), k[0]); - let ri2 = RecordIdentifier::new(n[0].id(), a[2].id(), k[0]); - - let range = Range::new(ri0.clone(), ri2.clone()); - assert!(range.contains(&ri0), "start"); - assert!(range.contains(&ri1), "inside"); - assert!(!range.contains(&ri2), "end"); - - assert!(ri0 < ri1); - assert!(ri1 < ri2); - } - - // Just key and namespace - { - let ri0 = RecordIdentifier::new(n[0].id(), a[0].id(), k[0]); - let ri1 = RecordIdentifier::new(n[1].id(), a[0].id(), k[1]); - let ri2 = RecordIdentifier::new(n[2].id(), a[0].id(), k[2]); - - let range = Range::new(ri0.clone(), ri2.clone()); - assert!(range.contains(&ri0), "start"); - assert!(range.contains(&ri1), "inside"); - assert!(!range.contains(&ri2), "end"); - - assert!(ri0 < ri1); - assert!(ri1 < ri2); - } - - // Mixed - { - // Ord should prioritize namespace - author - key - - let a0 = a[0].id(); - let a1 = a[1].id(); - let n0 = n[0].id(); - let n1 = n[1].id(); - let k0 = k[0]; - let k1 = k[1]; - - assert!(RecordIdentifier::new(n0, a0, k0) < RecordIdentifier::new(n1, a1, k1)); - assert!(RecordIdentifier::new(n0, a0, k1) < RecordIdentifier::new(n1, a0, k0)); - assert!(RecordIdentifier::new(n0, a1, k0) < RecordIdentifier::new(n0, a1, k1)); - assert!(RecordIdentifier::new(n1, a1, k0) < RecordIdentifier::new(n1, a1, k1)); - } - } - - #[test] - fn test_timestamps_memory() -> Result<()> { - let store = store::Store::memory(); - test_timestamps(store)?; - - Ok(()) - } - - #[test] - fn test_timestamps_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_timestamps(store)?; - Ok(()) - } - - fn test_timestamps(mut store: Store) -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let namespace = NamespaceSecret::new(&mut rng); - let _replica = store.new_replica(namespace.clone())?; - let author = store.new_author(&mut rng)?; - store.close_replica(namespace.id()); - let mut replica = store.open_replica(&namespace.id())?; - - let key = b"hello"; - let value = b"world"; - let entry = { - let timestamp = 2; - let id = RecordIdentifier::new(namespace.id(), author.id(), key); - let record = Record::from_data(value, timestamp); - Entry::new(id, record).sign(&namespace, &author) - }; - - replica - .insert_entry(entry.clone(), InsertOrigin::Local) - .unwrap(); - store.close_replica(namespace.id()); - let res = store - .get_exact(namespace.id(), author.id(), key, false)? - .unwrap(); - assert_eq!(res, entry); - - let entry2 = { - let timestamp = 1; - let id = RecordIdentifier::new(namespace.id(), author.id(), key); - let record = Record::from_data(value, timestamp); - Entry::new(id, record).sign(&namespace, &author) - }; - - let mut replica = store.open_replica(&namespace.id())?; - let res = replica.insert_entry(entry2, InsertOrigin::Local); - store.close_replica(namespace.id()); - assert!(matches!(res, Err(InsertError::NewerEntryExists))); - let res = store - .get_exact(namespace.id(), author.id(), key, false)? - .unwrap(); - assert_eq!(res, entry); - - Ok(()) - } - - #[test] - fn test_replica_sync_memory() -> Result<()> { - let alice_store = store::Store::memory(); - let bob_store = store::Store::memory(); - - test_replica_sync(alice_store, bob_store)?; - Ok(()) - } - - #[test] - fn test_replica_sync_fs() -> Result<()> { - let alice_dbfile = tempfile::NamedTempFile::new()?; - let alice_store = store::fs::Store::persistent(alice_dbfile.path())?; - let bob_dbfile = tempfile::NamedTempFile::new()?; - let bob_store = store::fs::Store::persistent(bob_dbfile.path())?; - test_replica_sync(alice_store, bob_store)?; - - Ok(()) - } - - fn test_replica_sync(mut alice_store: Store, mut bob_store: Store) -> Result<()> { - let alice_set = ["ape", "eel", "fox", "gnu"]; - let bob_set = ["bee", "cat", "doe", "eel", "fox", "hog"]; - - let mut rng = rand::thread_rng(); - let author = Author::new(&mut rng); - let myspace = NamespaceSecret::new(&mut rng); - let mut alice = alice_store.new_replica(myspace.clone())?; - for el in &alice_set { - alice.hash_and_insert(el, &author, el.as_bytes())?; - } - - let mut bob = bob_store.new_replica(myspace.clone())?; - for el in &bob_set { - bob.hash_and_insert(el, &author, el.as_bytes())?; - } - - let (alice_out, bob_out) = sync(&mut alice, &mut bob)?; - - assert_eq!(alice_out.num_sent, 2); - assert_eq!(bob_out.num_recv, 2); - assert_eq!(alice_out.num_recv, 6); - assert_eq!(bob_out.num_sent, 6); - - check_entries(&mut alice_store, &myspace.id(), &author, &alice_set)?; - check_entries(&mut alice_store, &myspace.id(), &author, &bob_set)?; - check_entries(&mut bob_store, &myspace.id(), &author, &alice_set)?; - check_entries(&mut bob_store, &myspace.id(), &author, &bob_set)?; - - Ok(()) - } - - #[test] - fn test_replica_timestamp_sync_memory() -> Result<()> { - let alice_store = store::Store::memory(); - let bob_store = store::Store::memory(); - - test_replica_timestamp_sync(alice_store, bob_store)?; - Ok(()) - } - - #[test] - fn test_replica_timestamp_sync_fs() -> Result<()> { - let alice_dbfile = tempfile::NamedTempFile::new()?; - let alice_store = store::fs::Store::persistent(alice_dbfile.path())?; - let bob_dbfile = tempfile::NamedTempFile::new()?; - let bob_store = store::fs::Store::persistent(bob_dbfile.path())?; - test_replica_timestamp_sync(alice_store, bob_store)?; - - Ok(()) - } - - fn test_replica_timestamp_sync(mut alice_store: Store, mut bob_store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let author = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - let mut alice = alice_store.new_replica(namespace.clone())?; - let mut bob = bob_store.new_replica(namespace.clone())?; - - let key = b"key"; - let alice_value = b"alice"; - let bob_value = b"bob"; - let _alice_hash = alice.hash_and_insert(key, &author, alice_value)?; - // system time increased - sync should overwrite - let bob_hash = bob.hash_and_insert(key, &author, bob_value)?; - sync(&mut alice, &mut bob)?; - assert_eq!( - get_content_hash(&mut alice_store, namespace.id(), author.id(), key)?, - Some(bob_hash) - ); - assert_eq!( - get_content_hash(&mut alice_store, namespace.id(), author.id(), key)?, - Some(bob_hash) - ); - - let mut alice = alice_store.new_replica(namespace.clone())?; - let mut bob = bob_store.new_replica(namespace.clone())?; - - let alice_value_2 = b"alice2"; - // system time increased - sync should overwrite - let _bob_hash_2 = bob.hash_and_insert(key, &author, bob_value)?; - let alice_hash_2 = alice.hash_and_insert(key, &author, alice_value_2)?; - sync(&mut alice, &mut bob)?; - assert_eq!( - get_content_hash(&mut alice_store, namespace.id(), author.id(), key)?, - Some(alice_hash_2) - ); - assert_eq!( - get_content_hash(&mut alice_store, namespace.id(), author.id(), key)?, - Some(alice_hash_2) - ); - - Ok(()) - } - - #[test] - fn test_future_timestamp() -> Result<()> { - let mut rng = rand::thread_rng(); - let mut store = store::Store::memory(); - let author = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - - let mut replica = store.new_replica(namespace.clone())?; - let key = b"hi"; - let t = system_time_now(); - let record = Record::from_data(b"1", t); - let entry0 = SignedEntry::from_parts(&namespace, &author, key, record); - replica.insert_entry(entry0.clone(), InsertOrigin::Local)?; - - assert_eq!( - get_entry(&mut store, namespace.id(), author.id(), key)?, - entry0 - ); - - let mut replica = store.new_replica(namespace.clone())?; - let t = system_time_now() + MAX_TIMESTAMP_FUTURE_SHIFT - 10000; - let record = Record::from_data(b"2", t); - let entry1 = SignedEntry::from_parts(&namespace, &author, key, record); - replica.insert_entry(entry1.clone(), InsertOrigin::Local)?; - assert_eq!( - get_entry(&mut store, namespace.id(), author.id(), key)?, - entry1 - ); - - let mut replica = store.new_replica(namespace.clone())?; - let t = system_time_now() + MAX_TIMESTAMP_FUTURE_SHIFT; - let record = Record::from_data(b"2", t); - let entry2 = SignedEntry::from_parts(&namespace, &author, key, record); - replica.insert_entry(entry2.clone(), InsertOrigin::Local)?; - assert_eq!( - get_entry(&mut store, namespace.id(), author.id(), key)?, - entry2 - ); - - let mut replica = store.new_replica(namespace.clone())?; - let t = system_time_now() + MAX_TIMESTAMP_FUTURE_SHIFT + 10000; - let record = Record::from_data(b"2", t); - let entry3 = SignedEntry::from_parts(&namespace, &author, key, record); - let res = replica.insert_entry(entry3, InsertOrigin::Local); - assert!(matches!( - res, - Err(InsertError::Validation( - ValidationFailure::TooFarInTheFuture - )) - )); - assert_eq!( - get_entry(&mut store, namespace.id(), author.id(), key)?, - entry2 - ); - - Ok(()) - } - - #[test] - fn test_insert_empty() -> Result<()> { - let mut store = store::Store::memory(); - let mut rng = rand::thread_rng(); - let alice = Author::new(&mut rng); - let myspace = NamespaceSecret::new(&mut rng); - let mut replica = store.new_replica(myspace.clone())?; - let hash = Hash::new(b""); - let res = replica.insert(b"foo", &alice, hash, 0); - assert!(matches!(res, Err(InsertError::EntryIsEmpty))); - Ok(()) - } - - #[test] - fn test_prefix_delete_memory() -> Result<()> { - let store = store::Store::memory(); - test_prefix_delete(store)?; - Ok(()) - } - - #[test] - fn test_prefix_delete_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_prefix_delete(store)?; - Ok(()) - } - - fn test_prefix_delete(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let alice = Author::new(&mut rng); - let myspace = NamespaceSecret::new(&mut rng); - let mut replica = store.new_replica(myspace.clone())?; - let hash1 = replica.hash_and_insert(b"foobar", &alice, b"hello")?; - let hash2 = replica.hash_and_insert(b"fooboo", &alice, b"world")?; - - // sanity checks - assert_eq!( - get_content_hash(&mut store, myspace.id(), alice.id(), b"foobar")?, - Some(hash1) - ); - assert_eq!( - get_content_hash(&mut store, myspace.id(), alice.id(), b"fooboo")?, - Some(hash2) - ); - - // delete - let mut replica = store.new_replica(myspace.clone())?; - let deleted = replica.delete_prefix(b"foo", &alice)?; - assert_eq!(deleted, 2); - assert_eq!( - store.get_exact(myspace.id(), alice.id(), b"foobar", false)?, - None - ); - assert_eq!( - store.get_exact(myspace.id(), alice.id(), b"fooboo", false)?, - None - ); - assert_eq!( - store.get_exact(myspace.id(), alice.id(), b"foo", false)?, - None - ); - - Ok(()) - } - - #[test] - fn test_replica_sync_delete_memory() -> Result<()> { - let alice_store = store::Store::memory(); - let bob_store = store::Store::memory(); - - test_replica_sync_delete(alice_store, bob_store) - } - - #[test] - fn test_replica_sync_delete_fs() -> Result<()> { - let alice_dbfile = tempfile::NamedTempFile::new()?; - let alice_store = store::fs::Store::persistent(alice_dbfile.path())?; - let bob_dbfile = tempfile::NamedTempFile::new()?; - let bob_store = store::fs::Store::persistent(bob_dbfile.path())?; - test_replica_sync_delete(alice_store, bob_store) - } - - fn test_replica_sync_delete(mut alice_store: Store, mut bob_store: Store) -> Result<()> { - let alice_set = ["foot"]; - let bob_set = ["fool", "foo", "fog"]; - - let mut rng = rand::thread_rng(); - let author = Author::new(&mut rng); - let myspace = NamespaceSecret::new(&mut rng); - let mut alice = alice_store.new_replica(myspace.clone())?; - for el in &alice_set { - alice.hash_and_insert(el, &author, el.as_bytes())?; - } - - let mut bob = bob_store.new_replica(myspace.clone())?; - for el in &bob_set { - bob.hash_and_insert(el, &author, el.as_bytes())?; - } - - sync(&mut alice, &mut bob)?; - - check_entries(&mut alice_store, &myspace.id(), &author, &alice_set)?; - check_entries(&mut alice_store, &myspace.id(), &author, &bob_set)?; - check_entries(&mut bob_store, &myspace.id(), &author, &alice_set)?; - check_entries(&mut bob_store, &myspace.id(), &author, &bob_set)?; - - let mut alice = alice_store.new_replica(myspace.clone())?; - let mut bob = bob_store.new_replica(myspace.clone())?; - alice.delete_prefix("foo", &author)?; - bob.hash_and_insert("fooz", &author, "fooz".as_bytes())?; - sync(&mut alice, &mut bob)?; - check_entries(&mut alice_store, &myspace.id(), &author, &["fog", "fooz"])?; - check_entries(&mut bob_store, &myspace.id(), &author, &["fog", "fooz"])?; - - Ok(()) - } - - #[test] - fn test_replica_remove_memory() -> Result<()> { - let alice_store = store::Store::memory(); - test_replica_remove(alice_store) - } - - #[test] - fn test_replica_remove_fs() -> Result<()> { - let alice_dbfile = tempfile::NamedTempFile::new()?; - let alice_store = store::fs::Store::persistent(alice_dbfile.path())?; - test_replica_remove(alice_store) - } - - fn test_replica_remove(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let namespace = NamespaceSecret::new(&mut rng); - let author = Author::new(&mut rng); - let mut replica = store.new_replica(namespace.clone())?; - - // insert entry - let hash = replica.hash_and_insert(b"foo", &author, b"bar")?; - let res = store - .get_many(namespace.id(), Query::all())? - .collect::>(); - assert_eq!(res.len(), 1); - - // remove replica - let res = store.remove_replica(&namespace.id()); - // may not remove replica while still open; - assert!(res.is_err()); - store.close_replica(namespace.id()); - store.remove_replica(&namespace.id())?; - let res = store - .get_many(namespace.id(), Query::all())? - .collect::>(); - assert_eq!(res.len(), 0); - - // may not reopen removed replica - let res = store.load_replica_info(&namespace.id()); - assert!(matches!(res, Err(OpenError::NotFound))); - - // may recreate replica - let mut replica = store.new_replica(namespace.clone())?; - replica.insert(b"foo", &author, hash, 3)?; - let res = store - .get_many(namespace.id(), Query::all())? - .collect::>(); - assert_eq!(res.len(), 1); - Ok(()) - } - - #[test] - fn test_replica_delete_edge_cases_memory() -> Result<()> { - let store = store::Store::memory(); - test_replica_delete_edge_cases(store) - } - - #[test] - fn test_replica_delete_edge_cases_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_replica_delete_edge_cases(store) - } - - fn test_replica_delete_edge_cases(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let author = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - - let edgecases = [0u8, 1u8, 255u8]; - let prefixes = [0u8, 255u8]; - let hash = Hash::new(b"foo"); - let len = 3; - for prefix in prefixes { - let mut expected = vec![]; - let mut replica = store.new_replica(namespace.clone())?; - for suffix in edgecases { - let key = [prefix, suffix].to_vec(); - expected.push(key.clone()); - replica.insert(&key, &author, hash, len)?; - } - assert_keys(&mut store, namespace.id(), expected); - let mut replica = store.new_replica(namespace.clone())?; - replica.delete_prefix([prefix], &author)?; - assert_keys(&mut store, namespace.id(), vec![]); - } - - let mut replica = store.new_replica(namespace.clone())?; - let key = vec![1u8, 0u8]; - replica.insert(key, &author, hash, len)?; - let key = vec![1u8, 1u8]; - replica.insert(key, &author, hash, len)?; - let key = vec![1u8, 2u8]; - replica.insert(key, &author, hash, len)?; - let prefix = vec![1u8, 1u8]; - replica.delete_prefix(prefix, &author)?; - assert_keys( - &mut store, - namespace.id(), - vec![vec![1u8, 0u8], vec![1u8, 2u8]], - ); - - let mut replica = store.new_replica(namespace.clone())?; - let key = vec![0u8, 255u8]; - replica.insert(key, &author, hash, len)?; - let key = vec![0u8, 0u8]; - replica.insert(key, &author, hash, len)?; - let prefix = vec![0u8]; - replica.delete_prefix(prefix, &author)?; - assert_keys( - &mut store, - namespace.id(), - vec![vec![1u8, 0u8], vec![1u8, 2u8]], - ); - Ok(()) - } - - #[test] - fn test_latest_iter_memory() -> Result<()> { - let store = store::Store::memory(); - test_latest_iter(store) - } - - #[test] - fn test_latest_iter_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_latest_iter(store) - } - - fn test_latest_iter(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let author0 = Author::new(&mut rng); - let author1 = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - let mut replica = store.new_replica(namespace.clone())?; - - replica.hash_and_insert(b"a0.1", &author0, b"hi")?; - let latest = store - .get_latest_for_each_author(namespace.id())? - .collect::>>()?; - assert_eq!(latest.len(), 1); - assert_eq!(latest[0].2, b"a0.1".to_vec()); - - let mut replica = store.new_replica(namespace.clone())?; - replica.hash_and_insert(b"a1.1", &author1, b"hi")?; - replica.hash_and_insert(b"a0.2", &author0, b"hi")?; - let latest = store - .get_latest_for_each_author(namespace.id())? - .collect::>>()?; - let mut latest_keys: Vec> = latest.iter().map(|r| r.2.to_vec()).collect(); - latest_keys.sort(); - assert_eq!(latest_keys, vec![b"a0.2".to_vec(), b"a1.1".to_vec()]); - - Ok(()) - } - - #[test] - fn test_replica_byte_keys_memory() -> Result<()> { - let store = store::Store::memory(); - - test_replica_byte_keys(store)?; - Ok(()) - } - - #[test] - fn test_replica_byte_keys_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_replica_byte_keys(store)?; - - Ok(()) - } - - fn test_replica_byte_keys(mut store: Store) -> Result<()> { - let mut rng = rand::thread_rng(); - let author = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - - let hash = Hash::new(b"foo"); - let len = 3; - - let key = vec![1u8, 0u8]; - let mut replica = store.new_replica(namespace.clone())?; - replica.insert(key, &author, hash, len)?; - assert_keys(&mut store, namespace.id(), vec![vec![1u8, 0u8]]); - let key = vec![1u8, 2u8]; - let mut replica = store.new_replica(namespace.clone())?; - replica.insert(key, &author, hash, len)?; - assert_keys( - &mut store, - namespace.id(), - vec![vec![1u8, 0u8], vec![1u8, 2u8]], - ); - - let key = vec![0u8, 255u8]; - let mut replica = store.new_replica(namespace.clone())?; - replica.insert(key, &author, hash, len)?; - assert_keys( - &mut store, - namespace.id(), - vec![vec![1u8, 0u8], vec![1u8, 2u8], vec![0u8, 255u8]], - ); - Ok(()) - } - - #[test] - fn test_replica_capability_memory() -> Result<()> { - let store = store::Store::memory(); - test_replica_capability(store) - } - - #[test] - fn test_replica_capability_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_replica_capability(store) - } - - #[allow(clippy::redundant_pattern_matching)] - fn test_replica_capability(mut store: Store) -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let author = store.new_author(&mut rng)?; - let namespace = NamespaceSecret::new(&mut rng); - - // import read capability - insert must fail - let capability = Capability::Read(namespace.id()); - store.import_namespace(capability)?; - let mut replica = store.open_replica(&namespace.id())?; - let res = replica.hash_and_insert(b"foo", &author, b"bar"); - assert!(matches!(res, Err(InsertError::ReadOnly))); - - // import write capability - insert must succeed - let capability = Capability::Write(namespace.clone()); - store.import_namespace(capability)?; - let mut replica = store.open_replica(&namespace.id())?; - let res = replica.hash_and_insert(b"foo", &author, b"bar"); - assert!(matches!(res, Ok(_))); - store.close_replica(namespace.id()); - let mut replica = store.open_replica(&namespace.id())?; - let res = replica.hash_and_insert(b"foo", &author, b"bar"); - assert!(res.is_ok()); - - // import read capability again - insert must still succeed - let capability = Capability::Read(namespace.id()); - store.import_namespace(capability)?; - store.close_replica(namespace.id()); - let mut replica = store.open_replica(&namespace.id())?; - let res = replica.hash_and_insert(b"foo", &author, b"bar"); - assert!(res.is_ok()); - Ok(()) - } - - #[tokio::test] - async fn test_actor_capability_memory() -> Result<()> { - let store = store::Store::memory(); - test_actor_capability(store).await - } - - #[tokio::test] - async fn test_actor_capability_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_actor_capability(store).await - } - - async fn test_actor_capability(store: Store) -> Result<()> { - // test with actor - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let author = Author::new(&mut rng); - let handle = SyncHandle::spawn(store, None, "test".into()); - let author = handle.import_author(author).await?; - let namespace = NamespaceSecret::new(&mut rng); - let id = namespace.id(); - - // import read capability - insert must fail - let capability = Capability::Read(namespace.id()); - handle.import_namespace(capability).await?; - handle.open(namespace.id(), Default::default()).await?; - let res = handle - .insert_local(id, author, b"foo".to_vec().into(), Hash::new(b"bar"), 3) - .await; - assert!(res.is_err()); - - // import write capability - insert must succeed - let capability = Capability::Write(namespace.clone()); - handle.import_namespace(capability).await?; - let res = handle - .insert_local(id, author, b"foo".to_vec().into(), Hash::new(b"bar"), 3) - .await; - assert!(res.is_ok()); - - // close and reopen - must still succeed - handle.close(namespace.id()).await?; - let res = handle - .insert_local(id, author, b"foo".to_vec().into(), Hash::new(b"bar"), 3) - .await; - assert!(res.is_err()); - handle.open(namespace.id(), Default::default()).await?; - let res = handle - .insert_local(id, author, b"foo".to_vec().into(), Hash::new(b"bar"), 3) - .await; - assert!(res.is_ok()); - Ok(()) - } - - fn drain(events: async_channel::Receiver) -> Vec { - let mut res = vec![]; - while let Ok(ev) = events.try_recv() { - res.push(ev); - } - res - } - - /// This tests that no events are emitted for entries received during sync which are obsolete - /// (too old) by the time they are actually inserted in the store. - #[test] - fn test_replica_no_wrong_remote_insert_events() -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let mut store1 = store::Store::memory(); - let mut store2 = store::Store::memory(); - let peer1 = [1u8; 32]; - let peer2 = [2u8; 32]; - let mut state1 = SyncOutcome::default(); - let mut state2 = SyncOutcome::default(); - - let author = Author::new(&mut rng); - let namespace = NamespaceSecret::new(&mut rng); - let mut replica1 = store1.new_replica(namespace.clone())?; - let mut replica2 = store2.new_replica(namespace.clone())?; - - let (events1_sender, events1) = async_channel::bounded(32); - let (events2_sender, events2) = async_channel::bounded(32); - - replica1.info.subscribe(events1_sender); - replica2.info.subscribe(events2_sender); - - replica1.hash_and_insert(b"foo", &author, b"init")?; - - let from1 = replica1.sync_initial_message()?; - let from2 = replica2 - .sync_process_message(from1, peer1, &mut state2) - .unwrap() - .unwrap(); - let from1 = replica1 - .sync_process_message(from2, peer2, &mut state1) - .unwrap() - .unwrap(); - // now we will receive the entry from rpelica1. we will insert a newer entry now, while the - // sync is already running. this means the entry from replica1 will be rejected. we make - // sure that no InsertRemote event is emitted for this entry. - replica2.hash_and_insert(b"foo", &author, b"update")?; - let from2 = replica2 - .sync_process_message(from1, peer1, &mut state2) - .unwrap(); - assert!(from2.is_none()); - let events1 = drain(events1); - let events2 = drain(events2); - assert_eq!(events1.len(), 1); - assert_eq!(events2.len(), 1); - assert!(matches!(events1[0], Event::LocalInsert { .. })); - assert!(matches!(events2[0], Event::LocalInsert { .. })); - assert_eq!(state1.num_sent, 1); - assert_eq!(state1.num_recv, 0); - assert_eq!(state2.num_sent, 0); - assert_eq!(state2.num_recv, 1); - - Ok(()) - } - - #[test] - fn test_replica_queries_mem() -> Result<()> { - let store = store::Store::memory(); - - test_replica_queries(store)?; - Ok(()) - } - - #[test] - fn test_replica_queries_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let store = store::fs::Store::persistent(dbfile.path())?; - test_replica_queries(store)?; - - Ok(()) - } - - fn test_replica_queries(mut store: Store) -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let namespace = NamespaceSecret::new(&mut rng); - let namespace_id = namespace.id(); - - let a1 = store.new_author(&mut rng)?; - let a2 = store.new_author(&mut rng)?; - let a3 = store.new_author(&mut rng)?; - println!( - "a1 {} a2 {} a3 {}", - a1.id().fmt_short(), - a2.id().fmt_short(), - a3.id().fmt_short() - ); - - let mut replica = store.new_replica(namespace.clone())?; - replica.hash_and_insert("hi/world", &a2, "a2")?; - replica.hash_and_insert("hi/world", &a1, "a1")?; - replica.hash_and_insert("hi/moon", &a2, "a1")?; - replica.hash_and_insert("hi", &a3, "a3")?; - - struct QueryTester<'a> { - store: &'a mut Store, - namespace: NamespaceId, - } - impl<'a> QueryTester<'a> { - fn assert(&mut self, query: impl Into, expected: Vec<(&'static str, &Author)>) { - let query = query.into(); - let actual = self - .store - .get_many(self.namespace, query.clone()) - .unwrap() - .map(|e| e.map(|e| (String::from_utf8(e.key().to_vec()).unwrap(), e.author()))) - .collect::>>() - .unwrap(); - let expected = expected - .into_iter() - .map(|(key, author)| (key.to_string(), author.id())) - .collect::>(); - assert_eq!(actual, expected, "query: {query:#?}") - } - } - - let mut qt = QueryTester { - store: &mut store, - namespace: namespace_id, - }; - - qt.assert( - Query::all(), - vec![ - ("hi/world", &a1), - ("hi/moon", &a2), - ("hi/world", &a2), - ("hi", &a3), - ], - ); - - qt.assert( - Query::single_latest_per_key(), - vec![("hi", &a3), ("hi/moon", &a2), ("hi/world", &a1)], - ); - - qt.assert( - Query::single_latest_per_key().sort_direction(SortDirection::Desc), - vec![("hi/world", &a1), ("hi/moon", &a2), ("hi", &a3)], - ); - - qt.assert( - Query::single_latest_per_key().key_prefix("hi/"), - vec![("hi/moon", &a2), ("hi/world", &a1)], - ); - - qt.assert( - Query::single_latest_per_key() - .key_prefix("hi/") - .sort_direction(SortDirection::Desc), - vec![("hi/world", &a1), ("hi/moon", &a2)], - ); - - qt.assert( - Query::all().sort_by(SortBy::KeyAuthor, SortDirection::Asc), - vec![ - ("hi", &a3), - ("hi/moon", &a2), - ("hi/world", &a1), - ("hi/world", &a2), - ], - ); - - qt.assert( - Query::all().sort_by(SortBy::KeyAuthor, SortDirection::Desc), - vec![ - ("hi/world", &a2), - ("hi/world", &a1), - ("hi/moon", &a2), - ("hi", &a3), - ], - ); - - qt.assert( - Query::all().key_prefix("hi/"), - vec![("hi/world", &a1), ("hi/moon", &a2), ("hi/world", &a2)], - ); - - qt.assert( - Query::all().key_prefix("hi/").offset(1).limit(1), - vec![("hi/moon", &a2)], - ); - - qt.assert( - Query::all() - .key_prefix("hi/") - .sort_by(SortBy::KeyAuthor, SortDirection::Desc), - vec![("hi/world", &a2), ("hi/world", &a1), ("hi/moon", &a2)], - ); - - qt.assert( - Query::all() - .key_prefix("hi/") - .sort_by(SortBy::KeyAuthor, SortDirection::Desc) - .offset(1) - .limit(1), - vec![("hi/world", &a1)], - ); - - qt.assert( - Query::all() - .key_prefix("hi/") - .sort_by(SortBy::AuthorKey, SortDirection::Asc), - vec![("hi/world", &a1), ("hi/moon", &a2), ("hi/world", &a2)], - ); - - qt.assert( - Query::all() - .key_prefix("hi/") - .sort_by(SortBy::AuthorKey, SortDirection::Desc), - vec![("hi/world", &a2), ("hi/moon", &a2), ("hi/world", &a1)], - ); - - qt.assert( - Query::all() - .sort_by(SortBy::KeyAuthor, SortDirection::Asc) - .limit(2) - .offset(1), - vec![("hi/moon", &a2), ("hi/world", &a1)], - ); - - let mut replica = store.new_replica(namespace)?; - replica.delete_prefix("hi/world", &a2)?; - let mut qt = QueryTester { - store: &mut store, - namespace: namespace_id, - }; - - qt.assert( - Query::all(), - vec![("hi/world", &a1), ("hi/moon", &a2), ("hi", &a3)], - ); - - qt.assert( - Query::all().include_empty(), - vec![ - ("hi/world", &a1), - ("hi/moon", &a2), - ("hi/world", &a2), - ("hi", &a3), - ], - ); - - Ok(()) - } - - #[test] - fn test_dl_policies_mem() -> Result<()> { - let mut store = store::Store::memory(); - test_dl_policies(&mut store) - } - - #[test] - fn test_dl_policies_fs() -> Result<()> { - let dbfile = tempfile::NamedTempFile::new()?; - let mut store = store::fs::Store::persistent(dbfile.path())?; - test_dl_policies(&mut store) - } - - fn test_dl_policies(store: &mut Store) -> Result<()> { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let namespace = NamespaceSecret::new(&mut rng); - let id = namespace.id(); - - let filter = store::FilterKind::Exact("foo".into()); - let policy = store::DownloadPolicy::NothingExcept(vec![filter]); - store - .set_download_policy(&id, policy.clone()) - .expect_err("document dos not exist"); - - // now create the document - store.new_replica(namespace)?; - - store.set_download_policy(&id, policy.clone())?; - let retrieved_policy = store.get_download_policy(&id)?; - assert_eq!(retrieved_policy, policy); - Ok(()) - } - - fn assert_keys(store: &mut Store, namespace: NamespaceId, mut expected: Vec>) { - expected.sort(); - assert_eq!(expected, get_keys_sorted(store, namespace)); - } - - fn get_keys_sorted(store: &mut Store, namespace: NamespaceId) -> Vec> { - let mut res = store - .get_many(namespace, Query::all()) - .unwrap() - .map(|e| e.map(|e| e.key().to_vec())) - .collect::>>() - .unwrap(); - res.sort(); - res - } - - fn get_entry( - store: &mut Store, - namespace: NamespaceId, - author: AuthorId, - key: &[u8], - ) -> anyhow::Result { - let entry = store - .get_exact(namespace, author, key, true)? - .ok_or_else(|| anyhow::anyhow!("not found"))?; - Ok(entry) - } - - fn get_content_hash( - store: &mut Store, - namespace: NamespaceId, - author: AuthorId, - key: &[u8], - ) -> anyhow::Result> { - let hash = store - .get_exact(namespace, author, key, false)? - .map(|e| e.content_hash()); - Ok(hash) - } - - fn sync(alice: &mut Replica, bob: &mut Replica) -> Result<(SyncOutcome, SyncOutcome)> { - let alice_peer_id = [1u8; 32]; - let bob_peer_id = [2u8; 32]; - let mut alice_state = SyncOutcome::default(); - let mut bob_state = SyncOutcome::default(); - // Sync alice - bob - let mut next_to_bob = Some(alice.sync_initial_message()?); - let mut rounds = 0; - while let Some(msg) = next_to_bob.take() { - assert!(rounds < 100, "too many rounds"); - rounds += 1; - println!("round {}", rounds); - if let Some(msg) = bob.sync_process_message(msg, alice_peer_id, &mut bob_state)? { - next_to_bob = alice.sync_process_message(msg, bob_peer_id, &mut alice_state)? - } - } - assert_eq!(alice_state.num_sent, bob_state.num_recv); - assert_eq!(alice_state.num_recv, bob_state.num_sent); - Ok((alice_state, bob_state)) - } - - fn check_entries( - store: &mut Store, - namespace: &NamespaceId, - author: &Author, - set: &[&str], - ) -> Result<()> { - for el in set { - store.get_exact(*namespace, author.id(), el, false)?; - } - Ok(()) - } -} diff --git a/iroh-docs/src/ticket.rs b/iroh-docs/src/ticket.rs deleted file mode 100644 index 0af620e49ca..00000000000 --- a/iroh-docs/src/ticket.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! Tickets for [`iroh-docs`] documents. - -use iroh_base::ticket; -use iroh_net::NodeAddr; -use serde::{Deserialize, Serialize}; - -use crate::Capability; - -/// Contains both a key (either secret or public) to a document, and a list of peers to join. -#[derive(Serialize, Deserialize, Clone, Debug, derive_more::Display)] -#[display("{}", ticket::Ticket::serialize(self))] -pub struct DocTicket { - /// either a public or private key - pub capability: Capability, - /// A list of nodes to contact. - pub nodes: Vec, -} - -/// Wire format for [`DocTicket`]. -/// -/// In the future we might have multiple variants (not versions, since they -/// might be both equally valid), so this is a single variant enum to force -/// postcard to add a discriminator. -#[derive(Serialize, Deserialize)] -enum TicketWireFormat { - Variant0(DocTicket), -} - -impl ticket::Ticket for DocTicket { - const KIND: &'static str = "doc"; - - fn to_bytes(&self) -> Vec { - let data = TicketWireFormat::Variant0(self.clone()); - postcard::to_stdvec(&data).expect("postcard serialization failed") - } - - fn from_bytes(bytes: &[u8]) -> Result { - let res: TicketWireFormat = postcard::from_bytes(bytes).map_err(ticket::Error::Postcard)?; - let TicketWireFormat::Variant0(res) = res; - if res.nodes.is_empty() { - return Err(ticket::Error::Verify("addressing info cannot be empty")); - } - Ok(res) - } -} - -impl DocTicket { - /// Create a new doc ticket - pub fn new(capability: Capability, peers: Vec) -> Self { - Self { - capability, - nodes: peers, - } - } -} - -impl std::str::FromStr for DocTicket { - type Err = ticket::Error; - fn from_str(s: &str) -> Result { - ticket::Ticket::deserialize(s) - } -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use iroh_base::base32; - use iroh_net::key::PublicKey; - use iroh_test::{assert_eq_hex, hexdump::parse_hexdump}; - - use super::*; - use crate::NamespaceId; - - #[test] - fn test_ticket_base32() { - let node_id = - PublicKey::from_str("ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6") - .unwrap(); - let namespace_id = NamespaceId::from( - &<[u8; 32]>::try_from( - hex::decode("ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6") - .unwrap(), - ) - .unwrap(), - ); - - let ticket = DocTicket { - capability: Capability::Read(namespace_id), - nodes: vec![NodeAddr::from_parts(node_id, None, [])], - }; - let base32 = base32::parse_vec(ticket.to_string().strip_prefix("doc").unwrap()).unwrap(); - let expected = parse_hexdump(" - 00 # variant - 01 # capability discriminator, 1 = read - ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6 # namespace id, 32 bytes, see above - 01 # one node - ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6 # node id, 32 bytes, see above - 00 # no relay url - 00 # no direct addresses - ").unwrap(); - assert_eq_hex!(base32, expected); - } -} diff --git a/iroh-gossip/Cargo.toml b/iroh-gossip/Cargo.toml deleted file mode 100644 index 50fe4846c27..00000000000 --- a/iroh-gossip/Cargo.toml +++ /dev/null @@ -1,67 +0,0 @@ -[package] -name = "iroh-gossip" -version = "0.27.0" -edition = "2021" -readme = "README.md" -description = "gossip messages over broadcast trees" -license = "MIT/Apache-2.0" -authors = ["n0 team"] -repository = "https://github.com/n0-computer/iroh" - -# Sadly this also needs to be updated in .github/workflows/ci.yml -rust-version = "1.76" - -[lints] -workspace = true - -[dependencies] -anyhow = { version = "1" } -async-channel = { version = "2.3.1", optional = true } -blake3 = { package = "iroh-blake3", version = "1.4.5"} -bytes = { version = "1.7", features = ["serde"] } -derive_more = { version = "1.0.0", features = ["add", "debug", "deref", "display", "from", "try_into", "into"] } -ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } -indexmap = "2.0" -iroh-base = { version = "0.27.0", path = "../iroh-base" } -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics" } -postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -rand = { version = "0.8.5", features = ["std_rng"] } -rand_core = "0.6.4" -serde = { version = "1.0.164", features = ["derive"] } - -# net dependencies (optional) -futures-lite = { version = "2.3", optional = true } -futures-concurrency = { version = "7.6.1", optional = true } -futures-util = { version = "0.3.30", optional = true } -iroh-net = { path = "../iroh-net", version = "0.27.0", optional = true, default-features = false } -tokio = { version = "1", optional = true, features = ["io-util", "sync", "rt", "macros", "net", "fs"] } -tokio-util = { version = "0.7.12", optional = true, features = ["codec", "rt"] } -tracing = "0.1" - -[dev-dependencies] -clap = { version = "4", features = ["derive"] } -iroh-net = { path = "../iroh-net", version = "0.27.0", default-features = false, features = ["test-utils"] } -iroh-test = { path = "../iroh-test" } -rand_chacha = "0.3.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -url = "2.4.0" - -[features] -default = ["net"] -net = [ - "dep:futures-lite", - "dep:iroh-net", - "dep:tokio", - "dep:tokio-util", - "dep:async-channel", - "dep:futures-util", - "dep:futures-concurrency" -] - -[[example]] -name = "chat" -required-features = ["net"] - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "iroh_docsrs"] diff --git a/iroh-gossip/README.md b/iroh-gossip/README.md deleted file mode 100644 index 40da90d54f4..00000000000 --- a/iroh-gossip/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# iroh-gossip - -This crate implements the `iroh-gossip` protocol. -It is based on *epidemic broadcast trees* to disseminate messages among a swarm of peers interested in a *topic*. -The implementation is based on the papers [HyParView](https://asc.di.fct.unl.pt/~jleitao/pdf/dsn07-leitao.pdf) and [PlumTree](https://asc.di.fct.unl.pt/~jleitao/pdf/srds07-leitao.pdf). - -The crate is made up from two modules: -The `proto` module is the protocol implementation, as a state machine without any IO. -The `net` module connects the protocol to the networking stack from `iroh-net`. - -The `net` module is optional behind the `net` feature flag (enabled by default). - - -# License - -This project is licensed under either of - - * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or - http://www.apache.org/licenses/LICENSE-2.0) - * MIT license ([LICENSE-MIT](LICENSE-MIT) or - http://opensource.org/licenses/MIT) - -at your option. - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in this project by you, as defined in the Apache-2.0 license, -shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-gossip/examples/chat.rs b/iroh-gossip/examples/chat.rs deleted file mode 100644 index 5d99ba10e6c..00000000000 --- a/iroh-gossip/examples/chat.rs +++ /dev/null @@ -1,325 +0,0 @@ -use std::{ - collections::HashMap, - fmt, - net::{Ipv4Addr, SocketAddrV4}, - str::FromStr, -}; - -use anyhow::{bail, Context, Result}; -use bytes::Bytes; -use clap::Parser; -use ed25519_dalek::Signature; -use futures_lite::StreamExt; -use iroh_base::base32; -use iroh_gossip::{ - net::{Event, Gossip, GossipEvent, GossipReceiver, GOSSIP_ALPN}, - proto::TopicId, -}; -use iroh_net::{ - key::{PublicKey, SecretKey}, - relay::{RelayMap, RelayMode, RelayUrl}, - Endpoint, NodeAddr, -}; -use serde::{Deserialize, Serialize}; -use tracing::warn; - -/// Chat over iroh-gossip -/// -/// This broadcasts signed messages over iroh-gossip and verifies signatures -/// on received messages. -/// -/// By default a new node id is created when starting the example. To reuse your identity, -/// set the `--secret-key` flag with the secret key printed on a previous invocation. -/// -/// By default, the relay server run by n0 is used. To use a local relay server, run -/// cargo run --bin iroh-relay --features iroh-relay -- --dev -/// in another terminal and then set the `-d http://localhost:3340` flag on this example. -#[derive(Parser, Debug)] -struct Args { - /// secret key to derive our node id from. - #[clap(long)] - secret_key: Option, - /// Set a custom relay server. By default, the relay server hosted by n0 will be used. - #[clap(short, long)] - relay: Option, - /// Disable relay completely. - #[clap(long)] - no_relay: bool, - /// Set your nickname. - #[clap(short, long)] - name: Option, - /// Set the bind port for our socket. By default, a random port will be used. - #[clap(short, long, default_value = "0")] - bind_port: u16, - #[clap(subcommand)] - command: Command, -} - -#[derive(Parser, Debug)] -enum Command { - /// Open a chat room for a topic and print a ticket for others to join. - /// - /// If no topic is provided, a new topic will be created. - Open { - /// Optionally set the topic id (32 bytes, as base32 string). - topic: Option, - }, - /// Join a chat room from a ticket. - Join { - /// The ticket, as base32 string. - ticket: String, - }, -} - -#[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt::init(); - let args = Args::parse(); - - // parse the cli command - let (topic, peers) = match &args.command { - Command::Open { topic } => { - let topic = topic.unwrap_or_else(|| TopicId::from_bytes(rand::random())); - println!("> opening chat room for topic {topic}"); - (topic, vec![]) - } - Command::Join { ticket } => { - let Ticket { topic, peers } = Ticket::from_str(ticket)?; - println!("> joining chat room for topic {topic}"); - (topic, peers) - } - }; - - // parse or generate our secret key - let secret_key = match args.secret_key { - None => SecretKey::generate(), - Some(key) => key.parse()?, - }; - println!("> our secret key: {secret_key}"); - - // configure our relay map - let relay_mode = match (args.no_relay, args.relay) { - (false, None) => RelayMode::Default, - (false, Some(url)) => RelayMode::Custom(RelayMap::from_url(url)), - (true, None) => RelayMode::Disabled, - (true, Some(_)) => bail!("You cannot set --no-relay and --relay at the same time"), - }; - println!("> using relay servers: {}", fmt_relay_mode(&relay_mode)); - - // build our magic endpoint - let endpoint = Endpoint::builder() - .secret_key(secret_key) - .alpns(vec![GOSSIP_ALPN.to_vec()]) - .relay_mode(relay_mode) - .bind_addr_v4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, args.bind_port)) - .bind() - .await?; - println!("> our node id: {}", endpoint.node_id()); - - let my_addr = endpoint.node_addr().await?; - // create the gossip protocol - let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default(), &my_addr.info); - - // print a ticket that includes our own node id and endpoint addresses - let ticket = { - let me = endpoint.node_addr().await?; - let peers = peers.iter().cloned().chain([me]).collect(); - Ticket { topic, peers } - }; - println!("> ticket to join us: {ticket}"); - - // spawn our endpoint loop that forwards incoming connections to the gossiper - tokio::spawn(endpoint_loop(endpoint.clone(), gossip.clone())); - - // join the gossip topic by connecting to known peers, if any - let peer_ids = peers.iter().map(|p| p.node_id).collect(); - if peers.is_empty() { - println!("> waiting for peers to join us..."); - } else { - println!("> trying to connect to {} peers...", peers.len()); - // add the peer addrs from the ticket to our endpoint's addressbook so that they can be dialed - for peer in peers.into_iter() { - endpoint.add_node_addr(peer)?; - } - }; - let (sender, receiver) = gossip.join(topic, peer_ids).await?.split(); - println!("> connected!"); - - // broadcast our name, if set - if let Some(name) = args.name { - let message = Message::AboutMe { name }; - let encoded_message = SignedMessage::sign_and_encode(endpoint.secret_key(), &message)?; - sender.broadcast(encoded_message).await?; - } - - // subscribe and print loop - tokio::spawn(subscribe_loop(receiver)); - - // spawn an input thread that reads stdin - // not using tokio here because they recommend this for "technical reasons" - let (line_tx, mut line_rx) = tokio::sync::mpsc::channel(1); - std::thread::spawn(move || input_loop(line_tx)); - - // broadcast each line we type - println!("> type a message and hit enter to broadcast..."); - while let Some(text) = line_rx.recv().await { - let message = Message::Message { text: text.clone() }; - let encoded_message = SignedMessage::sign_and_encode(endpoint.secret_key(), &message)?; - sender.broadcast(encoded_message).await?; - println!("> sent: {text}"); - } - - Ok(()) -} - -async fn subscribe_loop(mut receiver: GossipReceiver) -> Result<()> { - // init a peerid -> name hashmap - let mut names = HashMap::new(); - while let Some(event) = receiver.try_next().await? { - if let Event::Gossip(GossipEvent::Received(msg)) = event { - let (from, message) = SignedMessage::verify_and_decode(&msg.content)?; - match message { - Message::AboutMe { name } => { - names.insert(from, name.clone()); - println!("> {} is now known as {}", from.fmt_short(), name); - } - Message::Message { text } => { - let name = names - .get(&from) - .map_or_else(|| from.fmt_short(), String::to_string); - println!("{}: {}", name, text); - } - } - } - } - Ok(()) -} - -async fn endpoint_loop(endpoint: Endpoint, gossip: Gossip) { - while let Some(incoming) = endpoint.accept().await { - let conn = match incoming.accept() { - Ok(conn) => conn, - Err(err) => { - warn!("incoming connection failed: {err:#}"); - // we can carry on in these cases: - // this can be caused by retransmitted datagrams - continue; - } - }; - let gossip = gossip.clone(); - tokio::spawn(async move { - if let Err(err) = handle_connection(conn, gossip).await { - println!("> connection closed: {err}"); - } - }); - } -} - -async fn handle_connection( - mut conn: iroh_net::endpoint::Connecting, - gossip: Gossip, -) -> anyhow::Result<()> { - let alpn = conn.alpn().await?; - let conn = conn.await?; - let peer_id = iroh_net::endpoint::get_remote_node_id(&conn)?; - match alpn.as_ref() { - GOSSIP_ALPN => gossip.handle_connection(conn).await.context(format!( - "connection to {peer_id} with ALPN {} failed", - String::from_utf8_lossy(&alpn) - ))?, - _ => println!("> ignoring connection from {peer_id}: unsupported ALPN protocol"), - } - Ok(()) -} - -fn input_loop(line_tx: tokio::sync::mpsc::Sender) -> Result<()> { - let mut buffer = String::new(); - let stdin = std::io::stdin(); // We get `Stdin` here. - loop { - stdin.read_line(&mut buffer)?; - line_tx.blocking_send(buffer.clone())?; - buffer.clear(); - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct SignedMessage { - from: PublicKey, - data: Bytes, - signature: Signature, -} - -impl SignedMessage { - pub fn verify_and_decode(bytes: &[u8]) -> Result<(PublicKey, Message)> { - let signed_message: Self = postcard::from_bytes(bytes)?; - let key: PublicKey = signed_message.from; - key.verify(&signed_message.data, &signed_message.signature)?; - let message: Message = postcard::from_bytes(&signed_message.data)?; - Ok((signed_message.from, message)) - } - - pub fn sign_and_encode(secret_key: &SecretKey, message: &Message) -> Result { - let data: Bytes = postcard::to_stdvec(&message)?.into(); - let signature = secret_key.sign(&data); - let from: PublicKey = secret_key.public(); - let signed_message = Self { - from, - data, - signature, - }; - let encoded = postcard::to_stdvec(&signed_message)?; - Ok(encoded.into()) - } -} - -#[derive(Debug, Serialize, Deserialize)] -enum Message { - AboutMe { name: String }, - Message { text: String }, -} - -#[derive(Debug, Serialize, Deserialize)] -struct Ticket { - topic: TopicId, - peers: Vec, -} -impl Ticket { - /// Deserializes from bytes. - fn from_bytes(bytes: &[u8]) -> Result { - postcard::from_bytes(bytes).map_err(Into::into) - } - /// Serializes to bytes. - pub fn to_bytes(&self) -> Vec { - postcard::to_stdvec(self).expect("postcard::to_stdvec is infallible") - } -} - -/// Serializes to base32. -impl fmt::Display for Ticket { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", base32::fmt(self.to_bytes())) - } -} - -/// Deserializes from base32. -impl FromStr for Ticket { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - Self::from_bytes(&base32::parse_vec(s)?) - } -} - -// helpers - -fn fmt_relay_mode(relay_mode: &RelayMode) -> String { - match relay_mode { - RelayMode::Disabled => "None".to_string(), - RelayMode::Default => "Default Relay (production) servers".to_string(), - RelayMode::Staging => "Default Relay (staging) servers".to_string(), - RelayMode::Custom(map) => map - .urls() - .map(|url| url.to_string()) - .collect::>() - .join(", "), - } -} diff --git a/iroh-gossip/src/lib.rs b/iroh-gossip/src/lib.rs deleted file mode 100644 index 1db6ce71cc2..00000000000 --- a/iroh-gossip/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Broadcast messages to peers subscribed to a topic -//! -//! The crate is designed to be used from the [iroh] crate, which provides a -//! [high level interface](https://docs.rs/iroh/latest/iroh/client/gossip/index.html), -//! but can also be used standalone. -//! -//! [iroh]: https://docs.rs/iroh -#![deny(missing_docs, rustdoc::broken_intra_doc_links)] -#![cfg_attr(iroh_docsrs, feature(doc_cfg))] - -pub mod metrics; -#[cfg(feature = "net")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "net")))] -pub mod net; -pub mod proto; diff --git a/iroh-gossip/src/metrics.rs b/iroh-gossip/src/metrics.rs deleted file mode 100644 index 0de9680eb23..00000000000 --- a/iroh-gossip/src/metrics.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Metrics for iroh-gossip - -use iroh_metrics::{ - core::{Counter, Metric}, - struct_iterable::Iterable, -}; - -/// Enum of metrics for the module -#[allow(missing_docs)] -#[derive(Debug, Clone, Iterable)] -pub struct Metrics { - pub msgs_ctrl_sent: Counter, - pub msgs_ctrl_recv: Counter, - pub msgs_data_sent: Counter, - pub msgs_data_recv: Counter, - pub msgs_data_sent_size: Counter, - pub msgs_data_recv_size: Counter, - pub msgs_ctrl_sent_size: Counter, - pub msgs_ctrl_recv_size: Counter, - pub neighbor_up: Counter, - pub neighbor_down: Counter, - pub actor_tick_main: Counter, - pub actor_tick_rx: Counter, - pub actor_tick_endpoint: Counter, - pub actor_tick_dialer: Counter, - pub actor_tick_dialer_success: Counter, - pub actor_tick_dialer_failure: Counter, - pub actor_tick_in_event_rx: Counter, - pub actor_tick_timers: Counter, -} - -impl Default for Metrics { - fn default() -> Self { - Self { - msgs_ctrl_sent: Counter::new("Number of control messages sent"), - msgs_ctrl_recv: Counter::new("Number of control messages received"), - msgs_data_sent: Counter::new("Number of data messages sent"), - msgs_data_recv: Counter::new("Number of data messages received"), - msgs_data_sent_size: Counter::new("Total size of all data messages sent"), - msgs_data_recv_size: Counter::new("Total size of all data messages received"), - msgs_ctrl_sent_size: Counter::new("Total size of all control messages sent"), - msgs_ctrl_recv_size: Counter::new("Total size of all control messages received"), - neighbor_up: Counter::new("Number of times we connected to a peer"), - neighbor_down: Counter::new("Number of times we disconnected from a peer"), - actor_tick_main: Counter::new("Number of times the main actor loop ticked"), - actor_tick_rx: Counter::new("Number of times the actor ticked for a message received"), - actor_tick_endpoint: Counter::new( - "Number of times the actor ticked for an endpoint event", - ), - actor_tick_dialer: Counter::new("Number of times the actor ticked for a dialer event"), - actor_tick_dialer_success: Counter::new( - "Number of times the actor ticked for a successful dialer event", - ), - actor_tick_dialer_failure: Counter::new( - "Number of times the actor ticked for a failed dialer event", - ), - actor_tick_in_event_rx: Counter::new( - "Number of times the actor ticked for an incoming event", - ), - actor_tick_timers: Counter::new("Number of times the actor ticked for a timer event"), - } - } -} - -impl Metric for Metrics { - fn name() -> &'static str { - "gossip" - } -} diff --git a/iroh-gossip/src/net.rs b/iroh-gossip/src/net.rs deleted file mode 100644 index 02799609c6d..00000000000 --- a/iroh-gossip/src/net.rs +++ /dev/null @@ -1,1008 +0,0 @@ -//! Networking for the `iroh-gossip` protocol - -use std::{ - collections::{BTreeSet, HashMap, HashSet, VecDeque}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, - time::Instant, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::BytesMut; -use futures_concurrency::{ - future::TryJoin, - stream::{stream_group, StreamGroup}, -}; -use futures_lite::{stream::Stream, StreamExt}; -use futures_util::TryFutureExt; -use iroh_metrics::inc; -use iroh_net::{ - dialer::Dialer, - endpoint::{get_remote_node_id, Connection, DirectAddr}, - key::PublicKey, - AddrInfo, Endpoint, NodeAddr, NodeId, -}; -use rand::rngs::StdRng; -use rand_core::SeedableRng; -use tokio::{sync::mpsc, task::JoinSet}; -use tokio_util::task::AbortOnDropHandle; -use tracing::{debug, error_span, trace, warn, Instrument}; - -use self::util::{read_message, write_message, Timers}; -use crate::{ - metrics::Metrics, - proto::{self, PeerData, Scope, TopicId}, -}; - -mod handles; -pub mod util; - -pub use self::handles::{ - Command, CommandStream, Event, GossipEvent, GossipReceiver, GossipSender, GossipTopic, - JoinOptions, Message, -}; - -/// ALPN protocol name -pub const GOSSIP_ALPN: &[u8] = b"/iroh-gossip/0"; -/// Default channel capacity for topic subscription channels (one per topic) -const TOPIC_EVENTS_DEFAULT_CAP: usize = 2048; -/// Default channel capacity for topic subscription channels (one per topic) -const TOPIC_COMMANDS_DEFAULT_CAP: usize = 2048; -/// Channel capacity for the send queue (one per connection) -const SEND_QUEUE_CAP: usize = 64; -/// Channel capacity for the ToActor message queue (single) -const TO_ACTOR_CAP: usize = 64; -/// Channel capacity for the InEvent message queue (single) -const IN_EVENT_CAP: usize = 1024; -/// Name used for logging when new node addresses are added from gossip. -const SOURCE_NAME: &str = "gossip"; - -/// Events emitted from the gossip protocol -pub type ProtoEvent = proto::Event; -/// Commands for the gossip protocol -pub type ProtoCommand = proto::Command; - -type InEvent = proto::InEvent; -type OutEvent = proto::OutEvent; -type Timer = proto::Timer; -type ProtoMessage = proto::Message; - -/// Publish and subscribe on gossiping topics. -/// -/// Each topic is a separate broadcast tree with separate memberships. -/// -/// A topic has to be joined before you can publish or subscribe on the topic. -/// To join the swarm for a topic, you have to know the [`PublicKey`] of at least one peer that also joined the topic. -/// -/// Messages published on the swarm will be delivered to all peers that joined the swarm for that -/// topic. You will also be relaying (gossiping) messages published by other peers. -/// -/// With the default settings, the protocol will maintain up to 5 peer connections per topic. -/// -/// Even though the [`Gossip`] is created from a [`Endpoint`], it does not accept connections -/// itself. You should run an accept loop on the [`Endpoint`] yourself, check the ALPN protocol of incoming -/// connections, and if the ALPN protocol equals [`GOSSIP_ALPN`], forward the connection to the -/// gossip actor through [Self::handle_connection]. -/// -/// The gossip actor will, however, initiate new connections to other peers by itself. -#[derive(Debug, Clone)] -pub struct Gossip { - to_actor_tx: mpsc::Sender, - _actor_handle: Arc>, - max_message_size: usize, -} - -impl Gossip { - /// Spawn a gossip actor and get a handle for it - pub fn from_endpoint(endpoint: Endpoint, config: proto::Config, my_addr: &AddrInfo) -> Self { - let peer_id = endpoint.node_id(); - let dialer = Dialer::new(endpoint.clone()); - let state = proto::State::new( - peer_id, - encode_peer_data(my_addr).unwrap(), - config, - rand::rngs::StdRng::from_entropy(), - ); - let (to_actor_tx, to_actor_rx) = mpsc::channel(TO_ACTOR_CAP); - let (in_event_tx, in_event_rx) = mpsc::channel(IN_EVENT_CAP); - - let me = endpoint.node_id().fmt_short(); - let max_message_size = state.max_message_size(); - let actor = Actor { - endpoint, - state, - dialer, - to_actor_rx, - in_event_rx, - in_event_tx, - timers: Timers::new(), - command_rx: StreamGroup::new().keyed(), - peers: Default::default(), - topics: Default::default(), - quit_queue: Default::default(), - connection_tasks: Default::default(), - }; - - let actor_handle = tokio::spawn( - async move { - if let Err(err) = actor.run().await { - warn!("gossip actor closed with error: {err:?}"); - } - } - .instrument(error_span!("gossip", %me)), - ); - Self { - to_actor_tx, - _actor_handle: Arc::new(AbortOnDropHandle::new(actor_handle)), - max_message_size, - } - } - - /// Get the maximum message size configured for this gossip actor. - pub fn max_message_size(&self) -> usize { - self.max_message_size - } - - /// Handle an incoming [`Connection`]. - /// - /// Make sure to check the ALPN protocol yourself before passing the connection. - pub async fn handle_connection(&self, conn: Connection) -> anyhow::Result<()> { - let peer_id = get_remote_node_id(&conn)?; - self.send(ToActor::HandleConnection(peer_id, ConnOrigin::Accept, conn)) - .await?; - Ok(()) - } - - /// Join a gossip topic with the default options and wait for at least one active connection. - pub async fn join(&self, topic_id: TopicId, bootstrap: Vec) -> Result { - let mut sub = self.join_with_opts(topic_id, JoinOptions::with_bootstrap(bootstrap)); - sub.joined().await?; - Ok(sub) - } - - /// Join a gossip topic with options. - /// - /// Returns a [`GossipTopic`] instantly. To wait for at least one connection to be established, - /// you can await [`GossipTopic::joined`]. - /// - /// Messages will be queued until a first connection is available. If the internal channel becomes full, - /// the oldest messages will be dropped from the channel. - pub fn join_with_opts(&self, topic_id: TopicId, opts: JoinOptions) -> GossipTopic { - let (command_tx, command_rx) = async_channel::bounded(TOPIC_COMMANDS_DEFAULT_CAP); - let command_rx: CommandStream = Box::pin(command_rx); - let event_rx = self.join_with_stream(topic_id, opts, command_rx); - GossipTopic::new(command_tx, Box::pin(event_rx)) - } - - /// Join a gossip topic with options and an externally-created update stream. - /// - /// This method differs from [`Self::join_with_opts`] by letting you pass in a `updates` command stream yourself - /// instead of using a channel created for you. - /// - /// It returns a stream of events. If you want to wait for the topic to become active, wait for - /// the [`GossipEvent::Joined`] event. - pub fn join_with_stream( - &self, - topic_id: TopicId, - options: JoinOptions, - updates: CommandStream, - ) -> impl Stream> + Send + 'static { - let (event_tx, event_rx) = async_channel::bounded(options.subscription_capacity); - let to_actor_tx = self.to_actor_tx.clone(); - let channels = SubscriberChannels { - command_rx: updates, - event_tx, - }; - // We spawn a task to send the subscribe action to the actor, because we want the send to - // succeed even if the returned stream is dropped right away without being polled, because - // it is legit to keep only the `updates` stream and drop the event stream. This situation - // is handled fine within the actor, but we have to make sure that the message reaches the - // actor. - let task = tokio::task::spawn(async move { - to_actor_tx - .send(ToActor::Join { - topic_id, - bootstrap: options.bootstrap, - channels, - }) - .await - .map_err(|_| anyhow!("Gossip actor dropped")) - }); - async move { - task.await - .map_err(|err| anyhow!("Task for sending to gossip actor failed: {err:?}"))??; - Ok(event_rx) - } - .try_flatten_stream() - } - - async fn send(&self, event: ToActor) -> anyhow::Result<()> { - self.to_actor_tx - .send(event) - .await - .map_err(|_| anyhow!("gossip actor dropped")) - } -} - -/// Input messages for the gossip [`Actor`]. -#[derive(derive_more::Debug)] -enum ToActor { - /// Handle a new QUIC connection, either from accept (external to the actor) or from connect - /// (happens internally in the actor). - HandleConnection(PublicKey, ConnOrigin, #[debug("Connection")] Connection), - Join { - topic_id: TopicId, - bootstrap: BTreeSet, - channels: SubscriberChannels, - }, -} - -/// Actor that sends and handles messages between the connection and main state loops -struct Actor { - /// Protocol state - state: proto::State, - /// The endpoint through which we dial peers - endpoint: Endpoint, - /// Dial machine to connect to peers - dialer: Dialer, - /// Input messages to the actor - to_actor_rx: mpsc::Receiver, - /// Sender for the state input (cloned into the connection loops) - in_event_tx: mpsc::Sender, - /// Input events to the state (emitted from the connection loops) - in_event_rx: mpsc::Receiver, - /// Queued timers - timers: Timers, - /// Map of topics to their state. - topics: HashMap, - /// Map of peers to their state. - peers: HashMap, - /// Stream of commands from topic handles. - command_rx: stream_group::Keyed, - /// Internal queue of topic to close because all handles were dropped. - quit_queue: VecDeque, - /// Tasks for the connection loops, to keep track of panics. - connection_tasks: JoinSet<()>, -} - -impl Actor { - pub async fn run(mut self) -> anyhow::Result<()> { - // Watch for changes in direct addresses to update our peer data. - let mut direct_addresses_stream = self.endpoint.direct_addresses(); - // Watch for changes of our home relay to update our peer data. - let mut home_relay_stream = self.endpoint.watch_home_relay(); - - // With each gossip message we provide addressing information to reach our node. - // We wait until at least one direct address is discovered. - let mut current_addresses = direct_addresses_stream - .next() - .await - .ok_or_else(|| anyhow!("Failed to discover direct addresses"))?; - let peer_data = our_peer_data(&self.endpoint, ¤t_addresses)?; - self.handle_in_event(InEvent::UpdatePeerData(peer_data), Instant::now()) - .await?; - - let mut i = 0; - loop { - i += 1; - trace!(?i, "tick"); - inc!(Metrics, actor_tick_main); - tokio::select! { - biased; - msg = self.to_actor_rx.recv() => { - trace!(?i, "tick: to_actor_rx"); - inc!(Metrics, actor_tick_rx); - match msg { - Some(msg) => self.handle_to_actor_msg(msg, Instant::now()).await?, - None => { - debug!("all gossip handles dropped, stop gossip actor"); - break; - } - } - }, - Some((key, (topic, command))) = self.command_rx.next(), if !self.command_rx.is_empty() => { - trace!(?i, "tick: command_rx"); - self.handle_command(topic, key, command).await?; - }, - Some(new_addresses) = direct_addresses_stream.next() => { - trace!(?i, "tick: new_endpoints"); - inc!(Metrics, actor_tick_endpoint); - current_addresses = new_addresses; - let peer_data = our_peer_data(&self.endpoint, ¤t_addresses)?; - self.handle_in_event(InEvent::UpdatePeerData(peer_data), Instant::now()).await?; - } - Some(_relay_url) = home_relay_stream.next() => { - let peer_data = our_peer_data(&self.endpoint, ¤t_addresses)?; - self.handle_in_event(InEvent::UpdatePeerData(peer_data), Instant::now()).await?; - } - (peer_id, res) = self.dialer.next_conn() => { - trace!(?i, "tick: dialer"); - inc!(Metrics, actor_tick_dialer); - match res { - Ok(conn) => { - debug!(peer = ?peer_id, "dial successful"); - inc!(Metrics, actor_tick_dialer_success); - self.handle_connection(peer_id, ConnOrigin::Dial, conn); - } - Err(err) => { - warn!(peer = ?peer_id, "dial failed: {err}"); - inc!(Metrics, actor_tick_dialer_failure); - } - } - } - event = self.in_event_rx.recv() => { - trace!(?i, "tick: in_event_rx"); - inc!(Metrics, actor_tick_in_event_rx); - match event { - Some(event) => { - self.handle_in_event(event, Instant::now()).await.context("in_event_rx.recv -> handle_in_event")?; - } - None => unreachable!() - } - } - drain = self.timers.wait_and_drain() => { - trace!(?i, "tick: timers"); - inc!(Metrics, actor_tick_timers); - let now = Instant::now(); - for (_instant, timer) in drain { - self.handle_in_event(InEvent::TimerExpired(timer), now).await.context("timers.drain_expired -> handle_in_event")?; - } - } - Some(res) = self.connection_tasks.join_next(), if !self.connection_tasks.is_empty() => { - if let Err(err) = res { - if !err.is_cancelled() { - warn!("connection task panicked: {err:?}"); - } - } - } - } - } - Ok(()) - } - - async fn handle_command( - &mut self, - topic: TopicId, - key: stream_group::Key, - command: Option, - ) -> anyhow::Result<()> { - debug!(?topic, ?key, ?command, "handle command"); - let Some(state) = self.topics.get_mut(&topic) else { - // TODO: unreachable? - warn!("received command for unknown topic"); - return Ok(()); - }; - let TopicState { - command_rx_keys, - event_senders, - .. - } = state; - match command { - Some(command) => { - let command = match command { - Command::Broadcast(message) => ProtoCommand::Broadcast(message, Scope::Swarm), - Command::BroadcastNeighbors(message) => { - ProtoCommand::Broadcast(message, Scope::Neighbors) - } - Command::JoinPeers(peers) => ProtoCommand::Join(peers), - }; - self.handle_in_event(proto::InEvent::Command(topic, command), Instant::now()) - .await?; - } - None => { - command_rx_keys.remove(&key); - if command_rx_keys.is_empty() && event_senders.is_empty() { - self.quit_queue.push_back(topic); - self.process_quit_queue().await?; - } - } - } - Ok(()) - } - - fn handle_connection(&mut self, peer_id: NodeId, origin: ConnOrigin, conn: Connection) { - // Check that we only keep one connection per peer per direction. - if let Some(peer_info) = self.peers.get(&peer_id) { - if matches!(origin, ConnOrigin::Dial) && peer_info.conn_dialed.is_some() { - warn!(?peer_id, ?origin, "ignoring connection: already accepted"); - return; - } - if matches!(origin, ConnOrigin::Accept) && peer_info.conn_accepted.is_some() { - warn!(?peer_id, ?origin, "ignoring connection: already accepted"); - return; - } - } - - let mut peer_info = self.peers.remove(&peer_id).unwrap_or_default(); - - // Store the connection so that we can terminate it when the peer is removed. - match origin { - ConnOrigin::Dial => { - peer_info.conn_dialed = Some(conn.clone()); - } - ConnOrigin::Accept => { - peer_info.conn_accepted = Some(conn.clone()); - } - } - - // Extract the queue of pending messages. - let queue = match &mut peer_info.state { - PeerState::Pending { queue } => std::mem::take(queue), - PeerState::Active { .. } => Default::default(), - }; - - let (send_tx, send_rx) = mpsc::channel(SEND_QUEUE_CAP); - let max_message_size = self.state.max_message_size(); - let in_event_tx = self.in_event_tx.clone(); - - // Spawn a task for this connection - self.connection_tasks.spawn( - async move { - match connection_loop( - peer_id, - conn, - origin, - send_rx, - &in_event_tx, - max_message_size, - queue, - ) - .await - { - Ok(()) => debug!("connection closed without error"), - Err(err) => warn!("connection closed: {err:?}"), - } - in_event_tx - .send(InEvent::PeerDisconnected(peer_id)) - .await - .ok(); - } - .instrument(error_span!("gossip_conn", peer = %peer_id.fmt_short())), - ); - - peer_info.state = match peer_info.state { - PeerState::Pending { .. } => PeerState::Active { send_tx }, - PeerState::Active { send_tx } => PeerState::Active { send_tx }, - }; - - self.peers.insert(peer_id, peer_info); - } - - async fn handle_to_actor_msg(&mut self, msg: ToActor, now: Instant) -> anyhow::Result<()> { - trace!("handle to_actor {msg:?}"); - match msg { - ToActor::HandleConnection(peer_id, origin, conn) => { - self.handle_connection(peer_id, origin, conn) - } - ToActor::Join { - topic_id, - bootstrap, - channels, - } => { - let state = self.topics.entry(topic_id).or_default(); - let TopicState { - neighbors, - event_senders, - command_rx_keys, - joined, - } = state; - if *joined { - let neighbors = neighbors.iter().copied().collect(); - channels - .event_tx - .try_send(Ok(Event::Gossip(GossipEvent::Joined(neighbors)))) - .ok(); - } - - event_senders.push(channels.event_tx); - let command_rx = TopicCommandStream::new(topic_id, channels.command_rx); - let key = self.command_rx.insert(command_rx); - command_rx_keys.insert(key); - - self.handle_in_event( - InEvent::Command( - topic_id, - ProtoCommand::Join(bootstrap.into_iter().collect()), - ), - now, - ) - .await?; - } - } - Ok(()) - } - - async fn handle_in_event(&mut self, event: InEvent, now: Instant) -> anyhow::Result<()> { - self.handle_in_event_inner(event, now).await?; - self.process_quit_queue().await?; - Ok(()) - } - - async fn process_quit_queue(&mut self) -> anyhow::Result<()> { - while let Some(topic_id) = self.quit_queue.pop_front() { - self.handle_in_event_inner( - InEvent::Command(topic_id, ProtoCommand::Quit), - Instant::now(), - ) - .await?; - self.topics.remove(&topic_id); - } - Ok(()) - } - - async fn handle_in_event_inner(&mut self, event: InEvent, now: Instant) -> anyhow::Result<()> { - if matches!(event, InEvent::TimerExpired(_)) { - trace!(?event, "handle in_event"); - } else { - debug!(?event, "handle in_event"); - }; - if let InEvent::PeerDisconnected(peer) = &event { - self.peers.remove(peer); - } - let out = self.state.handle(event, now); - for event in out { - if matches!(event, OutEvent::ScheduleTimer(_, _)) { - trace!(?event, "handle out_event"); - } else { - debug!(?event, "handle out_event"); - }; - match event { - OutEvent::SendMessage(peer_id, message) => { - let info = self.peers.entry(peer_id).or_default(); - match &mut info.state { - PeerState::Active { send_tx } => { - if let Err(_err) = send_tx.send(message).await { - // Removing the peer is handled by the in_event PeerDisconnected sent - // at the end of the connection task. - warn!("connection loop for {peer_id:?} dropped"); - } - } - PeerState::Pending { queue } => { - if queue.is_empty() { - self.dialer.queue_dial(peer_id, GOSSIP_ALPN); - } - queue.push(message); - } - } - } - OutEvent::EmitEvent(topic_id, event) => { - let Some(state) = self.topics.get_mut(&topic_id) else { - // TODO: unreachable? - warn!(?topic_id, "gossip state emitted event for unknown topic"); - continue; - }; - let TopicState { - joined, - neighbors, - event_senders, - command_rx_keys, - } = state; - let event = if let ProtoEvent::NeighborUp(neighbor) = event { - neighbors.insert(neighbor); - if !*joined { - *joined = true; - GossipEvent::Joined(vec![neighbor]) - } else { - GossipEvent::NeighborUp(neighbor) - } - } else { - event.into() - }; - event_senders.send(&event); - if event_senders.is_empty() && command_rx_keys.is_empty() { - self.quit_queue.push_back(topic_id); - } - } - OutEvent::ScheduleTimer(delay, timer) => { - self.timers.insert(now + delay, timer); - } - OutEvent::DisconnectPeer(peer_id) => { - if let Some(peer) = self.peers.remove(&peer_id) { - if let Some(conn) = peer.conn_dialed { - conn.close(0u8.into(), b"close from disconnect"); - } - if let Some(conn) = peer.conn_accepted { - conn.close(0u8.into(), b"close from disconnect"); - } - drop(peer.state); - } - } - OutEvent::PeerData(node_id, data) => match decode_peer_data(&data) { - Err(err) => warn!("Failed to decode {data:?} from {node_id}: {err}"), - Ok(info) => { - debug!(peer = ?node_id, "add known addrs: {info:?}"); - let node_addr = NodeAddr { node_id, info }; - if let Err(err) = self - .endpoint - .add_node_addr_with_source(node_addr, SOURCE_NAME) - { - debug!(peer = ?node_id, "add known failed: {err:?}"); - } - } - }, - } - } - Ok(()) - } -} - -#[derive(Debug, Default)] -struct PeerInfo { - state: PeerState, - conn_dialed: Option, - conn_accepted: Option, -} - -#[derive(Debug)] -enum PeerState { - Pending { queue: Vec }, - Active { send_tx: mpsc::Sender }, -} - -impl Default for PeerState { - fn default() -> Self { - PeerState::Pending { queue: Vec::new() } - } -} - -#[derive(Debug, Default)] -struct TopicState { - joined: bool, - neighbors: BTreeSet, - event_senders: EventSenders, - command_rx_keys: HashSet, -} - -/// Whether a connection is initiated by us (Dial) or by the remote peer (Accept) -#[derive(Debug, Clone, Copy)] -enum ConnOrigin { - Accept, - Dial, -} -#[derive(derive_more::Debug)] -struct SubscriberChannels { - event_tx: async_channel::Sender>, - #[debug("CommandStream")] - command_rx: CommandStream, -} - -async fn connection_loop( - from: PublicKey, - conn: Connection, - origin: ConnOrigin, - mut send_rx: mpsc::Receiver, - in_event_tx: &mpsc::Sender, - max_message_size: usize, - queue: Vec, -) -> anyhow::Result<()> { - let (mut send, mut recv) = match origin { - ConnOrigin::Accept => conn.accept_bi().await?, - ConnOrigin::Dial => conn.open_bi().await?, - }; - debug!("connection established"); - let mut send_buf = BytesMut::new(); - let mut recv_buf = BytesMut::new(); - - let send_loop = async { - for msg in queue { - write_message(&mut send, &mut send_buf, &msg, max_message_size).await? - } - while let Some(msg) = send_rx.recv().await { - write_message(&mut send, &mut send_buf, &msg, max_message_size).await? - } - Ok::<_, anyhow::Error>(()) - }; - - let recv_loop = async { - loop { - let msg = read_message(&mut recv, &mut recv_buf, max_message_size).await?; - match msg { - None => break, - Some(msg) => in_event_tx.send(InEvent::RecvMessage(from, msg)).await?, - } - } - Ok::<_, anyhow::Error>(()) - }; - - (send_loop, recv_loop).try_join().await?; - - Ok(()) -} - -fn encode_peer_data(info: &AddrInfo) -> anyhow::Result { - let bytes = postcard::to_stdvec(info)?; - anyhow::ensure!(!bytes.is_empty(), "encoding empty peer data: {:?}", info); - Ok(PeerData::new(bytes)) -} - -fn decode_peer_data(peer_data: &PeerData) -> anyhow::Result { - let bytes = peer_data.as_bytes(); - if bytes.is_empty() { - return Ok(AddrInfo::default()); - } - let info = postcard::from_bytes(bytes)?; - Ok(info) -} - -#[derive(Debug, Default)] -struct EventSenders { - senders: Vec<(async_channel::Sender>, bool)>, -} - -impl EventSenders { - fn is_empty(&self) -> bool { - self.senders.is_empty() - } - - fn push(&mut self, sender: async_channel::Sender>) { - self.senders.push((sender, false)); - } - - /// Send an event to all subscribers. - /// - /// This will not wait until the sink is full, but send a `Lagged` response if the sink is almost full. - fn send(&mut self, event: &GossipEvent) { - self.senders.retain_mut(|(send, lagged)| { - // If the stream is disconnected, we don't need to send to it. - if send.is_closed() { - return false; - } - - // Check if the send buffer is almost full, and send a lagged response if it is. - let cap = send.capacity().expect("we only use bounded channels"); - let event = if send.len() >= cap - 1 { - if *lagged { - return true; - } - *lagged = true; - Event::Lagged - } else { - *lagged = false; - Event::Gossip(event.clone()) - }; - match send.try_send(Ok(event)) { - Ok(()) => true, - Err(async_channel::TrySendError::Full(_)) => true, - Err(async_channel::TrySendError::Closed(_)) => false, - } - }) - } -} - -#[derive(derive_more::Debug)] -struct TopicCommandStream { - topic_id: TopicId, - #[debug("CommandStream")] - stream: CommandStream, - closed: bool, -} - -impl TopicCommandStream { - fn new(topic_id: TopicId, stream: CommandStream) -> Self { - Self { - topic_id, - stream, - closed: false, - } - } -} - -impl Stream for TopicCommandStream { - type Item = (TopicId, Option); - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.closed { - return Poll::Ready(None); - } - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Ready(Some(item)) => Poll::Ready(Some((self.topic_id, Some(item)))), - Poll::Ready(None) => { - self.closed = true; - Poll::Ready(Some((self.topic_id, None))) - } - Poll::Pending => Poll::Pending, - } - } -} - -fn our_peer_data(endpoint: &Endpoint, direct_addresses: &BTreeSet) -> Result { - let addr = NodeAddr::from_parts( - endpoint.node_id(), - endpoint.home_relay(), - direct_addresses.iter().map(|x| x.addr), - ); - encode_peer_data(&addr.info) -} - -#[cfg(test)] -mod test { - use std::time::Duration; - - use bytes::Bytes; - use futures_concurrency::future::TryJoin; - use iroh_net::{ - key::SecretKey, - relay::{RelayMap, RelayMode}, - }; - use tokio::{spawn, time::timeout}; - use tokio_util::sync::CancellationToken; - use tracing::info; - - use super::*; - - async fn create_endpoint( - rng: &mut rand_chacha::ChaCha12Rng, - relay_map: RelayMap, - ) -> anyhow::Result { - let ep = Endpoint::builder() - .secret_key(SecretKey::generate_with_rng(rng)) - .alpns(vec![GOSSIP_ALPN.to_vec()]) - .relay_mode(RelayMode::Custom(relay_map)) - .insecure_skip_relay_cert_verify(true) - .bind() - .await?; - - ep.watch_home_relay().next().await; - Ok(ep) - } - - async fn endpoint_loop( - endpoint: Endpoint, - gossip: Gossip, - cancel: CancellationToken, - ) -> anyhow::Result<()> { - loop { - tokio::select! { - biased; - _ = cancel.cancelled() => break, - incoming = endpoint.accept() => match incoming { - None => break, - Some(incoming) => { - let connecting = match incoming.accept() { - Ok(connecting) => connecting, - Err(err) => { - warn!("incoming connection failed: {err:#}"); - // we can carry on in these cases: - // this can be caused by retransmitted datagrams - continue; - } - }; - gossip.handle_connection(connecting.await?).await? - } - } - } - } - Ok(()) - } - - #[tokio::test] - async fn gossip_net_smoke() { - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); - let _guard = iroh_test::logging::setup(); - let (relay_map, relay_url, _guard) = - iroh_net::test_utils::run_relay_server().await.unwrap(); - - let ep1 = create_endpoint(&mut rng, relay_map.clone()).await.unwrap(); - let ep2 = create_endpoint(&mut rng, relay_map.clone()).await.unwrap(); - let ep3 = create_endpoint(&mut rng, relay_map.clone()).await.unwrap(); - let addr1 = AddrInfo { - relay_url: Some(relay_url.clone()), - direct_addresses: Default::default(), - }; - let addr2 = AddrInfo { - relay_url: Some(relay_url.clone()), - direct_addresses: Default::default(), - }; - let addr3 = AddrInfo { - relay_url: Some(relay_url.clone()), - direct_addresses: Default::default(), - }; - - let go1 = Gossip::from_endpoint(ep1.clone(), Default::default(), &addr1); - let go2 = Gossip::from_endpoint(ep2.clone(), Default::default(), &addr2); - let go3 = Gossip::from_endpoint(ep3.clone(), Default::default(), &addr3); - debug!("peer1 {:?}", ep1.node_id()); - debug!("peer2 {:?}", ep2.node_id()); - debug!("peer3 {:?}", ep3.node_id()); - let pi1 = ep1.node_id(); - let pi2 = ep2.node_id(); - - let cancel = CancellationToken::new(); - let tasks = [ - spawn(endpoint_loop(ep1.clone(), go1.clone(), cancel.clone())), - spawn(endpoint_loop(ep2.clone(), go2.clone(), cancel.clone())), - spawn(endpoint_loop(ep3.clone(), go3.clone(), cancel.clone())), - ]; - - debug!("----- adding peers ----- "); - let topic: TopicId = blake3::hash(b"foobar").into(); - - let addr1 = NodeAddr::new(pi1).with_relay_url(relay_url.clone()); - let addr2 = NodeAddr::new(pi2).with_relay_url(relay_url); - ep2.add_node_addr(addr1.clone()).unwrap(); - ep3.add_node_addr(addr2).unwrap(); - - debug!("----- joining ----- "); - // join the topics and wait for the connection to succeed - let [sub1, mut sub2, mut sub3] = [ - go1.join(topic, vec![]), - go2.join(topic, vec![pi1]), - go3.join(topic, vec![pi2]), - ] - .try_join() - .await - .unwrap(); - - let (sink1, _stream1) = sub1.split(); - - let len = 2; - - // publish messages on node1 - let pub1 = spawn(async move { - for i in 0..len { - let message = format!("hi{}", i); - info!("go1 broadcast: {message:?}"); - sink1.broadcast(message.into_bytes().into()).await.unwrap(); - tokio::time::sleep(Duration::from_micros(1)).await; - } - }); - - // wait for messages on node2 - let sub2 = spawn(async move { - let mut recv = vec![]; - loop { - let ev = sub2.next().await.unwrap().unwrap(); - info!("go2 event: {ev:?}"); - if let Event::Gossip(GossipEvent::Received(msg)) = ev { - recv.push(msg.content); - } - if recv.len() == len { - return recv; - } - } - }); - - // wait for messages on node3 - let sub3 = spawn(async move { - let mut recv = vec![]; - loop { - let ev = sub3.next().await.unwrap().unwrap(); - info!("go3 event: {ev:?}"); - if let Event::Gossip(GossipEvent::Received(msg)) = ev { - recv.push(msg.content); - } - if recv.len() == len { - return recv; - } - } - }); - - timeout(Duration::from_secs(10), pub1) - .await - .unwrap() - .unwrap(); - let recv2 = timeout(Duration::from_secs(10), sub2) - .await - .unwrap() - .unwrap(); - let recv3 = timeout(Duration::from_secs(10), sub3) - .await - .unwrap() - .unwrap(); - - let expected: Vec = (0..len) - .map(|i| Bytes::from(format!("hi{i}").into_bytes())) - .collect(); - assert_eq!(recv2, expected); - assert_eq!(recv3, expected); - - cancel.cancel(); - for t in tasks { - timeout(Duration::from_secs(10), t) - .await - .unwrap() - .unwrap() - .unwrap(); - } - } -} diff --git a/iroh-gossip/src/net/handles.rs b/iroh-gossip/src/net/handles.rs deleted file mode 100644 index c944805afa0..00000000000 --- a/iroh-gossip/src/net/handles.rs +++ /dev/null @@ -1,276 +0,0 @@ -//! Topic handles for sending and receiving on a gossip topic. -//! -//! These are returned from [`super::Gossip`]. - -use std::{ - collections::{BTreeSet, HashSet}, - pin::Pin, - task::{Context, Poll}, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use iroh_net::NodeId; -use serde::{Deserialize, Serialize}; - -use crate::{net::TOPIC_EVENTS_DEFAULT_CAP, proto::DeliveryScope}; - -/// Sender for a gossip topic. -#[derive(Debug)] -pub struct GossipSender(async_channel::Sender); - -impl GossipSender { - pub(crate) fn new(sender: async_channel::Sender) -> Self { - Self(sender) - } - - /// Broadcast a message to all nodes. - pub async fn broadcast(&self, message: Bytes) -> anyhow::Result<()> { - self.0 - .send(Command::Broadcast(message)) - .await - .map_err(|_| anyhow!("Gossip actor dropped")) - } - - /// Broadcast a message to our direct neighbors. - pub async fn broadcast_neighbors(&self, message: Bytes) -> anyhow::Result<()> { - self.0 - .send(Command::BroadcastNeighbors(message)) - .await - .map_err(|_| anyhow!("Gossip actor dropped")) - } - - /// Join a set of peers. - pub async fn join_peers(&self, peers: Vec) -> anyhow::Result<()> { - self.0 - .send(Command::JoinPeers(peers)) - .await - .map_err(|_| anyhow!("Gossip actor dropped")) - } -} - -type EventStream = Pin> + Send + 'static>>; - -/// Subscribed gossip topic. -/// -/// This handle is a [`Stream`] of [`Event`]s from the topic, and can be used to send messages. -/// -/// It may be split into sender and receiver parts with [`Self::split`]. -#[derive(Debug)] -pub struct GossipTopic { - sender: GossipSender, - receiver: GossipReceiver, -} - -impl GossipTopic { - pub(crate) fn new(sender: async_channel::Sender, receiver: EventStream) -> Self { - Self { - sender: GossipSender::new(sender), - receiver: GossipReceiver::new(Box::pin(receiver)), - } - } - - /// Splits `self` into [`GossipSender`] and [`GossipReceiver`] parts. - pub fn split(self) -> (GossipSender, GossipReceiver) { - (self.sender, self.receiver) - } - - /// Sends a message to all peers. - pub async fn broadcast(&self, message: Bytes) -> anyhow::Result<()> { - self.sender.broadcast(message).await - } - - /// Sends a message to our direct neighbors in the swarm. - pub async fn broadcast_neighbors(&self, message: Bytes) -> anyhow::Result<()> { - self.sender.broadcast_neighbors(message).await - } - - /// Waits until we are connected to at least one node. - pub async fn joined(&mut self) -> Result<()> { - self.receiver.joined().await - } - - /// Returns true if we are connected to at least one node. - pub fn is_joined(&self) -> bool { - self.receiver.is_joined() - } -} - -impl Stream for GossipTopic { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.receiver).poll_next(cx) - } -} - -/// Receiver for gossip events on a topic. -/// -/// This is a [`Stream`] of [`Event`]s emitted from the topic. -#[derive(derive_more::Debug)] -pub struct GossipReceiver { - #[debug("EventStream")] - stream: EventStream, - neighbors: HashSet, - joined: bool, -} - -impl GossipReceiver { - pub(crate) fn new(events_rx: EventStream) -> Self { - Self { - stream: events_rx, - neighbors: Default::default(), - joined: false, - } - } - - /// Lists our current direct neighbors. - pub fn neighbors(&self) -> impl Iterator + '_ { - self.neighbors.iter().copied() - } - - /// Waits until we are connected to at least one node. - /// - /// This progresses the stream until we received [`GossipEvent::Joined`], which is the first - /// item emitted on the stream. - /// - /// Note that this consumes the [`GossipEvent::Joined`] event. If you want to act on these - /// initial neighbors, use [`Self::neighbors`] after awaiting [`Self::joined`]. - pub async fn joined(&mut self) -> Result<()> { - if !self.joined { - match self - .try_next() - .await? - .context("Gossip receiver closed before Joined event was received.")? - { - Event::Gossip(GossipEvent::Joined(_)) => {} - _ => anyhow::bail!("Expected Joined event to be the first event received."), - } - } - Ok(()) - } - - /// Returns true if we are connected to at least one node. - pub fn is_joined(&self) -> bool { - !self.neighbors.is_empty() - } -} - -impl Stream for GossipReceiver { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let item = std::task::ready!(Pin::new(&mut self.stream).poll_next(cx)); - if let Some(Ok(item)) = &item { - match item { - Event::Gossip(GossipEvent::Joined(neighbors)) => { - self.joined = true; - self.neighbors.extend(neighbors.iter().copied()); - } - Event::Gossip(GossipEvent::NeighborUp(node_id)) => { - self.neighbors.insert(*node_id); - } - Event::Gossip(GossipEvent::NeighborDown(node_id)) => { - self.neighbors.remove(node_id); - } - _ => {} - } - } - Poll::Ready(item) - } -} - -/// Events emitted from a gossip topic with a lagging notification. -/// -/// This is the item of the [`GossipReceiver`] stream. It wraps the actual gossip events to also -/// provide a notification if we missed gossip events for the topic. -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] -pub enum Event { - /// We received an event. - Gossip(GossipEvent), - /// We missed some messages because our [`GossipReceiver`] was not progressing fast enough. - Lagged, -} - -/// Events emitted from a gossip topic. -/// -/// These are the events emitted from a [`GossipReceiver`], wrapped in [`Event::Gossip`]. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)] -pub enum GossipEvent { - /// We joined the topic with at least one peer. - /// - /// This is the first event on a [`GossipReceiver`] and will only be emitted once. - Joined(Vec), - /// We have a new, direct neighbor in the swarm membership layer for this topic. - NeighborUp(NodeId), - /// We dropped direct neighbor in the swarm membership layer for this topic. - NeighborDown(NodeId), - /// We received a gossip message for this topic. - Received(Message), -} - -impl From> for GossipEvent { - fn from(event: crate::proto::Event) -> Self { - match event { - crate::proto::Event::NeighborUp(node_id) => Self::NeighborUp(node_id), - crate::proto::Event::NeighborDown(node_id) => Self::NeighborDown(node_id), - crate::proto::Event::Received(message) => Self::Received(Message { - content: message.content, - scope: message.scope, - delivered_from: message.delivered_from, - }), - } - } -} - -/// A gossip message -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, derive_more::Debug, Serialize, Deserialize)] -pub struct Message { - /// The content of the message - #[debug("Bytes({})", self.content.len())] - pub content: Bytes, - /// The scope of the message. - /// This tells us if the message is from a direct neighbor or actual gossip. - pub scope: DeliveryScope, - /// The node that delivered the message. This is not the same as the original author. - pub delivered_from: NodeId, -} - -/// A stream of commands for a gossip subscription. -pub type CommandStream = Pin + Send + Sync + 'static>>; - -/// Send a gossip message -#[derive(Serialize, Deserialize, derive_more::Debug)] -pub enum Command { - /// Broadcast a message to all nodes in the swarm - Broadcast(#[debug("Bytes({})", _0.len())] Bytes), - /// Broadcast a message to all direct neighbors - BroadcastNeighbors(#[debug("Bytes({})", _0.len())] Bytes), - /// Connect to a set of peers - JoinPeers(Vec), -} - -/// Options for joining a gossip topic. -#[derive(Serialize, Deserialize, Debug)] -pub struct JoinOptions { - /// The initial bootstrap nodes - pub bootstrap: BTreeSet, - /// The maximum number of messages that can be buffered in a subscription. - /// - /// If this limit is reached, the subscriber will receive a `Lagged` response, - /// the message will be dropped, and the subscriber will be closed. - /// - /// This is to prevent a single slow subscriber from blocking the dispatch loop. - /// If a subscriber is lagging, it should be closed and re-opened. - pub subscription_capacity: usize, -} - -impl JoinOptions { - /// Creates [`JoinOptions`] with the provided bootstrap nodes and the default subscription - /// capacity. - pub fn with_bootstrap(nodes: impl IntoIterator) -> Self { - Self { - bootstrap: nodes.into_iter().collect(), - subscription_capacity: TOPIC_EVENTS_DEFAULT_CAP, - } - } -} diff --git a/iroh-gossip/src/net/util.rs b/iroh-gossip/src/net/util.rs deleted file mode 100644 index 5bfbee633ec..00000000000 --- a/iroh-gossip/src/net/util.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! Utilities for iroh-gossip networking - -use std::{io, pin::Pin, time::Instant}; - -use anyhow::{bail, ensure, Context, Result}; -use bytes::{Bytes, BytesMut}; -use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, - time::{sleep_until, Sleep}, -}; - -use super::ProtoMessage; -use crate::proto::util::TimerMap; - -/// Write a `ProtoMessage` as a length-prefixed, postcard-encoded message. -pub async fn write_message( - writer: &mut W, - buffer: &mut BytesMut, - frame: &ProtoMessage, - max_message_size: usize, -) -> Result<()> { - let len = postcard::experimental::serialized_size(&frame)?; - ensure!(len < max_message_size); - buffer.clear(); - buffer.resize(len, 0u8); - let slice = postcard::to_slice(&frame, buffer)?; - writer.write_u32(len as u32).await?; - writer.write_all(slice).await?; - Ok(()) -} - -/// Read a length-prefixed message and decode as `ProtoMessage`; -pub async fn read_message( - reader: impl AsyncRead + Unpin, - buffer: &mut BytesMut, - max_message_size: usize, -) -> Result> { - match read_lp(reader, buffer, max_message_size).await? { - None => Ok(None), - Some(data) => { - let message = postcard::from_bytes(&data)?; - Ok(Some(message)) - } - } -} - -/// Reads a length prefixed message. -/// -/// # Returns -/// -/// The message as raw bytes. If the end of the stream is reached and there is no partial -/// message, returns `None`. -pub async fn read_lp( - mut reader: impl AsyncRead + Unpin, - buffer: &mut BytesMut, - max_message_size: usize, -) -> Result> { - let size = match reader.read_u32().await { - Ok(size) => size, - Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err.into()), - }; - let mut reader = reader.take(size as u64); - let size = usize::try_from(size).context("frame larger than usize")?; - if size > max_message_size { - bail!("Incoming message exceeds the maximum message size of {max_message_size} bytes"); - } - buffer.reserve(size); - loop { - let r = reader.read_buf(buffer).await?; - if r == 0 { - break; - } - } - Ok(Some(buffer.split_to(size).freeze())) -} - -/// A [`TimerMap`] with an async method to wait for the next timer expiration. -#[derive(Debug)] -pub struct Timers { - next: Option<(Instant, Pin>)>, - map: TimerMap, -} - -impl Default for Timers { - fn default() -> Self { - Self { - next: None, - map: TimerMap::default(), - } - } -} - -impl Timers { - /// Create a new timer map - pub fn new() -> Self { - Self::default() - } - - /// Insert a new entry at the specified instant - pub fn insert(&mut self, instant: Instant, item: T) { - self.map.insert(instant, item); - } - - fn reset(&mut self) { - self.next = self - .map - .first() - .map(|(instant, _)| (*instant, Box::pin(sleep_until((*instant).into())))) - } - - /// Wait for the next timer to expire and return an iterator of all expired timers - /// - /// If the [TimerMap] is empty, this will return a future that is pending forever. - /// After inserting a new entry, prior futures returned from this method will not become ready. - /// They should be dropped after calling [Self::insert], and a new future as returned from - /// this method should be awaited instead. - pub async fn wait_and_drain(&mut self) -> impl Iterator { - self.reset(); - match self.next.as_mut() { - Some((instant, sleep)) => { - sleep.await; - self.map.drain_until(instant) - } - None => std::future::pending().await, - } - } -} diff --git a/iroh-gossip/src/proto.rs b/iroh-gossip/src/proto.rs deleted file mode 100644 index f87eae8364c..00000000000 --- a/iroh-gossip/src/proto.rs +++ /dev/null @@ -1,376 +0,0 @@ -//! Implementation of the iroh-gossip protocol, as an IO-less state machine -//! -//! This module implements the iroh-gossip protocol. The entry point is [`State`], which contains -//! the protocol state for a node. -//! -//! The iroh-gossip protocol is made up from two parts: A swarm membership protocol, based on -//! [HyParView][hyparview], and a gossip broadcasting protocol, based on [PlumTree][plumtree]. -//! -//! For a full explanation it is recommended to read the two papers. What follows is a brief -//! outline of the protocols. -//! -//! All protocol messages are namespaced by a [`TopicId`], a 32 byte identifier. Topics are -//! separate swarms and broadcast scopes. The HyParView and PlumTree algorithms both work in the -//! scope of a single topic. Thus, joining multiple topics increases the number of open connections -//! to peers and the size of the local routing table. -//! -//! The **membership protocol** ([HyParView][hyparview]) is a cluster protocol where each peer -//! maintains a partial view of all nodes in the swarm. -//! A peer joins the swarm for a topic by connecting to any known peer that is a member of this -//! topic's swarm. Obtaining this initial contact info happens out of band. The peer then sends -//! a `Join` message to that initial peer. All peers maintain a list of -//! `active` and `passive` peers. Active peers are those that you maintain active connections to. -//! Passive peers is an addressbook of additional peers. If one of your active peers goes offline, -//! its slot is filled with a random peer from the passive set. In the default configuration, the -//! active view has a size of 5 and the passive view a size of 30. -//! The HyParView protocol ensures that active connections are always bidirectional, and regularly -//! exchanges nodes for the passive view in a `Shuffle` operation. -//! Thus, this protocol exposes a high degree of reliability and auto-recovery in the case of node -//! failures. -//! -//! The **gossip protocol** ([PlumTree][plumtree]) builds upon the membership protocol. It exposes -//! a method to broadcast messages to all peers in the swarm. On each node, it maintains two sets -//! of peers: An `eager` set and a `lazy` set. Both are subsets of the `active` view from the -//! membership protocol. When broadcasting a message from the local node, or upon receiving a -//! broadcast message, the message is pushed to all peers in the eager set. Additionally, the hash -//! of the message (which uniquely identifies it), but not the message content, is lazily pushed -//! to all peers in the `lazy` set. When receiving such lazy pushes (called `Ihaves`), those peers -//! may request the message content after a timeout if they didn't receive the message by one of -//! their eager peers before. When requesting a message from a currently-lazy peer, this peer is -//! also upgraded to be an eager peer from that moment on. This strategy self-optimizes the -//! messaging graph by latency. Note however that this optimization will work best if the messaging -//! paths are stable, i.e. if it's always the same peer that broadcasts. If not, the relative -//! message redundancy will grow and the ideal messaging graph might change frequently. -//! -//! [hyparview]: https://asc.di.fct.unl.pt/~jleitao/pdf/dsn07-leitao.pdf -//! [plumtree]: https://asc.di.fct.unl.pt/~jleitao/pdf/srds07-leitao.pdf - -use std::{fmt, hash::Hash}; - -use bytes::Bytes; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; - -mod hyparview; -mod plumtree; -pub mod state; -pub mod topic; -pub mod util; - -#[cfg(test)] -mod tests; - -pub use plumtree::{DeliveryScope, Scope}; -pub use state::{InEvent, Message, OutEvent, State, Timer, TopicId}; -pub use topic::{Command, Config, Event, IO}; - -/// The identifier for a peer. -/// -/// The protocol implementation is generic over this trait. When implementing the protocol, -/// a concrete type must be chosen that will then be used throughout the implementation to identify -/// and index individual peers. -/// -/// Note that the concrete type will be used in protocol messages. Therefore, implementations of -/// the protocol are only compatible if the same concrete type is supplied for this trait. -/// -/// TODO: Rename to `PeerId`? It does not necessarily refer to a peer's address, as long as the -/// networking layer can translate the value of its concrete type into an address. -pub trait PeerIdentity: Hash + Eq + Copy + fmt::Debug + Serialize + DeserializeOwned {} -impl PeerIdentity for T where T: Hash + Eq + Copy + fmt::Debug + Serialize + DeserializeOwned {} - -/// Opaque binary data that is transmitted on messages that introduce new peers. -/// -/// Implementations may use these bytes to supply addresses or other information needed to connect -/// to a peer that is not included in the peer's [`PeerIdentity`]. -#[derive(derive_more::Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)] -#[debug("PeerData({}b)", self.0.len())] -pub struct PeerData(Bytes); - -impl PeerData { - /// Create a new [`PeerData`] from a byte buffer. - pub fn new(data: impl Into) -> Self { - Self(data.into()) - } - - /// Get a reference to the contained [`bytes::Bytes`]. - pub fn inner(&self) -> &bytes::Bytes { - &self.0 - } - - /// Get the peer data as a byte slice. - pub fn as_bytes(&self) -> &[u8] { - &self.0 - } -} - -/// PeerInfo contains a peer's identifier and the opaque peer data as provided by the implementer. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -struct PeerInfo { - pub id: PI, - pub data: Option, -} - -impl From<(PI, Option)> for PeerInfo { - fn from((id, data): (PI, Option)) -> Self { - Self { id, data } - } -} - -#[cfg(test)] -mod test { - - use std::{collections::HashSet, env, time::Instant}; - - use rand::SeedableRng; - - use super::{Command, Config, Event, State}; - use crate::proto::{ - tests::{ - assert_synchronous_active, report_round_distribution, sort, Network, Simulator, - SimulatorConfig, - }, - Scope, TopicId, - }; - - #[test] - fn hyparview_smoke() { - let _guard = iroh_test::logging::setup(); - // Create a network with 4 nodes and active_view_capacity 2 - let mut config = Config::default(); - config.membership.active_view_capacity = 2; - let mut network = Network::new(Instant::now()); - let rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - for i in 0..4 { - network.push(State::new( - i, - Default::default(), - config.clone(), - rng.clone(), - )); - } - - let t: TopicId = [0u8; 32].into(); - - // Do some joins between nodes 0,1,2 - network.command(0, t, Command::Join(vec![1])); - network.command(0, t, Command::Join(vec![2])); - network.command(1, t, Command::Join(vec![2])); - network.command(2, t, Command::Join(vec![])); - network.ticks(10); - - // Confirm emitted events - let actual = network.events_sorted(); - let expected = sort(vec![ - (0, t, Event::NeighborUp(1)), - (0, t, Event::NeighborUp(2)), - (1, t, Event::NeighborUp(2)), - (1, t, Event::NeighborUp(0)), - (2, t, Event::NeighborUp(0)), - (2, t, Event::NeighborUp(1)), - ]); - assert_eq!(actual, expected); - - // Confirm active connections - assert_eq!(network.conns(), vec![(0, 1), (0, 2), (1, 2)]); - - // Now let node 3 join node 0. - // Node 0 is full, so it will disconnect from either node 1 or node 2. - network.command(3, t, Command::Join(vec![0])); - network.ticks(8); - - // Confirm emitted events. There's two options because whether node 0 disconnects from - // node 1 or node 2 is random. - let actual = network.events_sorted(); - eprintln!("actual {actual:?}"); - let expected1 = sort(vec![ - (3, t, Event::NeighborUp(0)), - (0, t, Event::NeighborUp(3)), - (0, t, Event::NeighborDown(1)), - (1, t, Event::NeighborDown(0)), - ]); - let expected2 = sort(vec![ - (3, t, Event::NeighborUp(0)), - (0, t, Event::NeighborUp(3)), - (0, t, Event::NeighborDown(2)), - (2, t, Event::NeighborDown(0)), - ]); - assert!((actual == expected1) || (actual == expected2)); - - // Confirm active connections. - if actual == expected1 { - assert_eq!(network.conns(), vec![(0, 2), (0, 3), (1, 2)]); - } else { - assert_eq!(network.conns(), vec![(0, 1), (0, 3), (1, 2)]); - } - assert!(assert_synchronous_active(&network)); - } - - #[test] - fn plumtree_smoke() { - let _guard = iroh_test::logging::setup(); - let config = Config::default(); - let mut network = Network::new(Instant::now()); - let broadcast_ticks = 12; - let join_ticks = 12; - // build a network with 6 nodes - let rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - for i in 0..6 { - network.push(State::new( - i, - Default::default(), - config.clone(), - rng.clone(), - )); - } - - let t = [0u8; 32].into(); - - // let node 0 join the topic but do not connect to any peers - network.command(0, t, Command::Join(vec![])); - // connect nodes 1 and 2 to node 0 - (1..3).for_each(|i| network.command(i, t, Command::Join(vec![0]))); - // connect nodes 4 and 5 to node 3 - network.command(3, t, Command::Join(vec![])); - (4..6).for_each(|i| network.command(i, t, Command::Join(vec![3]))); - // run ticks and drain events - network.ticks(join_ticks); - let _ = network.events(); - assert!(assert_synchronous_active(&network)); - - // now broadcast a first message - network.command( - 1, - t, - Command::Broadcast(b"hi1".to_vec().into(), Scope::Swarm), - ); - network.ticks(broadcast_ticks); - let events = network.events(); - let received = events.filter(|x| matches!(x, (_, _, Event::Received(_)))); - // message should be received by two other nodes - assert_eq!(received.count(), 2); - assert!(assert_synchronous_active(&network)); - - // now connect the two sections of the swarm - network.command(2, t, Command::Join(vec![5])); - network.ticks(join_ticks); - let _ = network.events(); - report_round_distribution(&network); - - // now broadcast again - network.command( - 1, - t, - Command::Broadcast(b"hi2".to_vec().into(), Scope::Swarm), - ); - network.ticks(broadcast_ticks); - let events = network.events(); - let received = events.filter(|x| matches!(x, (_, _, Event::Received(_)))); - // message should be received by all 5 other nodes - assert_eq!(received.count(), 5); - assert!(assert_synchronous_active(&network)); - report_round_distribution(&network); - } - - #[test] - fn big_multiple_sender() { - let _guard = iroh_test::logging::setup(); - let mut gossip_config = Config::default(); - gossip_config.broadcast.optimization_threshold = (read_var("OPTIM", 7) as u16).into(); - let config = SimulatorConfig { - peers_count: read_var("PEERS", 100), - ..Default::default() - }; - let rounds = read_var("ROUNDS", 10); - let mut simulator = Simulator::new(config, gossip_config); - simulator.init(); - simulator.bootstrap(); - for i in 0..rounds { - let from = i + 1; - let message = format!("m{i}").into_bytes().into(); - simulator.gossip_round(from, message) - } - simulator.report_round_sums(); - } - - #[test] - fn big_single_sender() { - let _guard = iroh_test::logging::setup(); - let mut gossip_config = Config::default(); - gossip_config.broadcast.optimization_threshold = (read_var("OPTIM", 7) as u16).into(); - let config = SimulatorConfig { - peers_count: read_var("PEERS", 100), - ..Default::default() - }; - let rounds = read_var("ROUNDS", 10); - let mut simulator = Simulator::new(config, gossip_config); - simulator.init(); - simulator.bootstrap(); - for i in 0..rounds { - let from = 2; - let message = format!("m{i}").into_bytes().into(); - simulator.gossip_round(from, message) - } - simulator.report_round_sums(); - } - - #[test] - fn quit() { - let _guard = iroh_test::logging::setup(); - // Create a network with 4 nodes and active_view_capacity 2 - let mut config = Config::default(); - config.membership.active_view_capacity = 2; - let mut network = Network::new(Instant::now()); - let num = 4; - let rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - for i in 0..num { - network.push(State::new( - i, - Default::default(), - config.clone(), - rng.clone(), - )); - } - - let t: TopicId = [0u8; 32].into(); - - // join all nodes - network.command(0, t, Command::Join(vec![])); - network.command(1, t, Command::Join(vec![0])); - network.command(2, t, Command::Join(vec![1])); - network.command(3, t, Command::Join(vec![2])); - network.ticks(10); - - // assert all peers appear in the connections - let all_conns: HashSet = HashSet::from_iter((0..4).flat_map(|pa| { - network - .get_active(&pa, &t) - .unwrap() - .into_iter() - .flat_map(|x| x.into_iter()) - })); - assert_eq!(all_conns, HashSet::from_iter([0, 1, 2, 3])); - assert!(assert_synchronous_active(&network)); - - // let node 3 leave the swarm - network.command(3, t, Command::Quit); - network.ticks(4); - assert!(network.peer(&3).unwrap().state(&t).is_none()); - - // assert all peers without peer 3 appear in the connections - let all_conns: HashSet = HashSet::from_iter((0..num).flat_map(|pa| { - network - .get_active(&pa, &t) - .unwrap() - .into_iter() - .flat_map(|x| x.into_iter()) - })); - assert_eq!(all_conns, HashSet::from_iter([0, 1, 2])); - assert!(assert_synchronous_active(&network)); - } - - fn read_var(name: &str, default: usize) -> usize { - env::var(name) - .unwrap_or_else(|_| default.to_string()) - .parse() - .unwrap() - } -} diff --git a/iroh-gossip/src/proto/hyparview.rs b/iroh-gossip/src/proto/hyparview.rs deleted file mode 100644 index e40f7cd7176..00000000000 --- a/iroh-gossip/src/proto/hyparview.rs +++ /dev/null @@ -1,718 +0,0 @@ -//! Implementation of the HyParView membership protocol -//! -//! The implementation is based on [this paper][paper] by Joao Leitao, Jose Pereira, Luıs Rodrigues -//! and the [example implementation][impl] by Bartosz Sypytkowski -//! -//! [paper]: https://asc.di.fct.unl.pt/~jleitao/pdf/dsn07-leitao.pdf -//! [impl]: https://gist.github.com/Horusiath/84fac596101b197da0546d1697580d99 - -use std::{ - collections::{HashMap, HashSet}, - time::{Duration, Instant}, -}; - -use derive_more::{From, Sub}; -use rand::{rngs::ThreadRng, Rng}; -use serde::{Deserialize, Serialize}; -use tracing::debug; - -use super::{util::IndexSet, PeerData, PeerIdentity, PeerInfo, IO}; - -/// Input event for HyParView -#[derive(Debug)] -pub enum InEvent { - /// A [`Message`] was received from a peer. - RecvMessage(PI, Message), - /// A timer has expired. - TimerExpired(Timer), - /// A peer was disconnected on the IO layer. - PeerDisconnected(PI), - /// Send a join request to a peer. - RequestJoin(PI), - /// Update the peer data that is transmitted on join requests. - UpdatePeerData(PeerData), - /// Quit the swarm, informing peers about us leaving. - Quit, -} - -/// Output event for HyParView -#[derive(Debug)] -pub enum OutEvent { - /// Ask the IO layer to send a [`Message`] to peer `PI`. - SendMessage(PI, Message), - /// Schedule a [`Timer`]. - ScheduleTimer(Duration, Timer), - /// Ask the IO layer to close the connection to peer `PI`. - DisconnectPeer(PI), - /// Emit an [`Event`] to the application. - EmitEvent(Event), - /// New [`PeerData`] was received for peer `PI`. - PeerData(PI, PeerData), -} - -/// Event emitted by the [`State`] to the application. -#[derive(Clone, Debug)] -pub enum Event { - /// A peer was added to our set of active connections. - NeighborUp(PI), - /// A peer was removed from our set of active connections. - NeighborDown(PI), -} - -/// Kinds of timers HyParView needs to schedule. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Timer { - DoShuffle, - PendingNeighborRequest(PI), -} - -/// Messages that we can send and receive from peers within the topic. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum Message { - /// Sent to a peer if you want to join the swarm - Join(Option), - /// When receiving Join, ForwardJoin is forwarded to the peer's ActiveView to introduce the - /// new member. - ForwardJoin(ForwardJoin), - /// A shuffle request is sent occasionally to re-shuffle the PassiveView with contacts from - /// other peers. - Shuffle(Shuffle), - /// Peers reply to [`Message::Shuffle`] requests with a random peers from their active and - /// passive views. - ShuffleReply(ShuffleReply), - /// Request to add sender to an active view of recipient. If [`Neighbor::priority`] is - /// [`Priority::High`], the request cannot be denied. - Neighbor(Neighbor), - /// Request to disconnect from a peer. - /// If [`Disconnect::alive`] is true, the other peer is not shutting down, so it should be - /// added to the passive set. - /// If [`Disconnect::respond`] is true, the peer should answer the disconnect request - /// before shutting down the connection. - Disconnect(Disconnect), -} - -/// The time-to-live for this message. -/// -/// Each time a message is forwarded, the `Ttl` is decreased by 1. If the `Ttl` reaches 0, it -/// should not be forwarded further. -#[derive(From, Sub, Eq, PartialEq, Clone, Debug, Copy, Serialize, Deserialize)] -pub struct Ttl(pub u16); -impl Ttl { - pub fn expired(&self) -> bool { - *self == Ttl(0) - } - pub fn next(&self) -> Ttl { - Ttl(self.0.saturating_sub(1)) - } -} - -/// A message informing other peers that a new peer joined the swarm for this topic. -/// -/// Will be forwarded in a random walk until `ttl` reaches 0. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct ForwardJoin { - /// The peer that newly joined the swarm - peer: PeerInfo, - /// The time-to-live for this message - ttl: Ttl, -} - -/// Shuffle messages are sent occasionally to shuffle our passive view with peers from other peer's -/// active and passive views. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct Shuffle { - /// The peer that initiated the shuffle request. - origin: PI, - /// A random subset of the active and passive peers of the `origin` peer. - nodes: Vec>, - /// The time-to-live for this message. - ttl: Ttl, -} - -/// Once a shuffle messages reaches a [`Ttl`] of 0, a peer replies with a `ShuffleReply`. -/// -/// The reply is sent to the peer that initiated the shuffle and contains a subset of the active -/// and passive views of the peer at the end of the random walk. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct ShuffleReply { - /// A random subset of the active and passive peers of the peer sending the `ShuffleReply`. - nodes: Vec>, -} - -/// The priority of a `Join` message -/// -/// This is `High` if the sender does not have any active peers, and `Low` otherwise. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum Priority { - /// High priority join that may not be denied. - /// - /// A peer may only send high priority joins if it doesn't have any active peers at the moment. - High, - /// Low priority join that can be denied. - Low, -} - -/// A neighbor message is sent after adding a peer to our active view to inform them that we are -/// now neighbors. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct Neighbor { - /// The priority of the `Join` or `ForwardJoin` message that triggered this neighbor request. - priority: Priority, - /// The user data of the peer sending this message. - data: Option, -} - -/// Message sent when leaving the swarm or closing down to inform peers about us being gone. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct Disconnect { - /// Whether we are actually shutting down or closing the connection only because our limits are - /// reached. - alive: Alive, - /// Whether we should reply to the peer with a Disconnect message. - respond: Respond, -} - -/// Configuration for the swarm membership layer -#[derive(Clone, Debug)] -pub struct Config { - /// Number of peers to which active connections are maintained - pub active_view_capacity: usize, - /// Number of peers for which contact information is remembered, - /// but to which we are not actively connected to. - pub passive_view_capacity: usize, - /// Number of hops a `ForwardJoin` message is propagated until the new peer's info - /// is added to a peer's active view. - pub active_random_walk_length: Ttl, - /// Number of hops a `ForwardJoin` message is propagated until the new peer's info - /// is added to a peer's passive view. - pub passive_random_walk_length: Ttl, - /// Number of hops a `Shuffle` message is propagated until a peer replies to it. - pub shuffle_random_walk_length: Ttl, - /// Number of active peers to be included in a `Shuffle` request. - pub shuffle_active_view_count: usize, - /// Number of passive peers to be included in a `Shuffle` request. - pub shuffle_passive_view_count: usize, - /// Interval duration for shuffle requests - pub shuffle_interval: Duration, - /// Timeout after which a `Neighbor` request is considered failed - pub neighbor_request_timeout: Duration, -} -impl Default for Config { - /// Default values for the HyParView layer - fn default() -> Self { - Self { - // From the paper (p9) - active_view_capacity: 5, - // From the paper (p9) - passive_view_capacity: 30, - // From the paper (p9) - active_random_walk_length: Ttl(6), - // From the paper (p9) - passive_random_walk_length: Ttl(3), - // From the paper (p9) - shuffle_random_walk_length: Ttl(6), - // From the paper (p9) - shuffle_active_view_count: 3, - // From the paper (p9) - shuffle_passive_view_count: 4, - // Wild guess - shuffle_interval: Duration::from_secs(60), - // Wild guess - neighbor_request_timeout: Duration::from_millis(500), - } - } -} - -pub type Respond = bool; -pub type Alive = bool; - -#[derive(Default, Debug, Clone)] -pub struct Stats { - total_connections: usize, -} - -/// The state of the HyParView protocol -#[derive(Debug)] -pub struct State { - /// Our peer identity - me: PI, - /// Our opaque user data to transmit to peers on join messages - me_data: Option, - /// The active view, i.e. peers we are connected to - pub(crate) active_view: IndexSet, - /// The passive view, i.e. peers we know about but are not connected to at the moment - pub(crate) passive_view: IndexSet, - /// Protocol configuration (cannot change at runtime) - config: Config, - /// Whether a shuffle timer is currently scheduled - shuffle_scheduled: bool, - /// Random number generator - rng: RG, - /// Statistics - stats: Stats, - /// The set of neighbor requests we sent out but did not yet receive a reply for - pending_neighbor_requests: HashSet, - /// The opaque user peer data we received for other peers - peer_data: HashMap, -} - -impl State -where - PI: PeerIdentity, - RG: Rng, -{ - pub fn new(me: PI, me_data: Option, config: Config, rng: RG) -> Self { - Self { - me, - me_data, - active_view: IndexSet::new(), - passive_view: IndexSet::new(), - config, - shuffle_scheduled: false, - rng, - stats: Stats::default(), - pending_neighbor_requests: Default::default(), - peer_data: Default::default(), - } - } - - pub fn handle(&mut self, event: InEvent, now: Instant, io: &mut impl IO) { - match event { - InEvent::RecvMessage(from, message) => self.handle_message(from, message, now, io), - InEvent::TimerExpired(timer) => match timer { - Timer::DoShuffle => self.handle_shuffle_timer(io), - Timer::PendingNeighborRequest(peer) => self.handle_pending_neighbor_timer(peer, io), - }, - InEvent::PeerDisconnected(peer) => self.handle_disconnect(peer, io), - InEvent::RequestJoin(peer) => self.handle_join(peer, io), - InEvent::UpdatePeerData(data) => { - self.me_data = Some(data); - } - InEvent::Quit => self.handle_quit(io), - } - - // this will only happen on the first call - if !self.shuffle_scheduled { - io.push(OutEvent::ScheduleTimer( - self.config.shuffle_interval, - Timer::DoShuffle, - )); - self.shuffle_scheduled = true; - } - } - - fn handle_message( - &mut self, - from: PI, - message: Message, - now: Instant, - io: &mut impl IO, - ) { - let is_disconnect = matches!(message, Message::Disconnect(Disconnect { .. })); - if !is_disconnect && !self.active_view.contains(&from) { - self.stats.total_connections += 1; - } - match message { - Message::Join(data) => self.on_join(from, data, now, io), - Message::ForwardJoin(details) => self.on_forward_join(from, details, now, io), - Message::Shuffle(details) => self.on_shuffle(from, details, io), - Message::ShuffleReply(details) => self.on_shuffle_reply(details, io), - Message::Neighbor(details) => self.on_neighbor(from, details, now, io), - Message::Disconnect(details) => self.on_disconnect(from, details, io), - } - - // Disconnect from passive nodes right after receiving a message. - if !is_disconnect && !self.active_view.contains(&from) { - io.push(OutEvent::DisconnectPeer(from)); - } - } - - fn handle_join(&mut self, peer: PI, io: &mut impl IO) { - io.push(OutEvent::SendMessage( - peer, - Message::Join(self.me_data.clone()), - )); - } - - fn handle_disconnect(&mut self, peer: PI, io: &mut impl IO) { - self.on_disconnect( - peer, - Disconnect { - alive: true, - respond: false, - }, - io, - ); - } - - fn handle_quit(&mut self, io: &mut impl IO) { - for peer in self.active_view.clone().into_iter() { - self.on_disconnect( - peer, - Disconnect { - alive: false, - respond: true, - }, - io, - ); - } - } - - fn on_join(&mut self, peer: PI, data: Option, now: Instant, io: &mut impl IO) { - // If the peer is already in our active view, there's nothing to do. - if self.active_view.contains(&peer) { - // .. but we still update the peer data. - self.insert_peer_info((peer, data).into(), io); - return; - } - // "A node that receives a join request will start by adding the new - // node to its active view, even if it has to drop a random node from it. (6)" - self.add_active(peer, data.clone(), Priority::High, now, io); - // "The contact node c will then send to all other nodes in its active view a ForwardJoin - // request containing the new node identifier. Associated to the join procedure, - // there are two configuration parameters, named Active Random Walk Length (ARWL), - // that specifies the maximum number of hops a ForwardJoin request is propagated, - // and Passive Random Walk Length (PRWL), that specifies at which point in the walk the node - // is inserted in a passive view. To use these parameters, the ForwardJoin request carries - // a “time to live” field that is initially set to ARWL and decreased at every hop. (7)" - let ttl = self.config.active_random_walk_length; - let peer_info = PeerInfo { id: peer, data }; - for node in self.active_view.iter_without(&peer) { - let message = Message::ForwardJoin(ForwardJoin { - peer: peer_info.clone(), - ttl, - }); - io.push(OutEvent::SendMessage(*node, message)); - } - } - - fn on_forward_join( - &mut self, - sender: PI, - message: ForwardJoin, - now: Instant, - io: &mut impl IO, - ) { - // "i) If the time to live is equal to zero or if the number of nodes in p’s active view is equal to one, - // it will add the new node to its active view (7)" - if message.ttl.expired() || self.active_view.len() <= 1 { - self.add_active( - message.peer.id, - message.peer.data.clone(), - Priority::High, - now, - io, - ); - } - // "ii) If the time to live is equal to PRWL, p will insert the new node into its passive view" - else if message.ttl == self.config.passive_random_walk_length { - self.add_passive(message.peer.id, message.peer.data.clone(), io); - } - // "iii) The time to live field is decremented." - // "iv) If, at this point, n has not been inserted - // in p’s active view, p will forward the request to a random node in its active view - // (different from the one from which the request was received)." - if !self.active_view.contains(&message.peer.id) { - match self - .active_view - .pick_random_without(&[&sender], &mut self.rng) - { - None => { - unreachable!("if the peer was not added, there are at least two peers in our active view."); - } - Some(next) => { - let message = Message::ForwardJoin(ForwardJoin { - peer: message.peer, - ttl: message.ttl.next(), - }); - io.push(OutEvent::SendMessage(*next, message)); - } - } - } - } - - fn on_neighbor(&mut self, from: PI, details: Neighbor, now: Instant, io: &mut impl IO) { - self.pending_neighbor_requests.remove(&from); - // "A node q that receives a high priority neighbor request will always accept the request, even - // if it has to drop a random member from its active view (again, the member that is dropped will - // receive a Disconnect notification). If a node q receives a low priority Neighbor request, it will - // only accept the request if it has a free slot in its active view, otherwise it will refuse the request." - match details.priority { - Priority::High => { - self.add_active(from, details.data, Priority::High, now, io); - } - Priority::Low if !self.active_is_full() => { - self.add_active(from, details.data, Priority::Low, now, io); - } - _ => {} - } - } - - /// Get the peer [`PeerInfo`] for a peer. - fn peer_info(&self, id: &PI) -> PeerInfo { - let data = self.peer_data.get(id).cloned(); - PeerInfo { id: *id, data } - } - - fn insert_peer_info(&mut self, peer_info: PeerInfo, io: &mut impl IO) { - if let Some(data) = peer_info.data { - let old = self.peer_data.remove(&peer_info.id); - let same = matches!(old, Some(old) if old == data); - if !same { - io.push(OutEvent::PeerData(peer_info.id, data.clone())); - } - self.peer_data.insert(peer_info.id, data); - } - } - - /// Handle a [`Message::Shuffle`] - /// - /// > A node q that receives a Shuffle request will first decrease its time to live. If the time - /// > to live of the message is greater than zero and the number of nodes in q’s active view is - /// > greater than 1, the node will select a random node from its active view, different from the - /// > one he received this shuffle message from, and simply forwards the Shuffle request. - /// > Otherwise, node q accepts the Shuffle request and send back (p.8) - fn on_shuffle(&mut self, from: PI, shuffle: Shuffle, io: &mut impl IO) { - if shuffle.ttl.expired() || self.active_view.len() <= 1 { - let len = shuffle.nodes.len(); - for node in shuffle.nodes { - self.add_passive(node.id, node.data, io); - } - let nodes = self - .passive_view - .shuffled_and_capped(len, &mut self.rng) - .into_iter() - .map(|id| self.peer_info(&id)); - let message = Message::ShuffleReply(ShuffleReply { - nodes: nodes.collect(), - }); - io.push(OutEvent::SendMessage(shuffle.origin, message)); - } else if let Some(node) = self - .active_view - .pick_random_without(&[&shuffle.origin, &from], &mut self.rng) - { - let message = Message::Shuffle(Shuffle { - origin: shuffle.origin, - nodes: shuffle.nodes, - ttl: shuffle.ttl.next(), - }); - io.push(OutEvent::SendMessage(*node, message)); - } - } - - fn on_shuffle_reply(&mut self, message: ShuffleReply, io: &mut impl IO) { - for node in message.nodes { - self.add_passive(node.id, node.data, io); - } - } - - fn on_disconnect(&mut self, peer: PI, details: Disconnect, io: &mut impl IO) { - self.pending_neighbor_requests.remove(&peer); - self.remove_active(&peer, details.respond, io); - if details.alive { - if let Some(data) = self.peer_data.remove(&peer) { - self.add_passive(peer, Some(data), io); - } - } else { - self.passive_view.remove(&peer); - } - } - - fn handle_shuffle_timer(&mut self, io: &mut impl IO) { - if let Some(node) = self.active_view.pick_random(&mut self.rng) { - let active = self.active_view.shuffled_without_and_capped( - &[node], - self.config.shuffle_active_view_count, - &mut self.rng, - ); - let passive = self.passive_view.shuffled_without_and_capped( - &[node], - self.config.shuffle_passive_view_count, - &mut self.rng, - ); - let nodes = active - .iter() - .chain(passive.iter()) - .map(|id| self.peer_info(id)); - let message = Shuffle { - origin: self.me, - nodes: nodes.collect(), - ttl: self.config.shuffle_random_walk_length, - }; - io.push(OutEvent::SendMessage(*node, Message::Shuffle(message))); - } - io.push(OutEvent::ScheduleTimer( - self.config.shuffle_interval, - Timer::DoShuffle, - )); - } - - fn passive_is_full(&self) -> bool { - self.passive_view.len() >= self.config.passive_view_capacity - } - - fn active_is_full(&self) -> bool { - self.active_view.len() >= self.config.active_view_capacity - } - - /// Add a peer to the passive view. - /// - /// If the passive view is full, it will first remove a random peer and then insert the new peer. - /// If a peer is currently in the active view it will not be added. - fn add_passive(&mut self, peer: PI, data: Option, io: &mut impl IO) { - self.insert_peer_info((peer, data).into(), io); - if self.active_view.contains(&peer) || self.passive_view.contains(&peer) || peer == self.me - { - return; - } - if self.passive_is_full() { - self.passive_view.remove_random(&mut self.rng); - } - self.passive_view.insert(peer); - } - - /// Remove a peer from the active view. - /// - /// If respond is true, a Disconnect message will be sent to the peer. - fn remove_active(&mut self, peer: &PI, respond: Respond, io: &mut impl IO) -> Option { - self.active_view.get_index_of(peer).map(|idx| { - let removed_peer = self - .remove_active_by_index(idx, respond, RemovalReason::Disconnect, io) - .unwrap(); - - self.refill_active_from_passive(&[&removed_peer], io); - - removed_peer - }) - } - - fn refill_active_from_passive(&mut self, skip_peers: &[&PI], io: &mut impl IO) { - if self.active_view.len() + self.pending_neighbor_requests.len() - >= self.config.active_view_capacity - { - return; - } - // "When a node p suspects that one of the nodes present in its active view has failed - // (by either disconnecting or blocking), it selects a random node q from its passive view and - // attempts to establish a TCP connection with q. If the connection fails to establish, - // node q is considered failed and removed from p’s passive view; another node q′ is selected - // at random and a new attempt is made. The procedure is repeated until a connection is established - // with success." (p7) - let mut skip_peers = skip_peers.to_vec(); - skip_peers.extend(self.pending_neighbor_requests.iter()); - - if let Some(node) = self - .passive_view - .pick_random_without(&skip_peers, &mut self.rng) - { - let priority = match self.active_view.is_empty() { - true => Priority::High, - false => Priority::Low, - }; - let message = Message::Neighbor(Neighbor { - priority, - data: self.me_data.clone(), - }); - io.push(OutEvent::SendMessage(*node, message)); - // schedule a timer that checks if the node replied with a neighbor message, - // otherwise try again with another passive node. - io.push(OutEvent::ScheduleTimer( - self.config.neighbor_request_timeout, - Timer::PendingNeighborRequest(*node), - )); - self.pending_neighbor_requests.insert(*node); - }; - } - - fn handle_pending_neighbor_timer(&mut self, peer: PI, io: &mut impl IO) { - if self.pending_neighbor_requests.remove(&peer) { - self.passive_view.remove(&peer); - self.refill_active_from_passive(&[], io); - } - } - - fn remove_active_by_index( - &mut self, - peer_index: usize, - respond: Respond, - reason: RemovalReason, - io: &mut impl IO, - ) -> Option { - if let Some(peer) = self.active_view.remove_index(peer_index) { - if respond { - let message = Message::Disconnect(Disconnect { - alive: true, - respond: false, - }); - io.push(OutEvent::SendMessage(peer, message)); - } - io.push(OutEvent::DisconnectPeer(peer)); - io.push(OutEvent::EmitEvent(Event::NeighborDown(peer))); - let data = self.peer_data.remove(&peer); - self.add_passive(peer, data, io); - debug!(other = ?peer, "removed from active view, reason: {reason:?}"); - Some(peer) - } else { - None - } - } - - /// Remove a random peer from the active view. - fn free_random_slot_in_active_view(&mut self, io: &mut impl IO) { - if let Some(index) = self.active_view.pick_random_index(&mut self.rng) { - self.remove_active_by_index(index, true, RemovalReason::Random, io); - } - } - - /// Add a peer to the active view. - /// - /// If the active view is currently full, a random peer will be removed first. - /// Sends a Neighbor message to the peer. If high_priority is true, the peer - /// may not deny the Neighbor request. - fn add_active( - &mut self, - peer: PI, - data: Option, - priority: Priority, - _now: Instant, - io: &mut impl IO, - ) -> bool { - self.insert_peer_info((peer, data).into(), io); - if self.active_view.contains(&peer) || peer == self.me { - return true; - } - match (priority, self.active_is_full()) { - (Priority::High, is_full) => { - if is_full { - self.free_random_slot_in_active_view(io); - } - self.add_active_unchecked(peer, Priority::High, io); - true - } - (Priority::Low, false) => { - self.add_active_unchecked(peer, Priority::Low, io); - true - } - (Priority::Low, true) => false, - } - } - - fn add_active_unchecked(&mut self, peer: PI, priority: Priority, io: &mut impl IO) { - self.passive_view.remove(&peer); - self.active_view.insert(peer); - debug!(other = ?peer, "add to active view"); - - let message = Message::Neighbor(Neighbor { - priority, - data: self.me_data.clone(), - }); - io.push(OutEvent::SendMessage(peer, message)); - io.push(OutEvent::EmitEvent(Event::NeighborUp(peer))); - } -} - -#[derive(Debug)] -enum RemovalReason { - Disconnect, - Random, -} diff --git a/iroh-gossip/src/proto/plumtree.rs b/iroh-gossip/src/proto/plumtree.rs deleted file mode 100644 index f5b66f039e3..00000000000 --- a/iroh-gossip/src/proto/plumtree.rs +++ /dev/null @@ -1,878 +0,0 @@ -//! Implementation of the Plumtree epidemic broadcast tree protocol -//! -//! The implementation is based on [this paper][paper] by Joao Leitao, Jose Pereira, Luıs Rodrigues -//! and the [example implementation][impl] by Bartosz Sypytkowski -//! -//! [paper]: https://asc.di.fct.unl.pt/~jleitao/pdf/srds07-leitao.pdf -//! [impl]: https://gist.github.com/Horusiath/84fac596101b197da0546d1697580d99 - -use std::{ - collections::{HashMap, HashSet, VecDeque}, - hash::Hash, - time::{Duration, Instant}, -}; - -use bytes::Bytes; -use derive_more::{Add, From, Sub}; -use serde::{Deserialize, Serialize}; -use tracing::warn; - -use super::{ - util::{idbytes_impls, TimeBoundCache}, - PeerIdentity, IO, -}; - -/// A message identifier, which is the message content's blake3 hash. -#[derive(Serialize, Deserialize, Clone, Hash, Copy, PartialEq, Eq)] -pub struct MessageId([u8; 32]); -idbytes_impls!(MessageId, "MessageId"); - -impl MessageId { - /// Create a `[MessageId]` by hashing the message content. - /// - /// This hashes the input with [`blake3::hash`]. - pub fn from_content(message: &[u8]) -> Self { - Self::from(blake3::hash(message)) - } -} - -/// Events Plumtree is informed of from the peer sampling service and IO layer. -#[derive(Debug)] -pub enum InEvent { - /// A [`Message`] was received from the peer. - RecvMessage(PI, Message), - /// Broadcast the contained payload to the given scope. - Broadcast(Bytes, Scope), - /// A timer has expired. - TimerExpired(Timer), - /// New member `PI` has joined the topic. - NeighborUp(PI), - /// Peer `PI` has disconnected from the topic. - NeighborDown(PI), -} - -/// Events Plumtree emits. -#[derive(Debug, PartialEq, Eq)] -pub enum OutEvent { - /// Ask the IO layer to send a [`Message`] to peer `PI`. - SendMessage(PI, Message), - /// Schedule a [`Timer`]. - ScheduleTimer(Duration, Timer), - /// Emit an [`Event`] to the application. - EmitEvent(Event), -} - -/// Kinds of timers Plumtree needs to schedule. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum Timer { - /// Request the content for [`MessageId`] by sending [`Message::Graft`]. - /// - /// The message will be sent to a peer that sent us an [`Message::IHave`] for this [`MessageId`], - /// which will send us the message content in reply and also move the peer into the eager set. - /// Will be a no-op if the message for [`MessageId`] was already received from another peer by now. - SendGraft(MessageId), - /// Dispatch the [`Message::IHave`] in our lazy push queue. - DispatchLazyPush, - /// Evict the message cache - EvictCache, -} - -/// Event emitted by the [`State`] to the application. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Event { - /// A new gossip message was received. - Received(GossipEvent), -} - -#[derive(Clone, derive_more::Debug, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] -pub struct GossipEvent { - /// The content of the gossip message. - #[debug("<{}b>", content.len())] - pub content: Bytes, - /// The peer that we received the gossip message from. Note that this is not the peer that - /// originally broadcasted the message, but the peer before us in the gossiping path. - pub delivered_from: PI, - /// The broadcast scope of the message. - pub scope: DeliveryScope, -} - -impl GossipEvent { - fn from_message(message: &Gossip, from: PI) -> Self { - Self { - content: message.content.clone(), - scope: message.scope, - delivered_from: from, - } - } -} - -/// Number of delivery hops a message has taken. -#[derive( - From, Add, Sub, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Debug, Hash, -)] -pub struct Round(u16); - -impl Round { - pub fn next(&self) -> Round { - Round(self.0 + 1) - } -} - -/// Messages that we can send and receive from peers within the topic. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum Message { - /// When receiving Gossip, emit as event and forward full message to eager peer and (after a - /// delay) message IDs to lazy peers. - Gossip(Gossip), - /// When receiving Prune, move the peer from the eager to the lazy set. - Prune, - /// When receiving Graft, move the peer to the eager set and send the full content for the - /// included message ID. - Graft(Graft), - /// When receiving IHave, do nothing initially, and request the messages for the included - /// message IDs after some time if they aren't pushed eagerly to us. - IHave(Vec), -} - -/// Payload messages transmitted by the protocol. -#[derive(Serialize, Deserialize, Clone, derive_more::Debug, PartialEq, Eq)] -pub struct Gossip { - /// Id of the message. - id: MessageId, - /// Message contents. - #[debug("<{}b>", content.len())] - content: Bytes, - /// Scope to broadcast to. - scope: DeliveryScope, -} - -impl Gossip { - fn round(&self) -> Option { - match self.scope { - DeliveryScope::Swarm(round) => Some(round), - DeliveryScope::Neighbors => None, - } - } -} - -/// The scope to deliver the message to. -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Copy)] -pub enum DeliveryScope { - /// This message was received from the swarm, with a distance (in hops) travelled from the - /// original broadcaster. - Swarm(Round), - /// This message was received from a direct neighbor that broadcasted the message to neighbors - /// only. - Neighbors, -} - -impl DeliveryScope { - /// Whether this message was directly received from its publisher. - pub fn is_direct(&self) -> bool { - matches!(self, Self::Neighbors | Self::Swarm(Round(0))) - } -} - -/// The broadcast scope of a gossip message. -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Copy)] -pub enum Scope { - /// The message is broadcast to all peers in the swarm. - Swarm, - /// The message is broadcast only to the immediate neighbors of a peer. - Neighbors, -} - -impl Gossip { - /// Get a clone of this `Gossip` message and increase the delivery round by 1. - pub fn next_round(&self) -> Option { - match self.scope { - DeliveryScope::Neighbors => None, - DeliveryScope::Swarm(round) => Some(Gossip { - id: self.id, - content: self.content.clone(), - scope: DeliveryScope::Swarm(round.next()), - }), - } - } - - /// Validate that the message id is the blake3 hash of the message content. - pub fn validate(&self) -> bool { - let expected = MessageId::from_content(&self.content); - expected == self.id - } -} - -/// Control message to inform peers we have a message without transmitting the whole payload. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct IHave { - /// Id of the message. - id: MessageId, - /// Delivery round of the message. - round: Round, -} - -/// Control message to signal a peer that they have been moved to the eager set, and to ask the -/// peer to do the same with this node. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct Graft { - /// Message id that triggers the graft, if any. - /// On receiving a graft, the payload message must be sent in reply if a message id is set. - id: Option, - /// Delivery round of the [`Message::IHave`] that triggered this Graft message. - round: Round, -} - -/// Configuration for the gossip broadcast layer. -/// -/// Currently, the expectation is that the configuration is the same for all peers in the -/// network (as recommended in the paper). -#[derive(Clone, Debug)] -pub struct Config { - /// When receiving an [`IHave`] message, this timeout is registered. If the message for the - /// [`IHave`] was not received once the timeout is expired, a [`Graft`] message is sent to the - /// peer that sent us the [`IHave`] to request the message payload. - /// - /// The plumtree paper notes: - /// > The timeout value is a protocol parameter that should be configured considering the - /// > diameter of the overlay and a target maximum recovery latency, defined by the application - /// > requirements. (p.8) - pub graft_timeout_1: Duration, - /// This timeout is registered when sending a [`Graft`] message. If a reply has not been - /// received once the timeout expires, we send another [`Graft`] message to the next peer that - /// sent us an [`IHave`] for this message. - /// - /// The plumtree paper notes: - /// > This second timeout value should be smaller that the first, in the order of an average - /// > round trip time to a neighbor. - pub graft_timeout_2: Duration, - /// Timeout after which [`IHave`] messages are pushed to peers. - pub dispatch_timeout: Duration, - /// The protocol performs a tree optimization, which promotes lazy peers to eager peers if the - /// [`Message::IHave`] messages received from them have a lower number of hops from the - /// message's origin as the [`InEvent::Broadcast`] messages received from our eager peers. This - /// parameter is the number of hops that the lazy peers must be closer to the origin than our - /// eager peers to be promoted to become an eager peer. - pub optimization_threshold: Round, - - /// Duration for which to keep gossip messages in the internal message cache. - /// - /// Messages broadcast from this node or received from other nodes are kept in an internal - /// cache for this duration before being evicted. If this is too low, other nodes will not be - /// able to retrieve messages once they need them. If this is high, the cache will grow. - /// - /// Should be at least around several round trip times to peers. - pub message_cache_retention: Duration, - - /// Duration for which to keep the [`MessageId`]s for received messages. - /// - /// Should be at least as long as [`Self::message_cache_retention`], usually will be longer to - /// not accidentally receive messages multiple times. - pub message_id_retention: Duration, - - /// How often the internal caches will be checked for expired items. - pub cache_evict_interval: Duration, -} - -impl Default for Config { - /// Sensible defaults for the plumtree configuration - // - // TODO: Find out what good defaults are for the three timeouts here. Current numbers are - // guesses that need validation. The paper does not have concrete recommendations for these - // numbers. - fn default() -> Self { - Self { - // Paper: "The timeout value is a protocol parameter that should be configured considering - // the diameter of the overlay and a target maximum recovery latency, defined by the - // application requirements. This is a parameter that should be statically configured - // at deployment time." (p. 8) - // - // Earthstar has 5ms it seems, see https://github.com/earthstar-project/earthstar/blob/1523c640fedf106f598bf79b184fb0ada64b1cc0/src/syncer/plum_tree.ts#L75 - // However in the paper it is more like a few roundtrips if I read things correctly. - graft_timeout_1: Duration::from_millis(80), - - // Paper: "This second timeout value should be smaller that the first, in the order of an - // average round trip time to a neighbor." (p. 9) - // - // Earthstar doesn't have this step from my reading. - graft_timeout_2: Duration::from_millis(40), - - // Again, paper does not tell a recommended number here. Likely should be quite small, - // as to not delay messages without need. This would also be the time frame in which - // `IHave`s are aggregated to save on packets. - // - // Eartstar dispatches immediately from my reading. - dispatch_timeout: Duration::from_millis(5), - - // This number comes from experiment settings the plumtree paper (p. 12) - optimization_threshold: Round(7), - - // This is a certainly-high-enough value for usual operation. - message_cache_retention: Duration::from_secs(30), - message_id_retention: Duration::from_secs(90), - cache_evict_interval: Duration::from_secs(1), - } - } -} - -/// Stats about this topic's plumtree. -#[derive(Debug, Default, Clone)] -pub struct Stats { - /// Number of payload messages received so far. - /// - /// See [`Message::Gossip`]. - pub payload_messages_received: u64, - /// Number of control messages received so far. - /// - /// See [`Message::Prune`], [`Message::Graft`], [`Message::IHave`]. - pub control_messages_received: u64, - /// Max round seen so far. - pub max_last_delivery_hop: u16, -} - -/// State of the plumtree. -#[derive(Debug)] -pub struct State { - /// Our address. - me: PI, - /// Configuration for this plumtree. - config: Config, - - /// Set of peers used for payload exchange. - pub(crate) eager_push_peers: HashSet, - /// Set of peers used for control message exchange. - pub(crate) lazy_push_peers: HashSet, - - lazy_push_queue: HashMap>, - - /// Messages for which a [`MessageId`] has been seen via a [`Message::IHave`] but we have not - /// yet received the full payload. For each, we store the peers that have claimed to have this - /// message. - missing_messages: HashMap>, - /// Messages for which the full payload has been seen. - received_messages: TimeBoundCache, - /// Payloads of received messages. - cache: TimeBoundCache, - - /// Message ids for which a [`Timer::SendGraft`] has been scheduled. - graft_timer_scheduled: HashSet, - /// Whether a [`Timer::DispatchLazyPush`] has been scheduled. - dispatch_timer_scheduled: bool, - - /// Set to false after the first message is received. Used for initial timer scheduling. - init: bool, - - /// [`Stats`] of this plumtree. - pub(crate) stats: Stats, -} - -impl State { - /// Initialize the [`State`] of a plumtree. - pub fn new(me: PI, config: Config) -> Self { - Self { - me, - eager_push_peers: Default::default(), - lazy_push_peers: Default::default(), - lazy_push_queue: Default::default(), - config, - missing_messages: Default::default(), - received_messages: Default::default(), - graft_timer_scheduled: Default::default(), - dispatch_timer_scheduled: false, - cache: Default::default(), - init: false, - stats: Default::default(), - } - } - - /// Handle an [`InEvent`]. - pub fn handle(&mut self, event: InEvent, now: Instant, io: &mut impl IO) { - if !self.init { - self.init = true; - self.on_evict_cache_timer(now, io) - } - match event { - InEvent::RecvMessage(from, message) => self.handle_message(from, message, now, io), - InEvent::Broadcast(data, scope) => self.broadcast(data, scope, now, io), - InEvent::NeighborUp(peer) => self.on_neighbor_up(peer), - InEvent::NeighborDown(peer) => self.on_neighbor_down(peer), - InEvent::TimerExpired(timer) => match timer { - Timer::DispatchLazyPush => self.on_dispatch_timer(io), - Timer::SendGraft(id) => { - self.on_send_graft_timer(id, io); - } - Timer::EvictCache => self.on_evict_cache_timer(now, io), - }, - } - } - - /// Get access to the [`Stats`] of the plumtree. - pub fn stats(&self) -> &Stats { - &self.stats - } - - /// Handle receiving a [`Message`]. - fn handle_message(&mut self, sender: PI, message: Message, now: Instant, io: &mut impl IO) { - if matches!(message, Message::Gossip(_)) { - self.stats.payload_messages_received += 1; - } else { - self.stats.control_messages_received += 1; - } - match message { - Message::Gossip(details) => self.on_gossip(sender, details, now, io), - Message::Prune => self.on_prune(sender), - Message::IHave(details) => self.on_ihave(sender, details, io), - Message::Graft(details) => self.on_graft(sender, details, io), - } - } - - /// Dispatches messages from lazy queue over to lazy peers. - fn on_dispatch_timer(&mut self, io: &mut impl IO) { - for (peer, list) in self.lazy_push_queue.drain() { - io.push(OutEvent::SendMessage(peer, Message::IHave(list))); - } - - self.dispatch_timer_scheduled = false; - } - - /// Send a gossip message. - /// - /// Will be pushed in full to eager peers. - /// Pushing the message id to the lazy peers is delayed by a timer. - fn broadcast(&mut self, content: Bytes, scope: Scope, now: Instant, io: &mut impl IO) { - let id = MessageId::from_content(&content); - let scope = match scope { - Scope::Neighbors => DeliveryScope::Neighbors, - Scope::Swarm => DeliveryScope::Swarm(Round(0)), - }; - let message = Gossip { id, content, scope }; - let me = self.me; - if let DeliveryScope::Swarm(_) = scope { - self.received_messages - .insert(id, (), now + self.config.message_id_retention); - self.cache.insert( - id, - message.clone(), - now + self.config.message_cache_retention, - ); - self.lazy_push(message.clone(), &me, io); - } - - self.eager_push(message.clone(), &me, io); - } - - /// Handle receiving a [`Message::Gossip`]. - fn on_gossip(&mut self, sender: PI, message: Gossip, now: Instant, io: &mut impl IO) { - // Validate that the message id is the blake3 hash of the message content. - if !message.validate() { - // TODO: Do we want to take any measures against the sender if we received a message - // with a spoofed message id? - warn!( - peer = ?sender, - "Received a message with spoofed message id ({})", message.id - ); - return; - } - - // if we already received this message: move peer to lazy set - // and notify peer about this. - if self.received_messages.contains_key(&message.id) { - self.add_lazy(sender); - io.push(OutEvent::SendMessage(sender, Message::Prune)); - // otherwise store the message, emit to application and forward to peers - } else { - if let DeliveryScope::Swarm(prev_round) = message.scope { - // insert the message in the list of received messages - self.received_messages.insert( - message.id, - (), - now + self.config.message_id_retention, - ); - // increase the round for forwarding the message, and add to cache - // to reply to Graft messages later - // TODO: add callback/event to application to get missing messages that were received before? - let message = message.next_round().expect("just checked"); - - self.cache.insert( - message.id, - message.clone(), - now + self.config.message_cache_retention, - ); - // push the message to our peers - self.eager_push(message.clone(), &sender, io); - self.lazy_push(message.clone(), &sender, io); - // cleanup places where we track missing messages - self.graft_timer_scheduled.remove(&message.id); - let previous_ihaves = self.missing_messages.remove(&message.id); - // do the optimization step from the paper - if let Some(previous_ihaves) = previous_ihaves { - self.optimize_tree(&sender, &message, previous_ihaves, io); - } - self.stats.max_last_delivery_hop = - self.stats.max_last_delivery_hop.max(prev_round.0); - } - - // emit event to application - io.push(OutEvent::EmitEvent(Event::Received( - GossipEvent::from_message(&message, sender), - ))); - } - } - - /// Optimize the tree by pruning the `sender` of a [`Message::Gossip`] if we previously - /// received a [`Message::IHave`] for the same message with a much lower number of delivery - /// hops from the original broadcaster of the message. - /// - /// See [Config::optimization_threshold]. - fn optimize_tree( - &mut self, - gossip_sender: &PI, - message: &Gossip, - previous_ihaves: VecDeque<(PI, Round)>, - io: &mut impl IO, - ) { - let round = message.round().expect("only called for swarm messages"); - let best_ihave = previous_ihaves - .iter() - .min_by(|(_a_peer, a_round), (_b_peer, b_round)| a_round.cmp(b_round)) - .copied(); - - if let Some((ihave_peer, ihave_round)) = best_ihave { - if (ihave_round < round) && (round - ihave_round) >= self.config.optimization_threshold - { - // Graft the sender of the IHave, but only if it's not already eager. - if !self.eager_push_peers.contains(&ihave_peer) { - let message = Message::Graft(Graft { - id: None, - round: ihave_round, - }); - io.push(OutEvent::SendMessage(ihave_peer, message)); - } - // Prune the sender of the Gossip. - io.push(OutEvent::SendMessage(*gossip_sender, Message::Prune)); - } - } - } - - /// Handle receiving a [`Message::Prune`]. - fn on_prune(&mut self, sender: PI) { - self.add_lazy(sender); - } - - /// Handle receiving a [`Message::IHave`]. - /// - /// > When a node receives a IHAVE message, it simply marks the corresponding message as - /// > missing It then starts a timer, with a predefined timeout value, and waits for the missing - /// > message to be received via eager push before the timer expires. The timeout value is a - /// > protocol parameter that should be configured considering the diameter of the overlay and a - /// > target maximum recovery latency, defined by the application requirements. This is a - /// > parameter that should be statically configured at deployment time. (p8) - fn on_ihave(&mut self, sender: PI, ihaves: Vec, io: &mut impl IO) { - for ihave in ihaves { - if !self.received_messages.contains_key(&ihave.id) { - self.missing_messages - .entry(ihave.id) - .or_default() - .push_back((sender, ihave.round)); - - if !self.graft_timer_scheduled.contains(&ihave.id) { - self.graft_timer_scheduled.insert(ihave.id); - io.push(OutEvent::ScheduleTimer( - self.config.graft_timeout_1, - Timer::SendGraft(ihave.id), - )); - } - } - } - } - - /// A scheduled [`Timer::SendGraft`] has reached it's deadline. - fn on_send_graft_timer(&mut self, id: MessageId, io: &mut impl IO) { - // if the message was received before the timer ran out, there is no need to request it - // again - if self.received_messages.contains_key(&id) { - return; - } - // get the first peer that advertised this message - let entry = self - .missing_messages - .get_mut(&id) - .and_then(|entries| entries.pop_front()); - if let Some((peer, round)) = entry { - self.add_eager(peer); - let message = Message::Graft(Graft { - id: Some(id), - round, - }); - io.push(OutEvent::SendMessage(peer, message)); - - // "when a GRAFT message is sent, another timer is started to expire after a certain timeout, - // to ensure that the message will be requested to another neighbor if it is not received - // meanwhile. This second timeout value should be smaller that the first, in the order of - // an average round trip time to a neighbor." (p9) - io.push(OutEvent::ScheduleTimer( - self.config.graft_timeout_2, - Timer::SendGraft(id), - )); - } - } - - /// Handle receiving a [`Message::Graft`]. - fn on_graft(&mut self, sender: PI, details: Graft, io: &mut impl IO) { - self.add_eager(sender); - if let Some(id) = details.id { - if let Some(message) = self.cache.get(&id) { - io.push(OutEvent::SendMessage( - sender, - Message::Gossip(message.clone()), - )); - } - } - } - - /// Handle a [`InEvent::NeighborUp`] when a peer joins the topic. - fn on_neighbor_up(&mut self, peer: PI) { - self.add_eager(peer); - } - - /// Handle a [`InEvent::NeighborDown`] when a peer leaves the topic. - /// > When a neighbor is detected to leave the overlay, it is simple removed from the - /// > membership. Furthermore, the record of IHAVE messages sent from failed members is deleted - /// > from the missing history. (p9) - fn on_neighbor_down(&mut self, peer: PI) { - self.missing_messages.retain(|_message_id, ihaves| { - ihaves.retain(|(ihave_peer, _round)| *ihave_peer != peer); - !ihaves.is_empty() - }); - self.eager_push_peers.remove(&peer); - self.lazy_push_peers.remove(&peer); - } - - fn on_evict_cache_timer(&mut self, now: Instant, io: &mut impl IO) { - self.cache.expire_until(now); - io.push(OutEvent::ScheduleTimer( - self.config.cache_evict_interval, - Timer::EvictCache, - )); - } - - /// Moves peer into eager set. - fn add_eager(&mut self, peer: PI) { - self.lazy_push_peers.remove(&peer); - self.eager_push_peers.insert(peer); - } - - /// Moves peer into lazy set. - fn add_lazy(&mut self, peer: PI) { - self.eager_push_peers.remove(&peer); - self.lazy_push_peers.insert(peer); - } - - /// Immediately sends message to eager peers. - fn eager_push(&mut self, gossip: Gossip, sender: &PI, io: &mut impl IO) { - for peer in self - .eager_push_peers - .iter() - .filter(|peer| **peer != self.me && *peer != sender) - { - io.push(OutEvent::SendMessage( - *peer, - Message::Gossip(gossip.clone()), - )); - } - } - - /// Queue lazy message announcements into the queue that will be sent out as batched - /// [`Message::IHave`] messages once the [`Timer::DispatchLazyPush`] timer is triggered. - fn lazy_push(&mut self, gossip: Gossip, sender: &PI, io: &mut impl IO) { - let Some(round) = gossip.round() else { - return; - }; - for peer in self.lazy_push_peers.iter().filter(|x| *x != sender) { - self.lazy_push_queue.entry(*peer).or_default().push(IHave { - id: gossip.id, - round, - }); - } - if !self.dispatch_timer_scheduled { - io.push(OutEvent::ScheduleTimer( - self.config.dispatch_timeout, - Timer::DispatchLazyPush, - )); - self.dispatch_timer_scheduled = true; - } - } -} - -#[cfg(test)] -mod test { - use super::*; - #[test] - fn optimize_tree() { - let mut io = VecDeque::new(); - let config: Config = Default::default(); - let mut state = State::new(1, config.clone()); - let now = Instant::now(); - - // we receive an IHave message from peer 2 - // it has `round: 2` which means that the the peer that sent us the IHave was - // two hops away from the original sender of the message - let content: Bytes = b"hi".to_vec().into(); - let id = MessageId::from_content(&content); - let event = InEvent::RecvMessage( - 2u32, - Message::IHave(vec![IHave { - id, - round: Round(2), - }]), - ); - state.handle(event, now, &mut io); - io.clear(); - // we then receive a `Gossip` message with the same `MessageId` from peer 3 - // the message has `round: 6`, which means it travelled 6 hops until it reached us - // this is less hops than to peer 2, but not enough to trigger the optimization - // because we use the default config which has `optimization_threshold: 7` - let event = InEvent::RecvMessage( - 3, - Message::Gossip(Gossip { - id, - content: content.clone(), - scope: DeliveryScope::Swarm(Round(6)), - }), - ); - state.handle(event, now, &mut io); - let expected = { - // we expect a dispatch timer schedule and receive event, but no Graft or Prune - // messages - let mut io = VecDeque::new(); - io.push(OutEvent::ScheduleTimer( - config.dispatch_timeout, - Timer::DispatchLazyPush, - )); - io.push(OutEvent::EmitEvent(Event::Received(GossipEvent { - content, - delivered_from: 3, - scope: DeliveryScope::Swarm(Round(6)), - }))); - io - }; - assert_eq!(io, expected); - io.clear(); - - // now we run the same flow again but this time peer 3 is 9 hops away from the message's - // sender. message's sender. this will trigger the optimization: - // peer 2 will be promoted to eager and peer 4 demoted to lazy - - let content: Bytes = b"hi2".to_vec().into(); - let id = MessageId::from_content(&content); - let event = InEvent::RecvMessage( - 2u32, - Message::IHave(vec![IHave { - id, - round: Round(2), - }]), - ); - state.handle(event, now, &mut io); - io.clear(); - - let event = InEvent::RecvMessage( - 3, - Message::Gossip(Gossip { - id, - content: content.clone(), - scope: DeliveryScope::Swarm(Round(9)), - }), - ); - state.handle(event, now, &mut io); - let expected = { - // this time we expect the Graft and Prune messages to be sent, performing the - // optimization step - let mut io = VecDeque::new(); - io.push(OutEvent::SendMessage( - 2, - Message::Graft(Graft { - id: None, - round: Round(2), - }), - )); - io.push(OutEvent::SendMessage(3, Message::Prune)); - io.push(OutEvent::EmitEvent(Event::Received(GossipEvent { - content, - delivered_from: 3, - scope: DeliveryScope::Swarm(Round(9)), - }))); - io - }; - assert_eq!(io, expected); - } - - #[test] - fn spoofed_messages_are_ignored() { - let config: Config = Default::default(); - let mut state = State::new(1, config.clone()); - let now = Instant::now(); - - // we recv a correct gossip message and expect the Received event to be emitted - let content: Bytes = b"hello1".to_vec().into(); - let message = Message::Gossip(Gossip { - content: content.clone(), - id: MessageId::from_content(&content), - scope: DeliveryScope::Swarm(Round(1)), - }); - let mut io = VecDeque::new(); - state.handle(InEvent::RecvMessage(2, message), now, &mut io); - let expected = { - let mut io = VecDeque::new(); - io.push(OutEvent::ScheduleTimer( - config.cache_evict_interval, - Timer::EvictCache, - )); - io.push(OutEvent::ScheduleTimer( - config.dispatch_timeout, - Timer::DispatchLazyPush, - )); - io.push(OutEvent::EmitEvent(Event::Received(GossipEvent { - content, - delivered_from: 2, - scope: DeliveryScope::Swarm(Round(1)), - }))); - io - }; - assert_eq!(io, expected); - - // now we recv with a spoofed id and expect no event to be emitted - let content: Bytes = b"hello2".to_vec().into(); - let message = Message::Gossip(Gossip { - content, - id: MessageId::from_content(b"foo"), - scope: DeliveryScope::Swarm(Round(1)), - }); - let mut io = VecDeque::new(); - state.handle(InEvent::RecvMessage(2, message), now, &mut io); - let expected = VecDeque::new(); - assert_eq!(io, expected); - } - - #[test] - fn cache_is_evicted() { - let config: Config = Default::default(); - let mut state = State::new(1, config.clone()); - let now = Instant::now(); - let content: Bytes = b"hello1".to_vec().into(); - let message = Message::Gossip(Gossip { - content: content.clone(), - id: MessageId::from_content(&content), - scope: DeliveryScope::Swarm(Round(1)), - }); - let mut io = VecDeque::new(); - state.handle(InEvent::RecvMessage(2, message), now, &mut io); - assert_eq!(state.cache.len(), 1); - - let now = now + Duration::from_secs(1); - state.handle(InEvent::TimerExpired(Timer::EvictCache), now, &mut io); - assert_eq!(state.cache.len(), 1); - - let now = now + config.message_cache_retention; - state.handle(InEvent::TimerExpired(Timer::EvictCache), now, &mut io); - assert_eq!(state.cache.len(), 0); - } -} diff --git a/iroh-gossip/src/proto/state.rs b/iroh-gossip/src/proto/state.rs deleted file mode 100644 index b8561aeeefb..00000000000 --- a/iroh-gossip/src/proto/state.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! The protocol state of the `iroh-gossip` protocol. - -use std::{ - collections::{hash_map, HashMap, HashSet}, - time::{Duration, Instant}, -}; - -use iroh_metrics::{inc, inc_by}; -use rand::Rng; -use serde::{Deserialize, Serialize}; -use tracing::trace; - -use crate::{ - metrics::Metrics, - proto::{ - topic::{self, Command}, - util::idbytes_impls, - Config, PeerData, PeerIdentity, - }, -}; - -/// The identifier for a topic -#[derive(Clone, Copy, Eq, PartialEq, Hash, Serialize, Ord, PartialOrd, Deserialize)] -pub struct TopicId([u8; 32]); -idbytes_impls!(TopicId, "TopicId"); - -/// Protocol wire message -/// -/// This is the wire frame of the `iroh-gossip` protocol. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Message { - topic: TopicId, - message: topic::Message, -} - -impl Message { - /// Get the kind of this message - pub fn kind(&self) -> MessageKind { - self.message.kind() - } -} - -/// Whether this is a control or data message -#[derive(Debug)] -pub enum MessageKind { - /// A data message. - Data, - /// A control message. - Control, -} - -impl Message { - /// Get the encoded size of this message - pub fn size(&self) -> postcard::Result { - postcard::experimental::serialized_size(&self) - } -} - -/// A timer to be registered into the runtime -/// -/// As the implementation of the protocol is an IO-less state machine, registering timers does not -/// happen within the protocol implementation. Instead, these `Timer` structs are emitted as -/// [`OutEvent`]s. The implementer must register the timer in its runtime to be emitted on the specified [`Instant`], -/// and once triggered inject an [`InEvent::TimerExpired`] into the protocol state. -#[derive(Clone, Debug)] -pub struct Timer { - topic: TopicId, - timer: topic::Timer, -} - -/// Input event to the protocol state. -#[derive(Clone, Debug)] -pub enum InEvent { - /// Message received from the network. - RecvMessage(PI, Message), - /// Execute a command from the application. - Command(TopicId, Command), - /// Trigger a previously scheduled timer. - TimerExpired(Timer), - /// Peer disconnected on the network level. - PeerDisconnected(PI), - /// Update the opaque peer data about yourself. - UpdatePeerData(PeerData), -} - -/// Output event from the protocol state. -#[derive(Debug, Clone)] -pub enum OutEvent { - /// Send a message on the network - SendMessage(PI, Message), - /// Emit an event to the application. - EmitEvent(TopicId, topic::Event), - /// Schedule a timer. The runtime is responsible for sending an [InEvent::TimerExpired] - /// after the duration. - ScheduleTimer(Duration, Timer), - /// Close the connection to a peer on the network level. - DisconnectPeer(PI), - /// Updated peer data - PeerData(PI, PeerData), -} - -type ConnsMap = HashMap>; -type Outbox = Vec>; - -enum InEventMapped { - All(topic::InEvent), - TopicEvent(TopicId, topic::InEvent), -} - -impl From> for InEventMapped { - fn from(event: InEvent) -> InEventMapped { - match event { - InEvent::RecvMessage(from, Message { topic, message }) => { - Self::TopicEvent(topic, topic::InEvent::RecvMessage(from, message)) - } - InEvent::Command(topic, command) => { - Self::TopicEvent(topic, topic::InEvent::Command(command)) - } - InEvent::TimerExpired(Timer { topic, timer }) => { - Self::TopicEvent(topic, topic::InEvent::TimerExpired(timer)) - } - InEvent::PeerDisconnected(peer) => Self::All(topic::InEvent::PeerDisconnected(peer)), - InEvent::UpdatePeerData(data) => Self::All(topic::InEvent::UpdatePeerData(data)), - } - } -} - -/// The state of the `iroh-gossip` protocol. -/// -/// The implementation works as an IO-less state machine. The implementer injects events through -/// [`Self::handle`], which returns an iterator of [`OutEvent`]s to be processed. -/// -/// This struct contains a map of [`topic::State`] for each topic that was joined. It mostly acts as -/// a forwarder of [`InEvent`]s to matching topic state. Each topic's state is completely -/// independent; thus the actual protocol logic lives with [`topic::State`]. -#[derive(Debug)] -pub struct State { - me: PI, - me_data: PeerData, - config: Config, - rng: R, - states: HashMap>, - outbox: Outbox, - peer_topics: ConnsMap, -} - -impl State { - /// Create a new protocol state instance. - /// - /// `me` is the [`PeerIdentity`] of the local node, `peer_data` is the initial [`PeerData`] - /// (which can be updated over time). - /// For the protocol to perform as recommended in the papers, the [`Config`] should be - /// identical for all nodes in the network. - pub fn new(me: PI, me_data: PeerData, config: Config, rng: R) -> Self { - Self { - me, - me_data, - config, - rng, - states: Default::default(), - outbox: Default::default(), - peer_topics: Default::default(), - } - } - - /// Get a reference to the node's [`PeerIdentity`] - pub fn me(&self) -> &PI { - &self.me - } - - /// Get a reference to the protocol state for a topic. - pub fn state(&self, topic: &TopicId) -> Option<&topic::State> { - self.states.get(topic) - } - - /// Get a reference to the protocol state for a topic. - #[cfg(test)] - pub fn state_mut(&mut self, topic: &TopicId) -> Option<&mut topic::State> { - self.states.get_mut(topic) - } - - /// Get an iterator of all joined topics. - pub fn topics(&self) -> impl Iterator { - self.states.keys() - } - - /// Get an iterator for the states of all joined topics. - pub fn states(&self) -> impl Iterator)> { - self.states.iter() - } - - /// Check if a topic has any active (connected) peers. - pub fn has_active_peers(&self, topic: &TopicId) -> bool { - self.state(topic) - .map(|s| s.has_active_peers()) - .unwrap_or(false) - } - - /// Returns the maximum message size configured in the gossip protocol. - pub fn max_message_size(&self) -> usize { - self.config.max_message_size - } - - /// Handle an [`InEvent`] - /// - /// This returns an iterator of [`OutEvent`]s that must be processed. - pub fn handle( - &mut self, - event: InEvent, - now: Instant, - ) -> impl Iterator> + '_ { - trace!("gossp event: {event:?}"); - track_in_event(&event); - - let event: InEventMapped = event.into(); - - match event { - InEventMapped::TopicEvent(topic, event) => { - // when receiving a join command, initialize state if it doesn't exist - if matches!(&event, topic::InEvent::Command(Command::Join(_peers))) { - if let hash_map::Entry::Vacant(e) = self.states.entry(topic) { - e.insert(topic::State::with_rng( - self.me, - Some(self.me_data.clone()), - self.config.clone(), - self.rng.clone(), - )); - } - } - - // when receiving a quit command, note this and drop the topic state after - // processing this last event - let quit = matches!(event, topic::InEvent::Command(Command::Quit)); - - // pass the event to the state handler - if let Some(state) = self.states.get_mut(&topic) { - // when receiving messages, update our conn map to take note that this topic state may want - // to keep this connection - if let topic::InEvent::RecvMessage(from, _message) = &event { - self.peer_topics.entry(*from).or_default().insert(topic); - } - let out = state.handle(event, now); - for event in out { - handle_out_event(topic, event, &mut self.peer_topics, &mut self.outbox); - } - } - - if quit { - self.states.remove(&topic); - } - } - // when a peer disconnected on the network level, forward event to all states - InEventMapped::All(event) => { - if let topic::InEvent::UpdatePeerData(data) = &event { - self.me_data = data.clone(); - } - for (topic, state) in self.states.iter_mut() { - let out = state.handle(event.clone(), now); - for event in out { - handle_out_event(*topic, event, &mut self.peer_topics, &mut self.outbox); - } - } - } - } - - // track metrics - track_out_events(&self.outbox); - - self.outbox.drain(..) - } -} - -fn handle_out_event( - topic: TopicId, - event: topic::OutEvent, - conns: &mut ConnsMap, - outbox: &mut Outbox, -) { - match event { - topic::OutEvent::SendMessage(to, message) => { - outbox.push(OutEvent::SendMessage(to, Message { topic, message })) - } - topic::OutEvent::EmitEvent(event) => outbox.push(OutEvent::EmitEvent(topic, event)), - topic::OutEvent::ScheduleTimer(delay, timer) => { - outbox.push(OutEvent::ScheduleTimer(delay, Timer { topic, timer })) - } - topic::OutEvent::DisconnectPeer(peer) => { - let empty = conns - .get_mut(&peer) - .map(|list| list.remove(&topic) && list.is_empty()) - .unwrap_or(false); - if empty { - conns.remove(&peer); - outbox.push(OutEvent::DisconnectPeer(peer)); - } - } - topic::OutEvent::PeerData(peer, data) => outbox.push(OutEvent::PeerData(peer, data)), - } -} - -fn track_out_events(events: &[OutEvent]) { - for event in events { - match event { - OutEvent::SendMessage(_to, message) => match message.kind() { - MessageKind::Data => { - inc!(Metrics, msgs_data_sent); - inc_by!( - Metrics, - msgs_data_sent_size, - message.size().unwrap_or(0) as u64 - ); - } - MessageKind::Control => { - inc!(Metrics, msgs_ctrl_sent); - inc_by!( - Metrics, - msgs_ctrl_sent_size, - message.size().unwrap_or(0) as u64 - ); - } - }, - OutEvent::EmitEvent(_topic, event) => match event { - super::Event::NeighborUp(_peer) => inc!(Metrics, neighbor_up), - super::Event::NeighborDown(_peer) => inc!(Metrics, neighbor_down), - _ => {} - }, - _ => {} - } - } -} - -fn track_in_event(event: &InEvent) { - if let InEvent::RecvMessage(_from, message) = event { - match message.kind() { - MessageKind::Data => { - inc!(Metrics, msgs_data_recv); - inc_by!( - Metrics, - msgs_data_recv_size, - message.size().unwrap_or(0) as u64 - ); - } - MessageKind::Control => { - inc!(Metrics, msgs_ctrl_recv); - inc_by!( - Metrics, - msgs_ctrl_recv_size, - message.size().unwrap_or(0) as u64 - ); - } - } - } -} diff --git a/iroh-gossip/src/proto/tests.rs b/iroh-gossip/src/proto/tests.rs deleted file mode 100644 index 5f5f3ef40b4..00000000000 --- a/iroh-gossip/src/proto/tests.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! Simulation framework for testing the protocol implementation - -use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, - time::{Duration, Instant}, -}; - -use bytes::Bytes; -use rand::Rng; -use rand_core::SeedableRng; -use tracing::{debug, warn}; - -use super::{ - util::TimerMap, Command, Config, Event, InEvent, OutEvent, PeerIdentity, State, Timer, TopicId, -}; -use crate::proto::Scope; - -const TICK_DURATION: Duration = Duration::from_millis(10); -const DEFAULT_LATENCY: Duration = TICK_DURATION.saturating_mul(3); - -/// Test network implementation. -/// -/// Stores events in VecDeques and processes on ticks. -/// Timers are checked after each tick. The local time is increased with TICK_DURATION before -/// each tick. -/// -/// Note: Panics when sending to an unknown peer. -pub struct Network { - start: Instant, - time: Instant, - tick_duration: Duration, - inqueues: Vec>>, - pub(crate) peers: Vec>, - peers_by_address: HashMap, - conns: HashSet>, - events: VecDeque<(PI, TopicId, Event)>, - timers: TimerMap<(usize, Timer)>, - transport: TimerMap<(usize, InEvent)>, - latencies: HashMap, Duration>, -} -impl Network { - pub fn new(time: Instant) -> Self { - Self { - start: time, - time, - tick_duration: TICK_DURATION, - inqueues: Default::default(), - peers: Default::default(), - peers_by_address: Default::default(), - conns: Default::default(), - events: Default::default(), - timers: TimerMap::new(), - transport: TimerMap::new(), - latencies: HashMap::new(), - } - } -} - -fn push_back( - inqueues: &mut [VecDeque>], - peer_pos: usize, - event: InEvent, -) { - inqueues.get_mut(peer_pos).unwrap().push_back(event); -} - -impl Network { - pub fn push(&mut self, peer: State) { - let idx = self.inqueues.len(); - self.inqueues.push(VecDeque::new()); - self.peers_by_address.insert(*peer.me(), idx); - self.peers.push(peer); - } - - pub fn events(&mut self) -> impl Iterator)> + '_ { - self.events.drain(..) - } - - pub fn events_sorted(&mut self) -> Vec<(PI, TopicId, Event)> { - sort(self.events().collect()) - } - - pub fn conns(&self) -> Vec<(PI, PI)> { - sort(self.conns.iter().cloned().map(Into::into).collect()) - } - - pub fn command(&mut self, peer: PI, topic: TopicId, command: Command) { - debug!(?peer, "~~ COMMAND {command:?}"); - let idx = *self.peers_by_address.get(&peer).unwrap(); - push_back(&mut self.inqueues, idx, InEvent::Command(topic, command)); - } - - pub fn ticks(&mut self, n: usize) { - (0..n).for_each(|_| self.tick()) - } - - pub fn get_tick(&self) -> u32 { - ((self.time - self.start) / self.tick_duration.as_millis() as u32).as_millis() as u32 - } - - pub fn tick(&mut self) { - self.time += self.tick_duration; - - // process timers - for (_time, (idx, timer)) in self.timers.drain_until(&self.time) { - push_back(&mut self.inqueues, idx, InEvent::TimerExpired(timer)); - } - - // move messages - for (_time, (peer, event)) in self.transport.drain_until(&self.time) { - push_back(&mut self.inqueues, peer, event); - } - - // process inqueues: let peer handle all incoming events - let mut messages_sent = 0; - for (idx, queue) in self.inqueues.iter_mut().enumerate() { - let state = self.peers.get_mut(idx).unwrap(); - let peer = *state.me(); - while let Some(event) = queue.pop_front() { - if let InEvent::RecvMessage(from, _message) = &event { - self.conns.insert((*from, peer).into()); - } - debug!(peer = ?peer, "IN {event:?}"); - let out = state.handle(event, self.time); - for event in out { - debug!(peer = ?peer, "OUT {event:?}"); - match event { - OutEvent::SendMessage(to, message) => { - let to_idx = *self.peers_by_address.get(&to).unwrap(); - let latency = latency_between(&mut self.latencies, &peer, &to); - self.transport.insert( - self.time + latency, - (to_idx, InEvent::RecvMessage(peer, message)), - ); - messages_sent += 1; - } - OutEvent::ScheduleTimer(latency, timer) => { - self.timers.insert(self.time + latency, (idx, timer)); - } - OutEvent::DisconnectPeer(to) => { - debug!(peer = ?peer, other = ?to, "disconnect"); - let to_idx = *self.peers_by_address.get(&to).unwrap(); - let latency = latency_between(&mut self.latencies, &peer, &to) - + Duration::from_nanos(1); - if self.conns.remove(&(peer, to).into()) { - self.transport.insert( - self.time + latency, - (to_idx, InEvent::PeerDisconnected(peer)), - ); - } - } - OutEvent::EmitEvent(topic, event) => { - debug!(peer = ?peer, "emit {event:?}"); - self.events.push_back((peer, topic, event)); - } - OutEvent::PeerData(_peer, _data) => {} - } - } - } - } - debug!( - tick = self.get_tick(), - "~~ TICK (messages sent: {messages_sent})" - ); - } - - pub fn peer(&self, peer: &PI) -> Option<&State> { - self.peers_by_address - .get(peer) - .cloned() - .and_then(|idx| self.peers.get(idx)) - } - - pub fn get_active(&self, peer: &PI, topic: &TopicId) -> Option>> { - let peer = self.peer(peer)?; - match peer.state(topic) { - Some(state) => Some(Some( - state.swarm.active_view.iter().cloned().collect::>(), - )), - None => Some(None), - } - } -} -fn latency_between( - _latencies: &mut HashMap, Duration>, - _a: &PI, - _b: &PI, -) -> Duration { - DEFAULT_LATENCY -} - -pub fn assert_synchronous_active( - network: &Network, -) -> bool { - for state in network.peers.iter() { - let peer = *state.me(); - for (topic, state) in state.states() { - for other in state.swarm.active_view.iter() { - let other_idx = network.peers_by_address.get(other).unwrap(); - let other_state = &network - .peers - .get(*other_idx) - .unwrap() - .state(topic) - .unwrap() - .swarm - .active_view; - if !other_state.contains(&peer) { - warn!(peer = ?peer, other = ?other, "missing active_view peer in other"); - return false; - } - } - for other in state.gossip.eager_push_peers.iter() { - let other_idx = network.peers_by_address.get(other).unwrap(); - let other_state = &network - .peers - .get(*other_idx) - .unwrap() - .state(topic) - .unwrap() - .gossip - .eager_push_peers; - if !other_state.contains(&peer) { - warn!(peer = ?peer, other = ?other, "missing eager_push peer in other"); - return false; - } - } - } - } - true -} - -pub type PeerId = usize; - -/// A simple simulator for the gossip protocol -pub struct Simulator { - simulator_config: SimulatorConfig, - protocol_config: Config, - network: Network, - round_stats: Vec, -} -pub struct SimulatorConfig { - pub peers_count: usize, - pub bootstrap_count: usize, - pub bootstrap_ticks: usize, - pub join_ticks: usize, - pub warmup_ticks: usize, - pub round_max_ticks: usize, -} -#[derive(Debug, Default)] -pub struct RoundStats { - ticks: usize, - rmr: f32, - ldh: u16, -} - -pub const TOPIC: TopicId = TopicId::from_bytes([0u8; 32]); - -impl Default for SimulatorConfig { - fn default() -> Self { - Self { - peers_count: 100, - bootstrap_count: 5, - bootstrap_ticks: 50, - join_ticks: 1, - warmup_ticks: 300, - round_max_ticks: 200, - } - } -} -impl Simulator { - pub fn new(simulator_config: SimulatorConfig, protocol_config: Config) -> Self { - Self { - protocol_config, - simulator_config, - network: Network::new(Instant::now()), - round_stats: Default::default(), - } - } - pub fn init(&mut self) { - for i in 0..self.simulator_config.peers_count { - let rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); - self.network.push(State::new( - i, - Default::default(), - self.protocol_config.clone(), - rng.clone(), - )); - } - } - pub fn bootstrap(&mut self) { - self.network.command(0, TOPIC, Command::Join(vec![])); - for i in 1..self.simulator_config.bootstrap_count { - self.network.command(i, TOPIC, Command::Join(vec![0])); - } - self.network.ticks(self.simulator_config.bootstrap_ticks); - let _ = self.network.events(); - - for i in self.simulator_config.bootstrap_count..self.simulator_config.peers_count { - let contact = i % self.simulator_config.bootstrap_count; - self.network.command(i, TOPIC, Command::Join(vec![contact])); - self.network.ticks(self.simulator_config.join_ticks); - let _ = self.network.events(); - } - self.network.ticks(self.simulator_config.warmup_ticks); - let _ = self.network.events(); - } - - pub fn gossip_round(&mut self, from: PeerId, message: Bytes) { - let prev_total_payload_counter = self.total_payload_messages(); - let mut expected: HashSet = HashSet::from_iter( - self.network - .peers - .iter() - .map(|p| *p.me()) - .filter(|p| *p != from), - ); - let expected_len = expected.len() as u64; - self.network.command( - from, - TOPIC, - Command::Broadcast(message.clone(), Scope::Swarm), - ); - - let mut tick = 0; - loop { - if expected.is_empty() { - break; - } - if tick > self.simulator_config.round_max_ticks { - break; - } - tick += 1; - self.network.tick(); - let events = self.network.events(); - let received: HashSet<_> = events - .filter( - |(_peer, _topic, event)| matches!(event, Event::Received(recv) if recv.content == message), - ) - .map(|(peer, _topic, _msg)| peer) - .collect(); - for peer in received.iter() { - expected.remove(peer); - } - } - - assert!(expected.is_empty(), "all nodes received the broadcast"); - let payload_counter = self.total_payload_messages() - prev_total_payload_counter; - let rmr = (payload_counter as f32 / (expected_len as f32 - 1.)) - 1.; - let ldh = self.max_ldh(); - let stats = RoundStats { - ticks: tick, - rmr, - ldh, - }; - self.round_stats.push(stats); - self.reset_stats() - } - - pub fn report_round_sums(&self) { - let len = self.round_stats.len(); - let mut rmr = 0.; - let mut ldh = 0.; - let mut ticks = 0.; - for round in self.round_stats.iter() { - rmr += round.rmr; - ldh += round.ldh as f32; - ticks += round.ticks as f32; - } - rmr /= len as f32; - ldh /= len as f32; - ticks /= len as f32; - eprintln!( - "average over {} rounds with {} peers: RMR {rmr:.2} LDH {ldh:.2} ticks {ticks:.2}", - self.round_stats.len(), - self.network.peers.len(), - ); - eprintln!("RMR = Relative Message Redundancy, LDH = Last Delivery Hop"); - } - - fn reset_stats(&mut self) { - for state in self.network.peers.iter_mut() { - let state = state.state_mut(&TOPIC).unwrap(); - state.gossip.stats = Default::default(); - } - } - - fn max_ldh(&self) -> u16 { - let mut max = 0; - for state in self.network.peers.iter() { - let state = state.state(&TOPIC).unwrap(); - let stats = state.gossip.stats(); - max = max.max(stats.max_last_delivery_hop); - } - max - } - - fn total_payload_messages(&self) -> u64 { - let mut sum = 0; - for state in self.network.peers.iter() { - let state = state.state(&TOPIC).unwrap(); - let stats = state.gossip.stats(); - sum += stats.payload_messages_received; - } - sum - } -} - -/// Helper struct for active connections. A sorted tuple. -#[derive(Debug, Clone, PartialOrd, Ord, Eq, PartialEq, Hash)] -pub struct ConnId([PI; 2]); -impl ConnId { - pub fn new(a: PI, b: PI) -> Self { - let mut conn = [a, b]; - conn.sort(); - Self(conn) - } -} -impl From<(PI, PI)> for ConnId { - fn from((a, b): (PI, PI)) -> Self { - Self::new(a, b) - } -} -impl From> for (PI, PI) { - fn from(conn: ConnId) -> (PI, PI) { - (conn.0[0], conn.0[1]) - } -} - -pub fn sort(items: Vec) -> Vec { - let mut sorted = items; - sorted.sort(); - sorted -} - -pub fn report_round_distribution(network: &Network) { - let mut eager_distrib: BTreeMap = BTreeMap::new(); - let mut lazy_distrib: BTreeMap = BTreeMap::new(); - let mut active_distrib: BTreeMap = BTreeMap::new(); - let mut passive_distrib: BTreeMap = BTreeMap::new(); - let mut payload_recv = 0; - let mut control_recv = 0; - for state in network.peers.iter() { - for (_topic, state) in state.states() { - let stats = state.gossip.stats(); - *eager_distrib - .entry(state.gossip.eager_push_peers.len()) - .or_default() += 1; - *lazy_distrib - .entry(state.gossip.lazy_push_peers.len()) - .or_default() += 1; - *active_distrib - .entry(state.swarm.active_view.len()) - .or_default() += 1; - *passive_distrib - .entry(state.swarm.passive_view.len()) - .or_default() += 1; - payload_recv += stats.payload_messages_received; - control_recv += stats.control_messages_received; - } - } - // eprintln!("distributions {round_distrib:?}"); - eprintln!("payload_recv {payload_recv} control_recv {control_recv}"); - eprintln!("eager_distrib {eager_distrib:?}"); - eprintln!("lazy_distrib {lazy_distrib:?}"); - eprintln!("active_distrib {active_distrib:?}"); - eprintln!("passive_distrib {passive_distrib:?}"); -} diff --git a/iroh-gossip/src/proto/topic.rs b/iroh-gossip/src/proto/topic.rs deleted file mode 100644 index f6358458871..00000000000 --- a/iroh-gossip/src/proto/topic.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! This module contains the implementation of the gossiping protocol for an individual topic - -use std::{ - collections::VecDeque, - time::{Duration, Instant}, -}; - -use bytes::Bytes; -use derive_more::From; -use rand::Rng; -use rand_core::SeedableRng; -use serde::{Deserialize, Serialize}; - -use super::{ - hyparview::{self, InEvent as SwarmIn}, - plumtree::{self, GossipEvent, InEvent as GossipIn, Scope}, - state::MessageKind, - PeerData, PeerIdentity, -}; - -/// The default maximum size in bytes for a gossip message. -/// This is a sane but arbitrary default and can be changed in the [`Config`]. -pub const DEFAULT_MAX_MESSAGE_SIZE: usize = 4096; - -/// Input event to the topic state handler. -#[derive(Clone, Debug)] -pub enum InEvent { - /// Message received from the network. - RecvMessage(PI, Message), - /// Execute a command from the application. - Command(Command), - /// Trigger a previously scheduled timer. - TimerExpired(Timer), - /// Peer disconnected on the network level. - PeerDisconnected(PI), - /// Update the opaque peer data about yourself. - UpdatePeerData(PeerData), -} - -/// An output event from the state handler. -#[derive(Debug, PartialEq, Eq)] -pub enum OutEvent { - /// Send a message on the network - SendMessage(PI, Message), - /// Emit an event to the application. - EmitEvent(Event), - /// Schedule a timer. The runtime is responsible for sending an [InEvent::TimerExpired] - /// after the duration. - ScheduleTimer(Duration, Timer), - /// Close the connection to a peer on the network level. - DisconnectPeer(PI), - /// Emitted when new [`PeerData`] was received for a peer. - PeerData(PI, PeerData), -} - -impl From> for OutEvent { - fn from(event: hyparview::OutEvent) -> Self { - use hyparview::OutEvent::*; - match event { - SendMessage(to, message) => Self::SendMessage(to, message.into()), - ScheduleTimer(delay, timer) => Self::ScheduleTimer(delay, timer.into()), - DisconnectPeer(peer) => Self::DisconnectPeer(peer), - EmitEvent(event) => Self::EmitEvent(event.into()), - PeerData(peer, data) => Self::PeerData(peer, data), - } - } -} - -impl From> for OutEvent { - fn from(event: plumtree::OutEvent) -> Self { - use plumtree::OutEvent::*; - match event { - SendMessage(to, message) => Self::SendMessage(to, message.into()), - ScheduleTimer(delay, timer) => Self::ScheduleTimer(delay, timer.into()), - EmitEvent(event) => Self::EmitEvent(event.into()), - } - } -} - -/// A trait for a concrete type to push `OutEvent`s to. -/// -/// The implementation is generic over this trait, which allows the upper layer to supply a -/// container of their choice for `OutEvent`s emitted from the protocol state. -pub trait IO { - /// Push an event in the IO container - fn push(&mut self, event: impl Into>); - - /// Push all events from an iterator into the IO container - fn push_from_iter(&mut self, iter: impl IntoIterator>>) { - for event in iter.into_iter() { - self.push(event); - } - } -} - -/// A protocol message for a particular topic -#[derive(From, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub enum Message { - /// A message of the swarm membership layer - Swarm(hyparview::Message), - /// A message of the gossip broadcast layer - Gossip(plumtree::Message), -} - -impl Message { - /// Get the kind of this message - pub fn kind(&self) -> MessageKind { - match self { - Message::Swarm(_) => MessageKind::Control, - Message::Gossip(message) => match message { - plumtree::Message::Gossip(_) => MessageKind::Data, - _ => MessageKind::Control, - }, - } - } -} - -/// An event to be emitted to the application for a particular topic. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)] -pub enum Event { - /// We have a new, direct neighbor in the swarm membership layer for this topic - NeighborUp(PI), - /// We dropped direct neighbor in the swarm membership layer for this topic - NeighborDown(PI), - /// A gossip message was received for this topic - Received(GossipEvent), -} - -impl From> for Event { - fn from(value: hyparview::Event) -> Self { - match value { - hyparview::Event::NeighborUp(peer) => Self::NeighborUp(peer), - hyparview::Event::NeighborDown(peer) => Self::NeighborDown(peer), - } - } -} - -impl From> for Event { - fn from(value: plumtree::Event) -> Self { - match value { - plumtree::Event::Received(event) => Self::Received(event), - } - } -} - -/// A timer to be registered for a particular topic. -/// -/// This should be treated as an opaque value by the implementer and, once emitted, simply returned -/// to the protocol through [`InEvent::TimerExpired`]. -#[derive(Clone, From, Debug, PartialEq, Eq)] -pub enum Timer { - /// A timer for the swarm layer - Swarm(hyparview::Timer), - /// A timer for the gossip layer - Gossip(plumtree::Timer), -} - -/// A command to the protocol state for a particular topic. -#[derive(Clone, derive_more::Debug)] -pub enum Command { - /// Join this topic and connect to peers. - /// - /// If the list of peers is empty, will prepare the state and accept incoming join requests, - /// but only become operational after the first join request by another peer. - Join(Vec), - /// Broadcast a message for this topic. - Broadcast(#[debug("<{}b>", _0.len())] Bytes, Scope), - /// Leave this topic and drop all state. - Quit, -} - -impl IO for VecDeque> { - fn push(&mut self, event: impl Into>) { - self.push_back(event.into()) - } -} - -/// Protocol configuration -#[derive(Clone, Debug)] -pub struct Config { - /// Configuration for the swarm membership layer - pub membership: hyparview::Config, - /// Configuration for the gossip broadcast layer - pub broadcast: plumtree::Config, - /// Max message size in bytes. - /// - /// This size should be the same across a network to ensure all nodes can transmit and read large messages. - /// - /// At minimum, this size should be large enough to send gossip control messages. This can vary, depending on the size of the [`PeerIdentity`] you use and the size of the [`PeerData`] you transmit in your messages. - /// - /// The default is [`DEFAULT_MAX_MESSAGE_SIZE`]. - pub max_message_size: usize, -} - -impl Default for Config { - fn default() -> Self { - Self { - membership: Default::default(), - broadcast: Default::default(), - max_message_size: DEFAULT_MAX_MESSAGE_SIZE, - } - } -} - -/// The topic state maintains the swarm membership and broadcast tree for a particular topic. -#[derive(Debug)] -pub struct State { - me: PI, - pub(crate) swarm: hyparview::State, - pub(crate) gossip: plumtree::State, - outbox: VecDeque>, - stats: Stats, -} - -impl State { - /// Initialize the local state with the default random number generator. - pub fn new(me: PI, me_data: Option, config: Config) -> Self { - Self::with_rng(me, me_data, config, rand::rngs::StdRng::from_entropy()) - } -} - -impl State { - /// The address of your local endpoint. - pub fn endpoint(&self) -> &PI { - &self.me - } -} - -impl State { - /// Initialize the local state with a custom random number generator. - pub fn with_rng(me: PI, me_data: Option, config: Config, rng: R) -> Self { - Self { - swarm: hyparview::State::new(me, me_data, config.membership, rng), - gossip: plumtree::State::new(me, config.broadcast), - me, - outbox: VecDeque::new(), - stats: Stats::default(), - } - } - - /// Handle an incoming event. - /// - /// Returns an iterator of outgoing events that must be processed by the application. - pub fn handle( - &mut self, - event: InEvent, - now: Instant, - ) -> impl Iterator> + '_ { - let io = &mut self.outbox; - // Process the event, store out events in outbox. - match event { - InEvent::Command(command) => match command { - Command::Join(peers) => { - for peer in peers { - self.swarm.handle(SwarmIn::RequestJoin(peer), now, io); - } - } - Command::Broadcast(data, scope) => { - self.gossip - .handle(GossipIn::Broadcast(data, scope), now, io) - } - Command::Quit => self.swarm.handle(SwarmIn::Quit, now, io), - }, - InEvent::RecvMessage(from, message) => { - self.stats.messages_received += 1; - match message { - Message::Swarm(message) => { - self.swarm - .handle(SwarmIn::RecvMessage(from, message), now, io) - } - Message::Gossip(message) => { - self.gossip - .handle(GossipIn::RecvMessage(from, message), now, io) - } - } - } - InEvent::TimerExpired(timer) => match timer { - Timer::Swarm(timer) => self.swarm.handle(SwarmIn::TimerExpired(timer), now, io), - Timer::Gossip(timer) => self.gossip.handle(GossipIn::TimerExpired(timer), now, io), - }, - InEvent::PeerDisconnected(peer) => { - self.swarm.handle(SwarmIn::PeerDisconnected(peer), now, io); - self.gossip.handle(GossipIn::NeighborDown(peer), now, io); - } - InEvent::UpdatePeerData(data) => { - self.swarm.handle(SwarmIn::UpdatePeerData(data), now, io) - } - } - - // Forward NeighborUp and NeighborDown events from hyparview to plumtree - let mut io = VecDeque::new(); - for event in self.outbox.iter() { - match event { - OutEvent::EmitEvent(Event::NeighborUp(peer)) => { - self.gossip - .handle(GossipIn::NeighborUp(*peer), now, &mut io) - } - OutEvent::EmitEvent(Event::NeighborDown(peer)) => { - self.gossip - .handle(GossipIn::NeighborDown(*peer), now, &mut io) - } - _ => {} - } - } - // Note that this is a no-op because plumtree::handle(NeighborUp | NeighborDown) - // above does not emit any OutEvents. - self.outbox.extend(io.drain(..)); - - // Update sent message counter - self.stats.messages_sent += self - .outbox - .iter() - .filter(|event| matches!(event, OutEvent::SendMessage(_, _))) - .count(); - - self.outbox.drain(..) - } - - /// Get stats on how many messages were sent and received - /// - /// TODO: Remove/replace with metrics? - pub fn stats(&self) -> &Stats { - &self.stats - } - - /// Get statistics for the gossip broadcast state - /// - /// TODO: Remove/replace with metrics? - pub fn gossip_stats(&self) -> &plumtree::Stats { - self.gossip.stats() - } - - /// Check if this topic has any active (connected) peers. - pub fn has_active_peers(&self) -> bool { - !self.swarm.active_view.is_empty() - } -} - -/// Statistics for the protocol state of a topic -#[derive(Clone, Debug, Default)] -pub struct Stats { - /// Number of messages sent - pub messages_sent: usize, - /// Number of messages received - pub messages_received: usize, -} diff --git a/iroh-gossip/src/proto/util.rs b/iroh-gossip/src/proto/util.rs deleted file mode 100644 index bd04c2b0483..00000000000 --- a/iroh-gossip/src/proto/util.rs +++ /dev/null @@ -1,470 +0,0 @@ -//! Utilities used in the protocol implementation - -use std::{ - collections::{BTreeMap, HashMap}, - hash::Hash, - time::{Duration, Instant}, -}; - -use rand::{ - seq::{IteratorRandom, SliceRandom}, - Rng, -}; - -/// Implement methods, display, debug and conversion traits for 32 byte identifiers. -macro_rules! idbytes_impls { - ($ty:ty, $name:expr) => { - impl $ty { - /// Create from a byte array. - pub const fn from_bytes(bytes: [u8; 32]) -> Self { - Self(bytes) - } - - /// Get as byte slice. - pub fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - } - - impl> ::std::convert::From for $ty { - fn from(value: T) -> Self { - Self::from_bytes(value.into()) - } - } - - impl ::std::fmt::Display for $ty { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { - write!(f, "{}", ::iroh_base::base32::fmt(&self.0)) - } - } - - impl ::std::fmt::Debug for $ty { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { - write!(f, "{}({})", $name, ::iroh_base::base32::fmt_short(&self.0)) - } - } - - impl ::std::str::FromStr for $ty { - type Err = ::anyhow::Error; - fn from_str(s: &str) -> ::std::result::Result { - Ok(Self::from_bytes(::iroh_base::base32::parse_array(s)?)) - } - } - - impl ::std::convert::AsRef<[u8]> for $ty { - fn as_ref(&self) -> &[u8] { - &self.0 - } - } - - impl ::std::convert::AsRef<[u8; 32]> for $ty { - fn as_ref(&self) -> &[u8; 32] { - &self.0 - } - } - }; -} - -pub(crate) use idbytes_impls; - -/// A hash set where the iteration order of the values is independent of their -/// hash values. -/// -/// This is wrapper around [indexmap::IndexSet] which couple of utility methods -/// to randomly select elements from the set. -#[derive(Default, Debug, Clone, derive_more::Deref)] -pub(crate) struct IndexSet { - inner: indexmap::IndexSet, -} - -impl PartialEq for IndexSet { - fn eq(&self, other: &Self) -> bool { - self.inner == other.inner - } -} - -impl IndexSet { - pub fn new() -> Self { - Self { - inner: indexmap::IndexSet::new(), - } - } - - pub fn insert(&mut self, value: T) -> bool { - self.inner.insert(value) - } - - /// Remove a random element from the set. - pub fn remove_random(&mut self, rng: &mut R) -> Option { - self.pick_random_index(rng) - .and_then(|idx| self.inner.shift_remove_index(idx)) - } - - /// Pick a random element from the set. - pub fn pick_random(&self, rng: &mut R) -> Option<&T> { - self.pick_random_index(rng) - .and_then(|idx| self.inner.get_index(idx)) - } - - /// Pick a random element from the set, but not any of the elements in `without`. - pub fn pick_random_without(&self, without: &[&T], rng: &mut R) -> Option<&T> { - self.iter().filter(|x| !without.contains(x)).choose(rng) - } - - /// Pick a random index for an element in the set. - pub fn pick_random_index(&self, rng: &mut R) -> Option { - if self.is_empty() { - None - } else { - Some(rng.gen_range(0..self.inner.len())) - } - } - - /// Remove an element from the set. - /// - /// NOTE: the value is removed by swapping it with the last element of the set and popping it off. - /// **This modifies the order of element by moving the last element** - pub fn remove(&mut self, value: &T) -> Option { - self.inner.swap_remove_full(value).map(|(_i, v)| v) - } - - /// Remove an element from the set by its index. - /// - /// NOTE: the value is removed by swapping it with the last element of the set and popping it off. - /// **This modifies the order of element by moving the last element** - pub fn remove_index(&mut self, index: usize) -> Option { - self.inner.swap_remove_index(index) - } - - /// Create an iterator over the set in the order of insertion, while skipping the element in - /// `without`. - pub fn iter_without<'a>(&'a self, value: &'a T) -> impl Iterator { - self.iter().filter(move |x| *x != value) - } -} - -impl IndexSet -where - T: Hash + Eq + Clone, -{ - /// Create a vector of all elements in the set in random order. - pub fn shuffled(&self, rng: &mut R) -> Vec { - let mut items: Vec<_> = self.inner.iter().cloned().collect(); - items.shuffle(rng); - items - } - - /// Create a vector of all elements in the set in random order, and shorten to - /// the first `len` elements after shuffling. - pub fn shuffled_and_capped(&self, len: usize, rng: &mut R) -> Vec { - let mut items = self.shuffled(rng); - items.truncate(len); - items - } - - /// Create a vector of the elements in the set in random order while omitting - /// the elements in `without`. - pub fn shuffled_without(&self, without: &[&T], rng: &mut R) -> Vec { - let mut items = self - .inner - .iter() - .filter(|x| !without.contains(x)) - .cloned() - .collect::>(); - items.shuffle(rng); - items - } - - /// Create a vector of the elements in the set in random order while omitting - /// the elements in `without`, and shorten to the first `len` elements. - pub fn shuffled_without_and_capped( - &self, - without: &[&T], - len: usize, - rng: &mut R, - ) -> Vec { - let mut items = self.shuffled_without(without, rng); - items.truncate(len); - items - } -} - -impl IntoIterator for IndexSet { - type Item = T; - type IntoIter = as IntoIterator>::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.inner.into_iter() - } -} - -impl FromIterator for IndexSet -where - T: Hash + Eq, -{ - fn from_iter>(iterable: I) -> Self { - IndexSet { - inner: indexmap::IndexSet::from_iter(iterable), - } - } -} - -/// A [`BTreeMap`] with [`Instant`] as key. Allows to process expired items. -#[derive(Debug)] -pub struct TimerMap(BTreeMap>); - -impl Default for TimerMap { - fn default() -> Self { - Self::new() - } -} - -impl TimerMap { - /// Create a new, empty TimerMap. - pub fn new() -> Self { - Self(Default::default()) - } - /// Insert a new entry at the specified instant. - pub fn insert(&mut self, instant: Instant, item: T) { - let entry = self.0.entry(instant).or_default(); - entry.push(item); - } - - /// Remove and return all entries before and equal to `from`. - pub fn drain_until(&mut self, from: &Instant) -> impl Iterator { - let split_point = *from + Duration::from_nanos(1); - let later_half = self.0.split_off(&split_point); - let expired = std::mem::replace(&mut self.0, later_half); - expired - .into_iter() - .flat_map(|(t, v)| v.into_iter().map(move |v| (t, v))) - } - - /// Get a reference to the earliest entry in the TimerMap. - pub fn first(&self) -> Option<(&Instant, &Vec)> { - self.0.iter().next() - } - - /// Iterate over all items in the timer map. - pub fn iter(&self) -> impl Iterator { - self.0 - .iter() - .flat_map(|(t, v)| v.iter().map(move |v| (t, v))) - } -} - -impl TimerMap { - /// Remove an entry from the specified instant. - pub fn remove(&mut self, instant: &Instant, item: &T) { - if let Some(items) = self.0.get_mut(instant) { - items.retain(|x| x != item) - } - } -} - -/// A hash map where entries expire after a time -#[derive(Debug)] -pub struct TimeBoundCache { - map: HashMap, - expiry: TimerMap, -} - -impl Default for TimeBoundCache { - fn default() -> Self { - Self { - map: Default::default(), - expiry: Default::default(), - } - } -} - -impl TimeBoundCache { - /// Insert an item into the cache, marked with an expiration time. - pub fn insert(&mut self, key: K, value: V, expires: Instant) { - self.remove(&key); - self.map.insert(key.clone(), (expires, value)); - self.expiry.insert(expires, key); - } - - /// Returns `true` if the map contains a value for the specified key. - pub fn contains_key(&self, key: &K) -> bool { - self.map.contains_key(key) - } - - /// Remove an item from the cache. - pub fn remove(&mut self, key: &K) -> Option { - if let Some((expires, value)) = self.map.remove(key) { - self.expiry.remove(&expires, key); - Some(value) - } else { - None - } - } - - /// Get the number of entries in the cache. - pub fn len(&self) -> usize { - self.map.len() - } - - /// Returns `true` if the map contains no elements. - pub fn is_empty(&self) -> bool { - self.map.is_empty() - } - - /// Get an item from the cache. - pub fn get(&self, key: &K) -> Option<&V> { - self.map.get(key).map(|(_expires, value)| value) - } - - /// Get the expiration time for an item. - pub fn expires(&self, key: &K) -> Option<&Instant> { - self.map.get(key).map(|(expires, _value)| expires) - } - - /// Iterate over all items in the cache. - pub fn iter(&self) -> impl Iterator { - self.map.iter().map(|(k, (expires, v))| (k, v, expires)) - } - - /// Remove all entries with an expiry instant lower or equal to `instant`. - /// - /// Returns the number of items that were removed. - pub fn expire_until(&mut self, instant: Instant) -> usize { - let drain = self.expiry.drain_until(&instant); - let mut count = 0; - for (_instant, key) in drain { - count += 1; - let _value = self.map.remove(&key); - } - count - } -} - -#[cfg(test)] -mod test { - use std::{ - str::FromStr, - time::{Duration, Instant}, - }; - - use rand_core::SeedableRng; - - use super::{IndexSet, TimeBoundCache, TimerMap}; - - fn test_rng() -> rand_chacha::ChaCha12Rng { - rand_chacha::ChaCha12Rng::seed_from_u64(42) - } - - #[test] - fn indexset() { - let elems = [1, 2, 3, 4]; - let set = IndexSet::from_iter(elems); - let x = set.shuffled(&mut test_rng()); - assert_eq!(x, vec![4, 2, 1, 3]); - let x = set.shuffled_and_capped(2, &mut test_rng()); - assert_eq!(x, vec![4, 2]); - let x = set.shuffled_without(&[&1], &mut test_rng()); - assert_eq!(x, vec![4, 3, 2]); - let x = set.shuffled_without_and_capped(&[&1], 2, &mut test_rng()); - assert_eq!(x, vec![4, 3]); - - // recreate the rng - otherwise we get failures on some architectures when cross-compiling, - // likely due to usize differences pulling different amounts of randomness. - let x = set.pick_random(&mut test_rng()); - assert_eq!(x, Some(&3)); - let x = set.pick_random_without(&[&3], &mut test_rng()); - assert_eq!(x, Some(&4)); - - let mut set = set; - set.remove_random(&mut test_rng()); - assert_eq!(set, IndexSet::from_iter([1, 2, 4])); - } - - #[test] - fn timer_map() { - let mut map = TimerMap::new(); - let now = Instant::now(); - - let times = [ - now - Duration::from_secs(1), - now, - now + Duration::from_secs(1), - now + Duration::from_secs(2), - ]; - map.insert(times[0], -1); - map.insert(times[0], -2); - map.insert(times[1], 0); - map.insert(times[2], 1); - map.insert(times[3], 2); - map.insert(times[3], 3); - - assert_eq!( - map.iter().collect::>(), - vec![ - (×[0], &-1), - (×[0], &-2), - (×[1], &0), - (×[2], &1), - (×[3], &2), - (×[3], &3) - ] - ); - - assert_eq!(map.first(), Some((×[0], &vec![-1, -2]))); - - let drain = map.drain_until(&now); - assert_eq!( - drain.collect::>(), - vec![(times[0], -1), (times[0], -2), (times[1], 0),] - ); - assert_eq!( - map.iter().collect::>(), - vec![(×[2], &1), (×[3], &2), (×[3], &3)] - ); - } - - #[test] - fn base32() { - #[derive(Eq, PartialEq)] - struct Id([u8; 32]); - idbytes_impls!(Id, "Id"); - let id: Id = [1u8; 32].into(); - assert_eq!(id, Id::from_str(&format!("{id}")).unwrap()); - assert_eq!( - &format!("{id}"), - "aeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaqcaibaeaq" - ); - assert_eq!(&format!("{id:?}"), "Id(aeaqcaibaeaqcaib)"); - assert_eq!(id.as_bytes(), &[1u8; 32]); - } - - #[test] - fn time_bound_cache() { - let mut cache = TimeBoundCache::default(); - - let t0 = Instant::now(); - let t1 = t0 + Duration::from_secs(1); - let t2 = t0 + Duration::from_secs(2); - - cache.insert(1, 10, t0); - cache.insert(2, 20, t1); - cache.insert(3, 30, t1); - cache.insert(4, 40, t2); - - assert_eq!(cache.get(&2), Some(&20)); - assert_eq!(cache.len(), 4); - let removed = cache.expire_until(t1); - assert_eq!(removed, 3); - assert_eq!(cache.len(), 1); - assert_eq!(cache.get(&2), None); - assert_eq!(cache.get(&4), Some(&40)); - - let t3 = t2 + Duration::from_secs(1); - cache.insert(5, 50, t2); - assert_eq!(cache.expires(&5), Some(&t2)); - cache.insert(5, 50, t3); - assert_eq!(cache.expires(&5), Some(&t3)); - cache.expire_until(t2); - assert_eq!(cache.get(&4), None); - assert_eq!(cache.get(&5), Some(&50)); - } -} diff --git a/iroh-metrics/Cargo.toml b/iroh-metrics/Cargo.toml index 85c57bbef31..c500e3845ef 100644 --- a/iroh-metrics/Cargo.toml +++ b/iroh-metrics/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-metrics" -version = "0.27.0" +version = "0.28.0" edition = "2021" readme = "README.md" description = "metrics for iroh" diff --git a/iroh-metrics/src/core.rs b/iroh-metrics/src/core.rs index c1218a7af1b..ca7168548c8 100644 --- a/iroh-metrics/src/core.rs +++ b/iroh-metrics/src/core.rs @@ -183,8 +183,10 @@ impl Core { self.metrics_map.get::() } + /// Encodes the current metrics registry to a string in + /// the prometheus text exposition format. #[cfg(feature = "metrics")] - pub(crate) fn encode(&self) -> Result { + pub fn encode(&self) -> Result { let mut buf = String::new(); encode(&mut buf, &self.registry)?; Ok(buf) diff --git a/iroh-metrics/src/lib.rs b/iroh-metrics/src/lib.rs index d7fa5beaf3e..b0f920b8c05 100644 --- a/iroh-metrics/src/lib.rs +++ b/iroh-metrics/src/lib.rs @@ -76,3 +76,33 @@ pub fn parse_prometheus_metrics(data: &str) -> anyhow::Result, + /// The password for basic auth for the push metrics collector. + pub password: String, +} diff --git a/iroh-metrics/src/metrics.rs b/iroh-metrics/src/metrics.rs index 49f413454bf..a8502d377e1 100644 --- a/iroh-metrics/src/metrics.rs +++ b/iroh-metrics/src/metrics.rs @@ -13,8 +13,10 @@ //! //! # Example: //! ```rust -//! use iroh_metrics::{inc, inc_by}; -//! use iroh_metrics::core::{Core, Metric, Counter}; +//! use iroh_metrics::{ +//! core::{Core, Counter, Metric}, +//! inc, inc_by, +//! }; //! use struct_iterable::Iterable; //! //! #[derive(Debug, Clone, Iterable)] @@ -25,15 +27,17 @@ //! impl Default for Metrics { //! fn default() -> Self { //! Self { -//! things_added: Counter::new("things_added tracks the number of things we have added"), +//! things_added: Counter::new( +//! "things_added tracks the number of things we have added", +//! ), //! } //! } //! } //! //! impl Metric for Metrics { -//! fn name() -> &'static str { +//! fn name() -> &'static str { //! "my_metrics" -//! } +//! } //! } //! //! Core::init(|reg, metrics| { @@ -62,3 +66,17 @@ pub async fn start_metrics_dumper( ) -> anyhow::Result<()> { crate::service::dumper(&path, interval).await } + +/// Start a metrics exporter service. +#[cfg(feature = "metrics")] +pub async fn start_metrics_exporter(cfg: crate::PushMetricsConfig) { + crate::service::exporter( + cfg.endpoint, + cfg.service_name, + cfg.instance_name, + cfg.username, + cfg.password, + std::time::Duration::from_secs(cfg.interval), + ) + .await; +} diff --git a/iroh-metrics/src/service.rs b/iroh-metrics/src/service.rs index d4a35963bfd..3cbd5989cc3 100644 --- a/iroh-metrics/src/service.rs +++ b/iroh-metrics/src/service.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::{anyhow, Context, Result}; use hyper::{service::service_fn, Request, Response}; use tokio::{io::AsyncWriteExt as _, net::TcpListener}; -use tracing::{error, info}; +use tracing::{debug, error, info, warn}; use crate::{core::Core, parse_prometheus_metrics}; @@ -116,3 +116,53 @@ async fn dump_metrics( } Ok(()) } + +/// Export metrics to a push gateway. +pub async fn exporter( + gateway_endpoint: String, + service_name: String, + instance_name: String, + username: Option, + password: String, + interval: Duration, +) { + let Some(core) = Core::get() else { + error!("metrics disabled"); + return; + }; + let push_client = reqwest::Client::new(); + let prom_gateway_uri = format!( + "{}/metrics/job/{}/instance/{}", + gateway_endpoint, service_name, instance_name + ); + loop { + tokio::time::sleep(interval).await; + let buff = core.encode(); + match buff { + Err(e) => error!("Failed to encode metrics: {e:#}"), + Ok(buff) => { + let mut req = push_client.post(&prom_gateway_uri); + if let Some(username) = username.clone() { + req = req.basic_auth(username, Some(password.clone())); + } + let res = match req.body(buff).send().await { + Ok(res) => res, + Err(e) => { + warn!("failed to push metrics: {}", e); + continue; + } + }; + match res.status() { + reqwest::StatusCode::OK => { + debug!("pushed metrics to gateway"); + } + _ => { + warn!("failed to push metrics to gateway: {:?}", res); + let body = res.text().await.unwrap(); + warn!("error body: {}", body); + } + } + } + } + } +} diff --git a/iroh-net/Cargo.toml b/iroh-net/Cargo.toml index e94fb778a8b..b7a6f842a45 100644 --- a/iroh-net/Cargo.toml +++ b/iroh-net/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-net" -version = "0.27.0" +version = "0.28.1" edition = "2021" readme = "README.md" description = "networking support for iroh" @@ -16,11 +16,14 @@ rust-version = "1.76" workspace = true [dependencies] +cc = "=1.1.31" # enforce cc version, because of https://github.com/rust-lang/cc-rs/issues/1278 + anyhow = { version = "1" } -base64 = "0.22.1" +axum = { version = "0.7.4", optional = true } backoff = "0.4.0" +base64 = "0.22.1" bytes = "1.7" -netdev = "0.30.0" +clap = { version = "4", features = ["derive"], optional = true } der = { version = "0.7", features = ["alloc", "derive"] } derive_more = { version = "1.0.0", features = ["debug", "display", "from", "try_into", "deref"] } futures-buffered = "0.2.8" @@ -38,19 +41,24 @@ http-body-util = "0.1.0" hyper = { version = "1", features = ["server", "client", "http1"] } hyper-util = "0.1.1" igd-next = { version = "0.15.1", features = ["aio_tokio"] } -iroh-base = { version = "0.27.0", path = "../iroh-base", features = ["key"] } +iroh-base = { version = "0.28.0", features = ["key"] } +iroh-relay = { version = "0.28", path = "../iroh-relay" } libc = "0.2.139" +netdev = "0.30.0" +netwatch = { version = "0.1.0", path = "../net-tools/netwatch" } num_enum = "0.7" once_cell = "1.18.0" parking_lot = "0.12.1" pin-project = "1" pkarr = { version = "2", default-features = false, features = ["async", "relay"] } +portmapper = { version = "0.1.0", path = "../net-tools/portmapper" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -quinn = { package = "iroh-quinn", version = "0.11" } -quinn-proto = { package = "iroh-quinn-proto", version = "0.11" } -quinn-udp = { package = "iroh-quinn-udp", version = "0.5" } +quinn = { package = "iroh-quinn", version = "0.12.0" } +quinn-proto = { package = "iroh-quinn-proto", version = "0.12.0" } +quinn-udp = { package = "iroh-quinn-udp", version = "0.5.5" } rand = "0.8" rcgen = "0.12" +regex = { version = "1.7.1", optional = true } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } ring = "0.17" rustls = { version = "0.23", default-features = false, features = ["ring"] } @@ -63,11 +71,12 @@ thiserror = "1" time = "0.3.20" tokio = { version = "1", features = ["io-util", "macros", "sync", "rt", "net", "fs", "io-std", "signal", "process"] } tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring"] } +tokio-stream = { version = "0.1.15" } tokio-tungstenite = "0.21" tokio-tungstenite-wasm = "0.3" tokio-util = { version = "0.7.12", features = ["io-util", "io", "codec", "rt"] } -tokio-stream = { version = "0.1.15" } tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } tungstenite = "0.21" url = { version = "2.4", features = ["serde"] } watchable = "1.1.2" @@ -76,18 +85,8 @@ webpki-roots = "0.26" x509-parser = "0.16" z32 = "1.0.3" -# iroh-relay -axum = { version = "0.7.4", optional = true } -clap = { version = "4", features = ["derive"], optional = true } -regex = { version = "1.7.1", optional = true } -rustls-pemfile = { version = "2.1", optional = true } -serde_with = { version = "3.3", optional = true } -toml = { version = "0.8", optional = true } -tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } -tokio-rustls-acme = { version = "0.4", optional = true } - # metrics -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics", default-features = false } +iroh-metrics = { version = "0.28.0", default-features = false } strum = { version = "0.26.2", features = ["derive"] } # local-swarm-discovery @@ -111,15 +110,12 @@ axum = { version = "0.7.4" } clap = { version = "4", features = ["derive"] } criterion = "0.5.1" crypto_box = { version = "0.9.1", features = ["serde", "chacha20"] } -ntest = "0.9" pretty_assertions = "1.4" -proptest = "1.2.0" rand_chacha = "0.3.1" -testdir = "0.9.1" tokio = { version = "1", features = ["io-util", "sync", "rt", "net", "fs", "macros", "time", "test-util"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } -iroh-test = { path = "../iroh-test" } -iroh-net = { path = ".", features = ["iroh-relay"] } +iroh-test = "0.28.0" +iroh-net = { path = "." } serde_json = "1.0.107" testresult = "0.4.0" mainline = "2.0.1" @@ -133,25 +129,11 @@ duct = "0.13.6" [features] default = ["metrics", "discovery-pkarr-dht"] -iroh-relay = [ - "dep:tokio-rustls-acme", - "dep:axum", - "dep:clap", - "dep:toml", - "dep:rustls-pemfile", - "dep:regex", - "dep:serde_with", - "dep:tracing-subscriber" -] metrics = ["iroh-metrics/metrics"] -test-utils = ["iroh-relay"] +test-utils = ["iroh-relay/test-utils", "iroh-relay/server", "dep:axum"] discovery-local-network = ["dep:swarm-discovery"] discovery-pkarr-dht = ["pkarr/dht", "dep:genawaiter"] -[[bin]] -name = "iroh-relay" -required-features = ["iroh-relay"] - [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "iroh_docsrs"] diff --git a/iroh-net/bench/Cargo.toml b/iroh-net/bench/Cargo.toml index 4e6a7b54cb3..a4243e2641c 100644 --- a/iroh-net/bench/Cargo.toml +++ b/iroh-net/bench/Cargo.toml @@ -11,7 +11,7 @@ bytes = "1.7" hdrhistogram = { version = "7.2", default-features = false } iroh-net = { path = ".." } iroh-metrics = { path = "../../iroh-metrics" } -quinn = { package = "iroh-quinn", version = "0.11" } +quinn = { package = "iroh-quinn", version = "0.12" } rcgen = "0.12" rustls = { version = "0.23", default-features = false, features = ["ring"] } clap = { version = "4", features = ["derive"] } diff --git a/iroh-net/bench/src/iroh.rs b/iroh-net/bench/src/iroh.rs index 72a6ad62fa8..702d8dc3f8e 100644 --- a/iroh-net/bench/src/iroh.rs +++ b/iroh-net/bench/src/iroh.rs @@ -8,8 +8,7 @@ use bytes::Bytes; use futures_lite::StreamExt as _; use iroh_net::{ endpoint::{Connection, ConnectionError, RecvStream, SendStream, TransportConfig}, - relay::{RelayMap, RelayMode, RelayUrl}, - Endpoint, NodeAddr, + Endpoint, NodeAddr, RelayMap, RelayMode, RelayUrl, }; use tracing::{trace, warn}; diff --git a/iroh-net/examples/connect-unreliable.rs b/iroh-net/examples/connect-unreliable.rs index aa352dbbb37..188d4b03652 100644 --- a/iroh-net/examples/connect-unreliable.rs +++ b/iroh-net/examples/connect-unreliable.rs @@ -10,11 +10,7 @@ use std::net::SocketAddr; use anyhow::Context; use clap::Parser; use futures_lite::StreamExt; -use iroh_net::{ - key::SecretKey, - relay::{RelayMode, RelayUrl}, - Endpoint, NodeAddr, -}; +use iroh_net::{key::SecretKey, Endpoint, NodeAddr, RelayMode, RelayUrl}; use tracing::info; // An example ALPN that we are using to communicate over the `Endpoint` diff --git a/iroh-net/examples/connect.rs b/iroh-net/examples/connect.rs index 2693825a142..b20a12289e7 100644 --- a/iroh-net/examples/connect.rs +++ b/iroh-net/examples/connect.rs @@ -10,11 +10,7 @@ use std::net::SocketAddr; use anyhow::Context; use clap::Parser; use futures_lite::StreamExt; -use iroh_net::{ - key::SecretKey, - relay::{RelayMode, RelayUrl}, - Endpoint, NodeAddr, -}; +use iroh_net::{key::SecretKey, Endpoint, NodeAddr, RelayMode, RelayUrl}; use tracing::info; // An example ALPN that we are using to communicate over the `Endpoint` diff --git a/iroh-net/examples/listen-unreliable.rs b/iroh-net/examples/listen-unreliable.rs index f54a5c88626..a963d445547 100644 --- a/iroh-net/examples/listen-unreliable.rs +++ b/iroh-net/examples/listen-unreliable.rs @@ -5,7 +5,7 @@ //! $ cargo run --example listen-unreliable use anyhow::Context; use futures_lite::StreamExt; -use iroh_net::{key::SecretKey, relay::RelayMode, Endpoint}; +use iroh_net::{key::SecretKey, Endpoint, RelayMode}; use tracing::{info, warn}; // An example ALPN that we are using to communicate over the `Endpoint` diff --git a/iroh-net/examples/listen.rs b/iroh-net/examples/listen.rs index 1a3828c6e43..e0c2f25f801 100644 --- a/iroh-net/examples/listen.rs +++ b/iroh-net/examples/listen.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::Context; use futures_lite::StreamExt; -use iroh_net::{endpoint::ConnectionError, key::SecretKey, relay::RelayMode, Endpoint}; +use iroh_net::{endpoint::ConnectionError, key::SecretKey, Endpoint, RelayMode}; use tracing::{debug, info, warn}; // An example ALPN that we are using to communicate over the `Endpoint` diff --git a/iroh-net/src/defaults.rs b/iroh-net/src/defaults.rs index 95ab1029539..1d822c12e01 100644 --- a/iroh-net/src/defaults.rs +++ b/iroh-net/src/defaults.rs @@ -2,7 +2,7 @@ use url::Url; -use crate::relay::{RelayMap, RelayNode}; +use crate::{RelayMap, RelayNode}; /// The default STUN port used by the Relay server. /// @@ -162,41 +162,6 @@ pub(crate) mod timeouts { /// The amount of time we wait for a hairpinned packet to come back. pub(crate) const HAIRPIN_CHECK_TIMEOUT: Duration = Duration::from_millis(100); - /// Maximum duration a UPnP search can take before timing out. - pub(crate) const UPNP_SEARCH_TIMEOUT: Duration = Duration::from_secs(1); - - /// Timeout to receive a response from a PCP server. - pub(crate) const PCP_RECV_TIMEOUT: Duration = Duration::from_millis(500); - /// Default Pinger timeout pub(crate) const DEFAULT_PINGER_TIMEOUT: Duration = Duration::from_secs(5); - - /// Timeout to receive a response from a NAT-PMP server. - pub(crate) const NAT_PMP_RECV_TIMEOUT: Duration = Duration::from_millis(500); - - /// Timeouts specifically used in the iroh-relay - pub(crate) mod relay { - use super::*; - - /// Timeout used by the relay client while connecting to the relay server, - /// using `TcpStream::connect` - pub(crate) const DIAL_NODE_TIMEOUT: Duration = Duration::from_millis(1500); - /// Timeout for expecting a pong from the relay server - pub(crate) const PING_TIMEOUT: Duration = Duration::from_secs(5); - /// Timeout for the entire relay connection, which includes dns, dialing - /// the server, upgrading the connection, and completing the handshake - pub(crate) const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); - /// Timeout for our async dns resolver - pub(crate) const DNS_TIMEOUT: Duration = Duration::from_secs(1); - - /// Maximum time the client will wait to receive on the connection, since - /// the last message. Longer than this time and the client will consider - /// the connection dead. - pub(crate) const CLIENT_RECV_TIMEOUT: Duration = Duration::from_secs(120); - - /// Maximum time the server will attempt to get a successful write to the connection. - #[cfg(feature = "iroh-relay")] - #[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] - pub(crate) const SERVER_WRITE_TIMEOUT: Duration = Duration::from_secs(2); - } } diff --git a/iroh-net/src/disco.rs b/iroh-net/src/disco.rs index c5d2ec89176..29a15258848 100644 --- a/iroh-net/src/disco.rs +++ b/iroh-net/src/disco.rs @@ -24,11 +24,12 @@ use std::{ }; use anyhow::{anyhow, bail, ensure, Context, Result}; +use iroh_relay::RelayUrl; use serde::{Deserialize, Serialize}; use url::Url; -use super::{key::PublicKey, stun}; -use crate::{key, relay::RelayUrl}; +use super::key::PublicKey; +use crate::key; // TODO: custom magicn /// The 6 byte header of all discovery messages. @@ -114,7 +115,7 @@ pub enum Message { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Ping { /// Random client-generated per-ping transaction ID. - pub tx_id: stun::TransactionId, + pub tx_id: stun_rs::TransactionId, /// Allegedly the ping sender's wireguard public key. /// It shouldn't be trusted by itself, but can be combined with @@ -127,7 +128,7 @@ pub struct Ping { /// It includes the sender's source IP + port, so it's effectively a STUN response. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Pong { - pub tx_id: stun::TransactionId, + pub tx_id: stun_rs::TransactionId, /// The observed address off the ping sender. /// /// 18 bytes (16+2) on the wire; v4-mapped ipv6 for IPv4. @@ -210,7 +211,7 @@ impl Ping { let tx_id: [u8; TX_LEN] = p[..TX_LEN].try_into().expect("length checked"); let raw_key = &p[TX_LEN..TX_LEN + key::PUBLIC_KEY_LENGTH]; let node_key = PublicKey::try_from(raw_key)?; - let tx_id = stun::TransactionId::from(tx_id); + let tx_id = stun_rs::TransactionId::from(tx_id); Ok(Ping { tx_id, node_key }) } @@ -290,7 +291,7 @@ impl Pong { fn from_bytes(ver: u8, p: &[u8]) -> Result { ensure!(ver == V0, "invalid version"); let tx_id: [u8; TX_LEN] = p[..TX_LEN].try_into().context("message too short")?; - let tx_id = stun::TransactionId::from(tx_id); + let tx_id = stun_rs::TransactionId::from(tx_id); let src = send_addr_from_bytes(&p[TX_LEN..])?; Ok(Pong { @@ -476,7 +477,7 @@ mod tests { let recv_key = SecretKey::generate(); let msg = Message::Ping(Ping { - tx_id: stun::TransactionId::default(), + tx_id: stun_rs::TransactionId::default(), node_key: sender_key.public(), }); diff --git a/iroh-net/src/discovery.rs b/iroh-net/src/discovery.rs index 29fdb146b1f..d05e07eb133 100644 --- a/iroh-net/src/discovery.rs +++ b/iroh-net/src/discovery.rs @@ -46,11 +46,11 @@ //! [`PkarrPublisher`] and [`DnsDiscovery`]: //! //! ```no_run -//! use iroh_net::discovery::dns::DnsDiscovery; -//! use iroh_net::discovery::pkarr::PkarrPublisher; -//! use iroh_net::discovery::ConcurrentDiscovery; -//! use iroh_net::key::SecretKey; -//! use iroh_net::Endpoint; +//! use iroh_net::{ +//! discovery::{dns::DnsDiscovery, pkarr::PkarrPublisher, ConcurrentDiscovery}, +//! key::SecretKey, +//! Endpoint, +//! }; //! //! # async fn wrapper() -> anyhow::Result<()> { //! let secret_key = SecretKey::generate(); @@ -114,6 +114,7 @@ pub mod dns; #[cfg_attr(iroh_docsrs, doc(cfg(feature = "discovery-local-network")))] pub mod local_swarm_discovery; pub mod pkarr; +pub mod static_provider; /// Node discovery for [`super::Endpoint`]. /// @@ -443,7 +444,7 @@ mod tests { use tokio_util::task::AbortOnDropHandle; use super::*; - use crate::{key::SecretKey, relay::RelayMode}; + use crate::{key::SecretKey, RelayMode}; #[derive(Debug, Clone, Default)] struct TestDiscoveryShared { @@ -735,13 +736,12 @@ mod test_dns_pkarr { use crate::{ discovery::pkarr::PkarrPublisher, dns::{node_info::NodeInfo, ResolverExt}, - relay::{RelayMap, RelayMode}, test_utils::{ dns_server::{create_dns_resolver, run_dns_server}, pkarr_dns_state::State, run_relay_server, DnsPkarrServer, }, - AddrInfo, Endpoint, NodeAddr, + AddrInfo, Endpoint, NodeAddr, RelayMap, RelayMode, }; const PUBLISH_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/iroh-net/src/discovery/dns.rs b/iroh-net/src/discovery/dns.rs index 4a00fd37b7e..1d5d74d4bad 100644 --- a/iroh-net/src/discovery/dns.rs +++ b/iroh-net/src/discovery/dns.rs @@ -6,7 +6,7 @@ use futures_lite::stream::Boxed as BoxStream; use crate::{ discovery::{Discovery, DiscoveryItem}, dns::ResolverExt, - relay::force_staging_infra, + endpoint::force_staging_infra, Endpoint, NodeId, }; diff --git a/iroh-net/src/discovery/local_swarm_discovery.rs b/iroh-net/src/discovery/local_swarm_discovery.rs index 8097d2203e8..6d7dc791105 100644 --- a/iroh-net/src/discovery/local_swarm_discovery.rs +++ b/iroh-net/src/discovery/local_swarm_discovery.rs @@ -7,29 +7,28 @@ //! //! ``` //! use std::time::Duration; -//! use iroh_net::endpoint::{Source, Endpoint}; +//! +//! use iroh_net::endpoint::{Endpoint, Source}; //! //! #[tokio::main] //! async fn main() { -//! let recent = Duration::from_secs(600); // 10 minutes in seconds +//! let recent = Duration::from_secs(600); // 10 minutes in seconds //! -//! let endpoint = Endpoint::builder().bind().await.unwrap(); -//! let remotes = endpoint.remote_info_iter(); -//! let locally_discovered: Vec<_> = remotes -//! .filter(|remote| { -//! remote -//! .sources() -//! .iter() -//! .any(|(source, duration)| { -//! if let Source::Discovery { name } = source { -//! name == iroh_net::discovery::local_swarm_discovery::NAME && *duration <= recent -//! } else { -//! false -//! } -//! }) -//! }) -//! .collect(); -//! println!("locally discovered nodes: {locally_discovered:?}"); +//! let endpoint = Endpoint::builder().bind().await.unwrap(); +//! let remotes = endpoint.remote_info_iter(); +//! let locally_discovered: Vec<_> = remotes +//! .filter(|remote| { +//! remote.sources().iter().any(|(source, duration)| { +//! if let Source::Discovery { name } = source { +//! name == iroh_net::discovery::local_swarm_discovery::NAME +//! && *duration <= recent +//! } else { +//! false +//! } +//! }) +//! }) +//! .collect(); +//! println!("locally discovered nodes: {locally_discovered:?}"); //! } //! ``` use std::{ diff --git a/iroh-net/src/discovery/pkarr.rs b/iroh-net/src/discovery/pkarr.rs index 6cf3ab25061..c53904d483f 100644 --- a/iroh-net/src/discovery/pkarr.rs +++ b/iroh-net/src/discovery/pkarr.rs @@ -60,8 +60,8 @@ use watchable::{Watchable, Watcher}; use crate::{ discovery::{Discovery, DiscoveryItem}, dns::node_info::NodeInfo, + endpoint::force_staging_infra, key::SecretKey, - relay::force_staging_infra, AddrInfo, Endpoint, NodeId, }; diff --git a/iroh-net/src/discovery/pkarr/dht.rs b/iroh-net/src/discovery/pkarr/dht.rs index 3063ae64f03..a2d3ddc541b 100644 --- a/iroh-net/src/discovery/pkarr/dht.rs +++ b/iroh-net/src/discovery/pkarr/dht.rs @@ -413,7 +413,7 @@ mod tests { #[tokio::test] #[ignore = "flaky"] async fn dht_discovery_smoke() -> TestResult { - let _ = tracing_subscriber::fmt::try_init(); + let _logging_guard = iroh_test::logging::setup(); let ep = crate::Endpoint::builder().bind().await?; let secret = ep.secret_key().clone(); let testnet = mainline::dht::Testnet::new(2); @@ -438,19 +438,27 @@ mod tests { }); // publish is fire and forget, so we have no way to wait until it is done. - tokio::time::sleep(Duration::from_secs(1)).await; - let items = discovery - .resolve(ep, secret.public()) - .unwrap() - .collect::>() - .await; - let mut found_relay_urls = BTreeSet::new(); - for item in items.into_iter().flatten() { - if let Some(url) = item.addr_info.relay_url { - found_relay_urls.insert(url); + tokio::time::timeout(Duration::from_secs(30), async move { + loop { + tokio::time::sleep(Duration::from_millis(200)).await; + let mut found_relay_urls = BTreeSet::new(); + let items = discovery + .resolve(ep.clone(), secret.public()) + .unwrap() + .collect::>() + .await; + for item in items.into_iter().flatten() { + if let Some(url) = item.addr_info.relay_url { + found_relay_urls.insert(url); + } + } + if found_relay_urls.contains(&relay_url) { + break; + } } - } - assert!(found_relay_urls.contains(&relay_url)); + }) + .await + .expect("timeout, relay_url not found on DHT"); Ok(()) } } diff --git a/iroh-net/src/discovery/static_provider.rs b/iroh-net/src/discovery/static_provider.rs new file mode 100644 index 00000000000..a6169d2b1d3 --- /dev/null +++ b/iroh-net/src/discovery/static_provider.rs @@ -0,0 +1,162 @@ +//! A static discovery implementation that allows adding info for nodes manually. +use std::{ + collections::{btree_map::Entry, BTreeMap}, + sync::{Arc, RwLock}, + time::SystemTime, +}; + +use futures_lite::stream::{self, StreamExt}; +use iroh_base::{ + key::NodeId, + node_addr::{AddrInfo, NodeAddr}, +}; + +use super::{Discovery, DiscoveryItem}; + +/// A static discovery implementation that allows providing info for nodes manually. +#[derive(Debug, Default)] +#[repr(transparent)] +pub struct StaticProvider { + nodes: Arc>>, +} + +#[derive(Debug)] +struct NodeInfo { + info: AddrInfo, + last_updated: SystemTime, +} + +impl StaticProvider { + /// The provenance string for this discovery implementation. + pub const PROVENANCE: &'static str = "static_discovery"; + + /// Create a new static discovery instance. + pub fn new() -> Self { + Self::default() + } + + /// Creates a static discovery instance from something that can be converted into node addresses. + /// + /// Example: + /// ```rust + /// use std::str::FromStr; + /// + /// use iroh_base::ticket::NodeTicket; + /// use iroh_net::{Endpoint, discovery::static_provider::StaticProvider}; + /// + /// # async fn example() -> anyhow::Result<()> { + /// # #[derive(Default)] struct Args { tickets: Vec } + /// # let args = Args::default(); + /// // get tickets from command line args + /// let tickets: Vec = args.tickets; + /// // create a StaticProvider from the tickets. Ticket info will be combined if multiple tickets refer to the same node. + /// let discovery = StaticProvider::from_node_addrs(tickets); + /// // create an endpoint with the discovery + /// let endpoint = Endpoint::builder() + /// .add_discovery(|_| Some(discovery)) + /// .bind().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn from_node_addrs(infos: impl IntoIterator>) -> Self { + let res = Self::default(); + for info in infos { + res.add_node_addr(info); + } + res + } + + /// Add node info for the given node id. + /// + /// This will completely overwrite any existing info for the node. + pub fn set_node_addr(&self, info: impl Into) -> Option { + let last_updated = SystemTime::now(); + let info: NodeAddr = info.into(); + let mut guard = self.nodes.write().unwrap(); + let previous = guard.insert( + info.node_id, + NodeInfo { + info: info.info, + last_updated, + }, + ); + previous.map(|x| NodeAddr { + node_id: info.node_id, + info: x.info, + }) + } + + /// Add node info for the given node id, combining it with any existing info. + /// + /// This will add any new direct addresses and overwrite the relay url. + pub fn add_node_addr(&self, info: impl Into) { + let info: NodeAddr = info.into(); + let last_updated = SystemTime::now(); + let mut guard = self.nodes.write().unwrap(); + match guard.entry(info.node_id) { + Entry::Occupied(mut entry) => { + let existing = entry.get_mut(); + existing + .info + .direct_addresses + .extend(info.info.direct_addresses); + existing.info.relay_url = info.info.relay_url; + existing.last_updated = last_updated; + } + Entry::Vacant(entry) => { + entry.insert(NodeInfo { + info: info.info, + last_updated, + }); + } + } + } + + /// Get node info for the given node id. + pub fn get_node_addr(&self, node_id: NodeId) -> Option { + let guard = self.nodes.read().unwrap(); + let info = guard.get(&node_id).map(|x| x.info.clone())?; + Some(NodeAddr { node_id, info }) + } + + /// Remove node info for the given node id. + pub fn remove_node_addr(&self, node_id: NodeId) -> Option { + let mut guard = self.nodes.write().unwrap(); + let res = guard.remove(&node_id)?; + Some(NodeAddr { + node_id, + info: res.info, + }) + } +} + +impl Discovery for StaticProvider { + fn publish(&self, _info: &AddrInfo) {} + + fn resolve( + &self, + _endpoint: crate::Endpoint, + node_id: NodeId, + ) -> Option>> { + let guard = self.nodes.read().unwrap(); + let info = guard.get(&node_id); + match info { + Some(addr_info) => { + let item = DiscoveryItem { + node_id, + provenance: Self::PROVENANCE, + last_updated: Some( + addr_info + .last_updated + .duration_since(SystemTime::UNIX_EPOCH) + .expect("time drift") + .as_micros() as u64, + ), + addr_info: addr_info.info.clone(), + }; + Some(stream::iter(Some(Ok(item))).boxed()) + } + None => None, + } + } +} diff --git a/iroh-net/src/endpoint.rs b/iroh-net/src/endpoint.rs index 1f334ad38fb..8ffcbeba8aa 100644 --- a/iroh-net/src/endpoint.rs +++ b/iroh-net/src/endpoint.rs @@ -36,16 +36,28 @@ use crate::{ dns::{default_resolver, DnsResolver}, key::{PublicKey, SecretKey}, magicsock::{self, Handle, QuicMappedAddr}, - relay::{force_staging_infra, RelayMode, RelayUrl}, - tls, NodeId, + tls, NodeId, RelayMode, RelayUrl, }; mod rtt_actor; +pub use bytes::Bytes; pub use iroh_base::node_addr::{AddrInfo, NodeAddr}; +// Missing still: SendDatagram and ConnectionClose::frame_type's Type. pub use quinn::{ - ApplicationClose, Connection, ConnectionClose, ConnectionError, ReadError, ReadExactError, - RecvStream, RetryError, SendStream, ServerConfig, TransportConfig, VarInt, WriteError, + AcceptBi, AcceptUni, AckFrequencyConfig, ApplicationClose, Chunk, ClosedStream, Connection, + ConnectionClose, ConnectionError, ConnectionStats, MtuDiscoveryConfig, OpenBi, OpenUni, + ReadDatagram, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, RetryError, + SendDatagramError, SendStream, ServerConfig, StoppedError, StreamId, TransportConfig, VarInt, + WeakConnectionHandle, WriteError, ZeroRttAccepted, +}; +pub use quinn_proto::{ + congestion::{Controller, ControllerFactory}, + crypto::{ + AeadKey, CryptoError, ExportKeyingMaterialError, HandshakeTokenKey, + ServerConfig as CryptoServerConfig, UnsupportedVersion, + }, + FrameStats, PathStats, TransportError, TransportErrorCode, UdpStats, Written, }; use self::rtt_actor::RttMessage; @@ -414,7 +426,7 @@ struct StaticConfig { impl StaticConfig { /// Create a [`quinn::ServerConfig`] with the specified ALPN protocols. - fn create_server_config(&self, alpn_protocols: Vec>) -> Result { + fn create_server_config(&self, alpn_protocols: Vec>) -> Result { let server_config = make_server_config( &self.secret_key, alpn_protocols, @@ -425,18 +437,18 @@ impl StaticConfig { } } -/// Creates a [`quinn::ServerConfig`] with the given secret key and limits. +/// Creates a [`ServerConfig`] with the given secret key and limits. // This return type can not longer be used anywhere in our public API. It is however still // used by iroh::node::Node (or rather iroh::node::Builder) to create a plain Quinn // endpoint. pub fn make_server_config( secret_key: &SecretKey, alpn_protocols: Vec>, - transport_config: Arc, + transport_config: Arc, keylog: bool, -) -> Result { +) -> Result { let quic_server_config = tls::make_server_config(secret_key, alpn_protocols, keylog)?; - let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_server_config)); + let mut server_config = ServerConfig::with_crypto(Arc::new(quic_server_config)); server_config.transport_config(transport_config); Ok(server_config) @@ -560,11 +572,7 @@ impl Endpoint { /// endpoint must support this `alpn`, otherwise the connection attempt will fail with /// an error. #[instrument(skip_all, fields(me = %self.node_id().fmt_short(), alpn = ?String::from_utf8_lossy(alpn)))] - pub async fn connect( - &self, - node_addr: impl Into, - alpn: &[u8], - ) -> Result { + pub async fn connect(&self, node_addr: impl Into, alpn: &[u8]) -> Result { let node_addr = node_addr.into(); tracing::Span::current().record("remote", node_addr.node_id.fmt_short()); // Connecting to ourselves is not supported. @@ -621,11 +629,7 @@ impl Endpoint { since = "0.27.0", note = "Please use `connect` directly with a NodeId. This fn will be removed in 0.28.0." )] - pub async fn connect_by_node_id( - &self, - node_id: NodeId, - alpn: &[u8], - ) -> Result { + pub async fn connect_by_node_id(&self, node_id: NodeId, alpn: &[u8]) -> Result { let addr = NodeAddr::new(node_id); self.connect(addr, alpn).await } @@ -639,7 +643,7 @@ impl Endpoint { node_id: NodeId, alpn: &[u8], addr: QuicMappedAddr, - ) -> Result { + ) -> Result { debug!("Attempting connection..."); let client_config = { let alpn_protocols = vec![alpn.to_vec()]; @@ -840,7 +844,7 @@ impl Endpoint { /// /// # let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); /// # rt.block_on(async move { - /// let mep = Endpoint::builder().bind().await.unwrap(); + /// let mep = Endpoint::builder().bind().await.unwrap(); /// let _addrs = mep.direct_addresses().next().await; /// # }); /// ``` @@ -952,8 +956,10 @@ impl Endpoint { /// This will close all open QUIC connections with the provided error_code and /// reason. See [`quinn::Connection`] for details on how these are interpreted. /// - /// It will then wait for all connections to actually be shutdown, and afterwards - /// close the magic socket. + /// It will then wait for all connections to actually be shutdown, and afterwards close + /// the magic socket. Be aware however that the underlying UDP sockets are only closed + /// on [`Drop`], bearing in mind the [`Endpoint`] is only dropped once all the clones + /// are dropped. /// /// Returns an error if closing the magic socket failed. /// TODO: Document error cases. @@ -1210,7 +1216,7 @@ pub struct Connecting { impl Connecting { /// Convert into a 0-RTT or 0.5-RTT connection at the cost of weakened security. - pub fn into_0rtt(self) -> Result<(quinn::Connection, quinn::ZeroRttAccepted), Self> { + pub fn into_0rtt(self) -> Result<(Connection, ZeroRttAccepted), Self> { match self.inner.into_0rtt() { Ok((conn, zrtt_accepted)) => { try_send_rtt_msg(&conn, &self.ep); @@ -1221,7 +1227,7 @@ impl Connecting { } /// Parameters negotiated during the handshake - pub async fn handshake_data(&mut self) -> Result, quinn::ConnectionError> { + pub async fn handshake_data(&mut self) -> Result, ConnectionError> { self.inner.handshake_data().await } @@ -1251,7 +1257,7 @@ impl Connecting { } impl Future for Connecting { - type Output = Result; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { let this = self.project(); @@ -1268,7 +1274,7 @@ impl Future for Connecting { /// Extract the [`PublicKey`] from the peer's TLS certificate. // TODO: make this a method now -pub fn get_remote_node_id(connection: &quinn::Connection) -> Result { +pub fn get_remote_node_id(connection: &Connection) -> Result { let data = connection.peer_identity(); match data { None => bail!("no peer certificate found"), @@ -1292,7 +1298,7 @@ pub fn get_remote_node_id(connection: &quinn::Connection) -> Result { /// /// If we can't notify the actor that will impact performance a little, but we can still /// function. -fn try_send_rtt_msg(conn: &quinn::Connection, magic_ep: &Endpoint) { +fn try_send_rtt_msg(conn: &Connection, magic_ep: &Endpoint) { // If we can't notify the rtt-actor that's not great but not critical. let Ok(peer_id) = get_remote_node_id(conn) else { warn!(?conn, "failed to get remote node id"); @@ -1351,6 +1357,15 @@ fn proxy_url_from_env() -> Option { None } +/// Environment variable to force the use of staging relays. +#[cfg_attr(iroh_docsrs, doc(cfg(not(test))))] +pub const ENV_FORCE_STAGING_RELAYS: &str = "IROH_FORCE_STAGING_RELAYS"; + +/// Returns `true` if the use of staging relays is forced. +pub fn force_staging_infra() -> bool { + matches!(std::env::var(ENV_FORCE_STAGING_RELAYS), Ok(value) if !value.is_empty()) +} + /// Returns the default relay mode. /// /// If the `IROH_FORCE_STAGING_RELAYS` environment variable is non empty, it will return `RelayMode::Staging`. @@ -1383,7 +1398,7 @@ mod tests { use tracing::{error_span, info, info_span, Instrument}; use super::*; - use crate::test_utils::run_relay_server; + use crate::test_utils::{run_relay_server, run_relay_server_with}; const TEST_ALPN: &[u8] = b"n0/iroh/test"; @@ -1505,7 +1520,7 @@ mod tests { ); let (server, client) = tokio::time::timeout( - Duration::from_secs(5), + Duration::from_secs(30), futures_lite::future::zip(server, client), ) .await @@ -1849,4 +1864,23 @@ mod tests { r1.expect("ep1 timeout").unwrap(); r2.expect("ep2 timeout").unwrap(); } + + #[tokio::test] + async fn test_direct_addresses_no_stun_relay() { + let _guard = iroh_test::logging::setup(); + let (relay_map, _, _guard) = run_relay_server_with(None).await.unwrap(); + + let ep = Endpoint::builder() + .alpns(vec![TEST_ALPN.to_vec()]) + .relay_mode(RelayMode::Custom(relay_map)) + .insecure_skip_relay_cert_verify(true) + .bind() + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(10), ep.direct_addresses().next()) + .await + .unwrap() + .unwrap(); + } } diff --git a/iroh-net/src/endpoint/rtt_actor.rs b/iroh-net/src/endpoint/rtt_actor.rs index 48dfd6f1e40..d72fdc61dd5 100644 --- a/iroh-net/src/endpoint/rtt_actor.rs +++ b/iroh-net/src/endpoint/rtt_actor.rs @@ -5,19 +5,23 @@ use std::collections::HashMap; use futures_concurrency::stream::stream_group; use futures_lite::StreamExt; use iroh_base::key::NodeId; +use iroh_metrics::inc; use tokio::{ sync::{mpsc, Notify}, - task::JoinHandle, time::Duration, }; -use tracing::{debug, error, info_span, trace, warn, Instrument}; +use tokio_util::task::AbortOnDropHandle; +use tracing::{debug, error, info_span, trace, Instrument}; -use crate::magicsock::{ConnectionType, ConnectionTypeStream}; +use crate::{ + magicsock::{ConnectionType, ConnectionTypeStream}, + metrics::MagicsockMetrics, +}; #[derive(Debug)] pub(super) struct RttHandle { // We should and some point use this to propagate panics and errors. - pub(super) _handle: JoinHandle<()>, + pub(super) _handle: AbortOnDropHandle<()>, pub(super) msg_tx: mpsc::Sender, } @@ -29,13 +33,16 @@ impl RttHandle { tick: Notify::new(), }; let (msg_tx, msg_rx) = mpsc::channel(16); - let _handle = tokio::spawn( + let handle = tokio::spawn( async move { actor.run(msg_rx).await; } .instrument(info_span!("rtt-actor")), ); - Self { _handle, msg_tx } + Self { + _handle: AbortOnDropHandle::new(handle), + msg_tx, + } } } @@ -65,7 +72,9 @@ struct RttActor { /// /// These are weak references so not to keep the connections alive. The key allows /// removing the corresponding stream from `conn_type_changes`. - connections: HashMap, + /// The boolean is an indiciator of whether this connection was direct before. + /// This helps establish metrics on number of connections that became direct. + connections: HashMap, /// A way to notify the main actor loop to run over. /// /// E.g. when a new stream was added. @@ -81,12 +90,18 @@ impl RttActor { cleanup_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { - Some(msg) = msg_rx.recv() => self.handle_msg(msg), - item = self.connection_events.next(), - if !self.connection_events.is_empty() => self.do_reset_rtt(item), + biased; + msg = msg_rx.recv() => { + match msg { + Some(msg) => self.handle_msg(msg), + None => break, + } + } + item = self.connection_events.next(), if !self.connection_events.is_empty() => { + self.do_reset_rtt(item); + } _ = cleanup_interval.tick() => self.do_connections_cleanup(), () = self.tick.notified() => continue, - else => break, } } debug!("rtt-actor finished"); @@ -113,8 +128,9 @@ impl RttActor { node_id: NodeId, ) { let key = self.connection_events.insert(conn_type_changes); - self.connections.insert(key, (connection, node_id)); + self.connections.insert(key, (connection, node_id, false)); self.tick.notify_one(); + inc!(MagicsockMetrics, connection_handshake_success); } /// Performs the congestion controller reset for a magic socket path change. @@ -125,14 +141,19 @@ impl RttActor { /// happens commonly. fn do_reset_rtt(&mut self, item: Option<(stream_group::Key, ConnectionType)>) { match item { - Some((key, new_conn_type)) => match self.connections.get(&key) { - Some((handle, node_id)) => { - if handle.reset_congestion_state() { + Some((key, new_conn_type)) => match self.connections.get_mut(&key) { + Some((handle, node_id, was_direct_before)) => { + if handle.network_path_changed() { debug!( node_id = %node_id.fmt_short(), new_type = ?new_conn_type, "Congestion controller state reset", ); + if !*was_direct_before && matches!(new_conn_type, ConnectionType::Direct(_)) + { + *was_direct_before = true; + inc!(MagicsockMetrics, connection_became_direct); + } } else { debug!( node_id = %node_id.fmt_short(), @@ -144,14 +165,14 @@ impl RttActor { None => error!("No connection found for stream item"), }, None => { - warn!("self.conn_type_changes is empty but was polled"); + trace!("No more connections"); } } } /// Performs cleanup for closed connection. fn do_connections_cleanup(&mut self) { - for (key, (handle, node_id)) in self.connections.iter() { + for (key, (handle, node_id, _)) in self.connections.iter() { if !handle.is_alive() { trace!(node_id = %node_id.fmt_short(), "removing stale connection"); self.connection_events.remove(*key); @@ -159,3 +180,29 @@ impl RttActor { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_actor_mspc_close() { + let mut actor = RttActor { + connection_events: stream_group::StreamGroup::new().keyed(), + connections: HashMap::new(), + tick: Notify::new(), + }; + let (msg_tx, msg_rx) = mpsc::channel(16); + let handle = tokio::spawn(async move { + actor.run(msg_rx).await; + }); + + // Dropping the msg_tx should stop the actor + drop(msg_tx); + + let task_res = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("timeout - actor did not finish"); + assert!(task_res.is_ok()); + } +} diff --git a/iroh-net/src/lib.rs b/iroh-net/src/lib.rs index 759eaefa651..dcb1441120f 100644 --- a/iroh-net/src/lib.rs +++ b/iroh-net/src/lib.rs @@ -4,7 +4,8 @@ //! interface to [QUIC] connections and streams to the user, while implementing direct //! connectivity using [hole punching] complemented by relay servers under the hood. //! -//! Connecting to a remote node looks roughly like this: +//! An iroh-net node is created and controlled by the [`Endpoint`], e.g. connecting to +//! another node: //! //! ```no_run //! # use iroh_net::{Endpoint, NodeAddr}; @@ -18,7 +19,7 @@ //! # } //! ``` //! -//! The other side can accept incoming connections like this: +//! The other node can accept incoming connections using the [`Endpoint`] as well: //! //! ```no_run //! # use iroh_net::{Endpoint, NodeAddr}; @@ -183,8 +184,7 @@ //! ```no_run //! use anyhow::{Context, Result}; //! use futures_lite::StreamExt; -//! use iroh_net::ticket::NodeTicket; -//! use iroh_net::{Endpoint, NodeAddr}; +//! use iroh_net::{ticket::NodeTicket, Endpoint, NodeAddr}; //! //! async fn accept() -> Result<()> { //! // To accept connections at least one ALPN must be configured. @@ -224,7 +224,7 @@ //! [`discovery`]: crate::endpoint::Builder::discovery //! [`DnsDiscovery`]: crate::discovery::dns::DnsDiscovery //! [number 0]: https://n0.computer -//! [`RelayMode::Default`]: crate::relay::RelayMode::Default +//! [`RelayMode::Default`]: crate::RelayMode::Default //! [the discovery module]: crate::discovery //! [`Connection::open_bi`]: crate::endpoint::Connection::open_bi //! [`Connection::accept_bi`]: crate::endpoint::Connection::accept_bi @@ -241,18 +241,18 @@ pub mod dns; pub mod endpoint; mod magicsock; pub mod metrics; -pub mod net; pub mod netcheck; pub mod ping; -pub mod portmapper; -pub mod relay; -pub mod stun; +mod relay_map; pub mod ticket; pub mod tls; + pub(crate) mod util; pub use endpoint::{AddrInfo, Endpoint, NodeAddr}; pub use iroh_base::{key, key::NodeId}; +pub use iroh_relay as relay; +pub use relay_map::{RelayMap, RelayMode, RelayNode, RelayUrl}; #[cfg(any(test, feature = "test-utils"))] #[cfg_attr(iroh_docsrs, doc(cfg(any(test, feature = "test-utils"))))] diff --git a/iroh-net/src/magicsock.rs b/iroh-net/src/magicsock.rs index 1a9341e18fe..4dc720d7217 100644 --- a/iroh-net/src/magicsock.rs +++ b/iroh-net/src/magicsock.rs @@ -35,6 +35,8 @@ use futures_lite::{FutureExt, Stream, StreamExt}; use futures_util::stream::BoxStream; use iroh_base::key::NodeId; use iroh_metrics::{inc, inc_by}; +use iroh_relay::protos::stun; +use netwatch::{interfaces, ip::LocalAddresses, netmon}; use quinn::AsyncUdpSocket; use rand::{seq::SliceRandom, Rng, SeedableRng}; use smallvec::{smallvec, SmallVec}; @@ -64,10 +66,7 @@ use crate::{ dns::DnsResolver, endpoint::NodeAddr, key::{PublicKey, SecretKey, SharedSecret}, - net::{interfaces, ip::LocalAddresses, netmon}, - netcheck, portmapper, - relay::{RelayMap, RelayUrl}, - stun, AddrInfo, + netcheck, AddrInfo, RelayMap, RelayUrl, }; mod metrics; @@ -1699,7 +1698,7 @@ impl quinn::UdpPoller for IoPoller { enum ActorMessage { Shutdown, ReceiveRelay(RelayReadResult), - EndpointPingExpired(usize, stun::TransactionId), + EndpointPingExpired(usize, stun_rs::TransactionId), NetcheckReport(Result>>, &'static str), NetworkChange, #[cfg(test)] @@ -2784,7 +2783,7 @@ mod tests { use tokio_util::task::AbortOnDropHandle; use super::*; - use crate::{defaults::staging::EU_RELAY_HOSTNAME, relay::RelayMode, tls, Endpoint}; + use crate::{defaults::staging::EU_RELAY_HOSTNAME, tls, Endpoint, RelayMode}; const ALPN: &[u8] = b"n0/test/1"; diff --git a/iroh-net/src/magicsock/metrics.rs b/iroh-net/src/magicsock/metrics.rs index 5c0a97148f4..90b5ae9d471 100644 --- a/iroh-net/src/magicsock/metrics.rs +++ b/iroh-net/src/magicsock/metrics.rs @@ -6,6 +6,7 @@ use iroh_metrics::{ /// Enum of metrics for the module #[allow(missing_docs)] #[derive(Debug, Clone, Iterable)] +#[non_exhaustive] pub struct Metrics { pub re_stun_calls: Counter, pub update_direct_addrs: Counter, @@ -68,6 +69,16 @@ pub struct Metrics { pub actor_tick_direct_addr_update_receiver: Counter, pub actor_link_change: Counter, pub actor_tick_other: Counter, + + /// Number of nodes we have attempted to contact. + pub nodes_contacted: Counter, + /// Number of nodes we have managed to contact directly. + pub nodes_contacted_directly: Counter, + + /// Number of connections with a successful handshake. + pub connection_handshake_success: Counter, + /// Number of connections with a successful handshake that became direct. + pub connection_became_direct: Counter, } impl Default for Metrics { @@ -132,6 +143,12 @@ impl Default for Metrics { ), actor_link_change: Counter::new("actor_link_change"), actor_tick_other: Counter::new("actor_tick_other"), + + nodes_contacted: Counter::new("nodes_contacted"), + nodes_contacted_directly: Counter::new("nodes_contacted_directly"), + + connection_handshake_success: Counter::new("connection_handshake_success"), + connection_became_direct: Counter::new("connection_became_direct"), } } } diff --git a/iroh-net/src/magicsock/node_map.rs b/iroh-net/src/magicsock/node_map.rs index 9ba9f2222db..c3728ec6449 100644 --- a/iroh-net/src/magicsock/node_map.rs +++ b/iroh-net/src/magicsock/node_map.rs @@ -10,6 +10,7 @@ use std::{ use futures_lite::stream::Stream; use iroh_base::key::NodeId; use iroh_metrics::inc; +use iroh_relay::RelayUrl; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use stun_rs::TransactionId; @@ -25,8 +26,7 @@ use super::{ use crate::{ disco::{CallMeMaybe, Pong, SendAddr}, key::PublicKey, - relay::RelayUrl, - stun, NodeAddr, + NodeAddr, }; mod best_addr; @@ -161,7 +161,7 @@ impl NodeMap { &self, id: usize, dst: SendAddr, - tx_id: stun::TransactionId, + tx_id: stun_rs::TransactionId, purpose: DiscoPingPurpose, msg_sender: tokio::sync::mpsc::Sender, ) { @@ -170,7 +170,7 @@ impl NodeMap { } } - pub(super) fn notify_ping_timeout(&self, id: usize, tx_id: stun::TransactionId) { + pub(super) fn notify_ping_timeout(&self, id: usize, tx_id: stun_rs::TransactionId) { if let Some(ep) = self.inner.lock().get_mut(NodeStateKey::Idx(id)) { ep.ping_timeout(tx_id); } @@ -778,7 +778,7 @@ mod tests { info!("Adding alive addresses"); for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES { let addr = SendAddr::Udp(SocketAddr::new(LOCALHOST, 7000 + i as u16)); - let txid = stun::TransactionId::from([i as u8; 12]); + let txid = stun_rs::TransactionId::from([i as u8; 12]); // Note that this already invokes .prune_direct_addresses() because these are // new UDP paths. endpoint.handle_ping(addr, txid); diff --git a/iroh-net/src/magicsock/node_map/node_state.rs b/iroh-net/src/magicsock/node_map/node_state.rs index c7faa12c148..2f23c1c6c3d 100644 --- a/iroh-net/src/magicsock/node_map/node_state.rs +++ b/iroh-net/src/magicsock/node_map/node_state.rs @@ -6,6 +6,8 @@ use std::{ }; use iroh_metrics::inc; +use iroh_relay::{protos::stun, RelayUrl}; +use netwatch::ip::is_unicast_link_local; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tracing::{debug, event, info, instrument, trace, warn, Level}; @@ -22,9 +24,6 @@ use crate::{ endpoint::AddrInfo, key::PublicKey, magicsock::{ActorMessage, MagicsockMetrics, QuicMappedAddr, Timer, HEARTBEAT_INTERVAL}, - net::ip::is_unicast_link_local, - relay::RelayUrl, - stun, util::relay_only_mode, NodeAddr, NodeId, }; @@ -134,6 +133,10 @@ pub(super) struct NodeState { last_call_me_maybe: Option, /// The type of connection we have to the node, either direct, relay, mixed, or none. conn_type: Watchable, + /// Whether the conn_type was ever observed to be `Direct` at some point. + /// + /// Used for metric reporting. + has_been_direct: bool, } /// Options for creating a new [`NodeState`]. @@ -173,6 +176,7 @@ impl NodeState { last_used: options.active.then(Instant::now), last_call_me_maybe: None, conn_type: Watchable::new(ConnectionType::None), + has_been_direct: false, } } @@ -304,6 +308,10 @@ impl NodeState { (None, Some(relay_url)) => ConnectionType::Relay(relay_url), (None, None) => ConnectionType::None, }; + if !self.has_been_direct && matches!(&typ, ConnectionType::Direct(_)) { + self.has_been_direct = true; + inc!(MagicsockMetrics, nodes_contacted_directly); + } if let Ok(prev_typ) = self.conn_type.update(typ.clone()) { // The connection type has changed. event!( @@ -1132,7 +1140,11 @@ impl NodeState { have_ipv6: bool, ) -> (Option, Option, Vec) { let now = Instant::now(); - self.last_used.replace(now); + let prev = self.last_used.replace(now); + if prev.is_none() { + // this is the first time we are trying to connect to this node + inc!(MagicsockMetrics, nodes_contacted); + } let (udp_addr, relay_url) = self.addr_for_send(&now, have_ipv6); let mut ping_msgs = Vec::new(); @@ -1485,6 +1497,7 @@ mod tests { last_used: Some(now), last_call_me_maybe: None, conn_type: Watchable::new(ConnectionType::Direct(ip_port.into())), + has_been_direct: true, }, ip_port.into(), ) @@ -1504,6 +1517,7 @@ mod tests { last_used: Some(now), last_call_me_maybe: None, conn_type: Watchable::new(ConnectionType::Relay(send_addr.clone())), + has_been_direct: false, } }; @@ -1530,6 +1544,7 @@ mod tests { last_used: Some(now), last_call_me_maybe: None, conn_type: Watchable::new(ConnectionType::Relay(send_addr.clone())), + has_been_direct: false, } }; @@ -1569,6 +1584,7 @@ mod tests { socket_addr, send_addr.clone(), )), + has_been_direct: false, }, socket_addr, ) diff --git a/iroh-net/src/magicsock/node_map/path_state.rs b/iroh-net/src/magicsock/node_map/path_state.rs index 871f018a828..b8ff645e63f 100644 --- a/iroh-net/src/magicsock/node_map/path_state.rs +++ b/iroh-net/src/magicsock/node_map/path_state.rs @@ -7,13 +7,14 @@ use std::{ }; use iroh_base::key::NodeId; +use iroh_relay::protos::stun; use tracing::{debug, event, Level}; use super::{ node_state::{ControlMsg, PongReply, SESSION_ACTIVE_TIMEOUT}, IpPort, PingRole, Source, }; -use crate::{disco::SendAddr, magicsock::HEARTBEAT_INTERVAL, stun}; +use crate::{disco::SendAddr, magicsock::HEARTBEAT_INTERVAL}; /// The minimum time between pings to an endpoint. /// diff --git a/iroh-net/src/magicsock/relay_actor.rs b/iroh-net/src/magicsock/relay_actor.rs index 7998c37c806..5c07c0a494d 100644 --- a/iroh-net/src/magicsock/relay_actor.rs +++ b/iroh-net/src/magicsock/relay_actor.rs @@ -10,6 +10,7 @@ use anyhow::Context; use backoff::backoff::Backoff; use bytes::{Bytes, BytesMut}; use iroh_metrics::{inc, inc_by}; +use iroh_relay::{self as relay, client::ClientError, ReceivedMessage, RelayUrl, MAX_PACKET_SIZE}; use tokio::{ sync::{mpsc, oneshot}, task::{JoinHandle, JoinSet}, @@ -19,14 +20,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, info, info_span, trace, warn, Instrument}; use super::{ActorMessage, MagicSock, Metrics as MagicsockMetrics, RelayContents}; -use crate::{ - key::{NodeId, PUBLIC_KEY_LENGTH}, - relay::{ - self, - client::{conn::ReceivedMessage, ClientError}, - RelayUrl, MAX_PACKET_SIZE, - }, -}; +use crate::key::{NodeId, PUBLIC_KEY_LENGTH}; /// How long a non-home relay connection needs to be idle (last written to) before we close it. const RELAY_INACTIVE_CLEANUP_TIME: Duration = Duration::from_secs(60); @@ -215,7 +209,7 @@ impl ActiveRelay { } match msg { - relay::client::conn::ReceivedMessage::ReceivedPacket { source, data } => { + ReceivedMessage::ReceivedPacket { source, data } => { trace!(len=%data.len(), "received msg"); // If this is a new sender we hadn't seen before, remember it and // register a route for this peer. @@ -242,7 +236,7 @@ impl ActiveRelay { ReadResult::Continue } - relay::client::conn::ReceivedMessage::Ping(data) => { + ReceivedMessage::Ping(data) => { // Best effort reply to the ping. let dc = self.relay_client.clone(); tokio::task::spawn(async move { @@ -252,8 +246,8 @@ impl ActiveRelay { }); ReadResult::Continue } - relay::client::conn::ReceivedMessage::Health { .. } => ReadResult::Continue, - relay::client::conn::ReceivedMessage::PeerGone(key) => { + ReceivedMessage::Health { .. } => ReadResult::Continue, + ReceivedMessage::PeerGone(key) => { self.node_present.remove(&key); ReadResult::Continue } diff --git a/iroh-net/src/magicsock/udp_conn.rs b/iroh-net/src/magicsock/udp_conn.rs index db376521c54..c176ae8144c 100644 --- a/iroh-net/src/magicsock/udp_conn.rs +++ b/iroh-net/src/magicsock/udp_conn.rs @@ -9,13 +9,12 @@ use std::{ }; use anyhow::{bail, Context as _}; +use netwatch::UdpSocket; use quinn::AsyncUdpSocket; use quinn_udp::{Transmit, UdpSockRef}; use tokio::io::Interest; use tracing::{debug, trace}; -use crate::net::UdpSocket; - /// A UDP socket implementing Quinn's [`AsyncUdpSocket`]. #[derive(Clone, Debug)] pub struct UdpConn { @@ -197,11 +196,12 @@ where #[cfg(test)] mod tests { use anyhow::Result; + use netwatch::IpFamily; use tokio::sync::mpsc; use tracing::{info_span, Instrument}; use super::*; - use crate::{key, net::IpFamily, tls}; + use crate::{key, tls}; const ALPN: &[u8] = b"n0/test/1"; diff --git a/iroh-net/src/metrics.rs b/iroh-net/src/metrics.rs index 90ebbae09e1..655cfe44761 100644 --- a/iroh-net/src/metrics.rs +++ b/iroh-net/src/metrics.rs @@ -1,8 +1,7 @@ //! Co-locating all of the iroh-net metrics structs -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub use crate::relay::server::Metrics as RelayMetrics; -pub use crate::{ - magicsock::Metrics as MagicsockMetrics, netcheck::Metrics as NetcheckMetrics, - portmapper::Metrics as PortmapMetrics, -}; +#[cfg(feature = "test-utils")] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "test-utils")))] +pub use iroh_relay::server::Metrics as RelayMetrics; +pub use portmapper::Metrics as PortmapMetrics; + +pub use crate::{magicsock::Metrics as MagicsockMetrics, netcheck::Metrics as NetcheckMetrics}; diff --git a/iroh-net/src/netcheck.rs b/iroh-net/src/netcheck.rs index 5c5f904a5e7..1675889548a 100644 --- a/iroh-net/src/netcheck.rs +++ b/iroh-net/src/netcheck.rs @@ -15,7 +15,10 @@ use std::{ use anyhow::{anyhow, Context as _, Result}; use bytes::Bytes; +use hickory_resolver::TokioAsyncResolver as DnsResolver; use iroh_metrics::inc; +use iroh_relay::protos::stun; +use netwatch::{IpFamily, UdpSocket}; use tokio::{ sync::{self, mpsc, oneshot}, time::{Duration, Instant}, @@ -23,12 +26,7 @@ use tokio::{ use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{debug, error, info_span, trace, warn, Instrument}; -use super::{portmapper, relay::RelayMap, stun}; -use crate::{ - dns::DnsResolver, - net::{IpFamily, UdpSocket}, - relay::RelayUrl, -}; +use crate::{RelayMap, RelayUrl}; mod metrics; mod reportgen; @@ -783,18 +781,150 @@ mod tests { use crate::{ defaults::{staging::EU_RELAY_HOSTNAME, DEFAULT_STUN_PORT}, ping::Pinger, - relay::RelayNode, + RelayNode, }; + mod stun_utils { + //! Utils for testing that expose a simple stun server. + + use std::{net::IpAddr, sync::Arc}; + + use anyhow::Result; + use tokio::{ + net, + sync::{oneshot, Mutex}, + }; + use tracing::{debug, trace}; + + use super::*; + use crate::{RelayMap, RelayNode, RelayUrl}; + + // TODO: make all this private + + /// A drop guard to clean up test infrastructure. + /// + /// After dropping the test infrastructure will asynchronously shutdown and release its + /// resources. + // Nightly sees the sender as dead code currently, but we only rely on Drop of the + // sender. + #[derive(Debug)] + pub struct CleanupDropGuard { + _guard: oneshot::Sender<()>, + } + + // (read_ipv4, read_ipv6) + #[derive(Debug, Default, Clone)] + pub struct StunStats(Arc>); + + impl StunStats { + pub async fn total(&self) -> usize { + let s = self.0.lock().await; + s.0 + s.1 + } + } + + pub fn relay_map_of(stun: impl Iterator) -> RelayMap { + relay_map_of_opts(stun.map(|addr| (addr, true))) + } + + pub fn relay_map_of_opts(stun: impl Iterator) -> RelayMap { + let nodes = stun.map(|(addr, stun_only)| { + let host = addr.ip(); + let port = addr.port(); + + let url: RelayUrl = format!("http://{host}:{port}").parse().unwrap(); + RelayNode { + url, + stun_port: port, + stun_only, + } + }); + RelayMap::from_nodes(nodes).expect("generated invalid nodes") + } + + /// Sets up a simple STUN server binding to `0.0.0.0:0`. + /// + /// See [`serve`] for more details. + pub(crate) async fn serve_v4() -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { + serve(std::net::Ipv4Addr::UNSPECIFIED.into()).await + } + + /// Sets up a simple STUN server. + pub(crate) async fn serve(ip: IpAddr) -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { + let stats = StunStats::default(); + + let pc = net::UdpSocket::bind((ip, 0)).await?; + let mut addr = pc.local_addr()?; + match addr.ip() { + IpAddr::V4(ip) => { + if ip.octets() == [0, 0, 0, 0] { + addr.set_ip("127.0.0.1".parse().unwrap()); + } + } + _ => unreachable!("using ipv4"), + } + + println!("STUN listening on {}", addr); + let (_guard, r) = oneshot::channel(); + let stats_c = stats.clone(); + tokio::task::spawn(async move { + run_stun(pc, stats_c, r).await; + }); + + Ok((addr, stats, CleanupDropGuard { _guard })) + } + + async fn run_stun(pc: net::UdpSocket, stats: StunStats, mut done: oneshot::Receiver<()>) { + let mut buf = vec![0u8; 64 << 10]; + loop { + trace!("read loop"); + tokio::select! { + _ = &mut done => { + debug!("shutting down"); + break; + } + res = pc.recv_from(&mut buf) => match res { + Ok((n, addr)) => { + trace!("read packet {}bytes from {}", n, addr); + let pkt = &buf[..n]; + if !stun::is(pkt) { + debug!("received non STUN pkt"); + continue; + } + if let Ok(txid) = stun::parse_binding_request(pkt) { + debug!("received binding request"); + let mut s = stats.0.lock().await; + if addr.is_ipv4() { + s.0 += 1; + } else { + s.1 += 1; + } + drop(s); + + let res = stun::response(txid, addr); + if let Err(err) = pc.send_to(&res, addr).await { + eprintln!("STUN server write failed: {:?}", err); + } + } + } + Err(err) => { + eprintln!("failed to read: {:?}", err); + } + } + } + } + } + } + #[tokio::test] async fn test_basic() -> Result<()> { let _guard = iroh_test::logging::setup(); let (stun_addr, stun_stats, _cleanup_guard) = - stun::tests::serve("127.0.0.1".parse().unwrap()).await?; + stun_utils::serve("127.0.0.1".parse().unwrap()).await?; let resolver = crate::dns::default_resolver(); let mut client = Client::new(None, resolver.clone())?; - let dm = stun::tests::relay_map_of([stun_addr].into_iter()); + let dm = stun_utils::relay_map_of([stun_addr].into_iter()); // Note that the ProbePlan will change with each iteration. for i in 0..5 { @@ -885,7 +1015,7 @@ mod tests { // the STUN server being blocked will look like from the client's perspective. let blackhole = tokio::net::UdpSocket::bind("127.0.0.1:0").await?; let stun_addr = blackhole.local_addr()?; - let dm = stun::tests::relay_map_of_opts([(stun_addr, false)].into_iter()); + let dm = stun_utils::relay_map_of_opts([(stun_addr, false)].into_iter()); // Now create a client and generate a report. let resolver = crate::dns::default_resolver().clone(); @@ -1122,8 +1252,8 @@ mod tests { // can easily use to identify the packet. // Setup STUN server and create relay_map. - let (stun_addr, _stun_stats, _done) = stun::tests::serve_v4().await?; - let dm = stun::tests::relay_map_of([stun_addr].into_iter()); + let (stun_addr, _stun_stats, _done) = stun_utils::serve_v4().await?; + let dm = stun_utils::relay_map_of([stun_addr].into_iter()); dbg!(&dm); let resolver = crate::dns::default_resolver().clone(); diff --git a/iroh-net/src/netcheck/reportgen.rs b/iroh-net/src/netcheck/reportgen.rs index 007a04a410b..34fcb89393e 100644 --- a/iroh-net/src/netcheck/reportgen.rs +++ b/iroh-net/src/netcheck/reportgen.rs @@ -26,6 +26,8 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; use iroh_metrics::inc; +use iroh_relay::{http::RELAY_PROBE_PATH, protos::stun}; +use netwatch::{interfaces, UdpSocket}; use rand::seq::IteratorRandom; use tokio::{ sync::{mpsc, oneshot}, @@ -34,18 +36,16 @@ use tokio::{ }; use tokio_util::task::AbortOnDropHandle; use tracing::{debug, debug_span, error, info_span, trace, warn, Instrument, Span}; +use url::Host; use super::NetcheckMetrics; use crate::{ defaults::DEFAULT_STUN_PORT, dns::{DnsResolver, ResolverExt}, - net::{interfaces, UdpSocket}, netcheck::{self, Report}, ping::{PingError, Pinger}, - portmapper, - relay::{RelayMap, RelayNode, RelayUrl}, - stun, util::MaybeFuture, + RelayMap, RelayNode, RelayUrl, }; mod hairpin; @@ -467,6 +467,7 @@ impl Actor { .as_ref() .and_then(|l| l.preferred_relay.clone()); + let dns_resolver = self.dns_resolver.clone(); let dm = self.relay_map.clone(); self.outstanding_tasks.captive_task = true; MaybeFuture { @@ -475,7 +476,7 @@ impl Actor { debug!("Captive portal check started after {CAPTIVE_PORTAL_DELAY:?}"); let captive_portal_check = tokio::time::timeout( CAPTIVE_PORTAL_TIMEOUT, - check_captive_portal(&dm, preferred_relay) + check_captive_portal(&dns_resolver, &dm, preferred_relay) .instrument(debug_span!("captive-portal")), ); match captive_portal_check.await { @@ -744,7 +745,7 @@ async fn run_probe( } Probe::Https { ref node, .. } => { debug!("sending probe HTTPS"); - match measure_https_latency(node).await { + match measure_https_latency(&dns_resolver, node, None).await { Ok((latency, ip)) => { result.latency = Some(latency); // We set these IPv4 and IPv6 but they're not really used @@ -854,7 +855,11 @@ async fn run_stun_probe( /// return a "204 No Content" response and checking if that's what we get. /// /// The boolean return is whether we think we have a captive portal. -async fn check_captive_portal(dm: &RelayMap, preferred_relay: Option) -> Result { +async fn check_captive_portal( + dns_resolver: &DnsResolver, + dm: &RelayMap, + preferred_relay: Option, +) -> Result { // If we have a preferred relay node and we can use it for non-STUN requests, try that; // otherwise, pick a random one suitable for non-STUN requests. let preferred_relay = preferred_relay.and_then(|url| match dm.get_node(&url) { @@ -882,9 +887,22 @@ async fn check_captive_portal(dm: &RelayMap, preferred_relay: Option) } }; - let client = reqwest::ClientBuilder::new() - .redirect(reqwest::redirect::Policy::none()) - .build()?; + let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + if let Some(Host::Domain(domain)) = url.host() { + // Use our own resolver rather than getaddrinfo + // + // Be careful, a non-zero port will override the port in the URI. + // + // Ideally we would try to resolve **both** IPv4 and IPv6 rather than purely race + // them. But our resolver doesn't support that yet. + let addrs: Vec<_> = dns_resolver + .lookup_ipv4_ipv6_staggered(domain, DNS_TIMEOUT, DNS_STAGGERING_MS) + .await? + .map(|ipaddr| SocketAddr::new(ipaddr, 0)) + .collect(); + builder = builder.resolve_to_addrs(domain, &addrs); + } + let client = builder.build()?; // Note: the set of valid characters in a challenge and the total // length is limited; see is_challenge_char in bin/iroh-relay for more @@ -1024,33 +1042,71 @@ async fn run_icmp_probe( Ok(report) } +/// Executes an HTTPS probe. +/// +/// If `certs` is provided they will be added to the trusted root certificates, allowing the +/// use of self-signed certificates for servers. Currently this is used for testing. #[allow(clippy::unused_async)] -async fn measure_https_latency(_node: &RelayNode) -> Result<(Duration, IpAddr)> { - bail!("not implemented"); - // TODO: - // - needs relayhttp::Client - // - measurement hooks to measure server processing time - - // metricHTTPSend.Add(1) - // let ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout); - // let dc := relayhttp.NewNetcheckClient(c.logf); - // let tlsConn, tcpConn, node := dc.DialRegionTLS(ctx, reg)?; - // if ta, ok := tlsConn.RemoteAddr().(*net.TCPAddr); - // req, err := http.NewRequestWithContext(ctx, "GET", "https://"+node.HostName+"/relay/latency-check", nil); - // resp, err := hc.Do(req); - - // // relays should give us a nominal status code, so anything else is probably - // // an access denied by a MITM proxy (or at the very least a signal not to - // // trust this latency check). - // if resp.StatusCode > 299 { - // return 0, ip, fmt.Errorf("unexpected status code: %d (%s)", resp.StatusCode, resp.Status) - // } - // _, err = io.Copy(io.Discard, io.LimitReader(resp.Body, 8<<10)); - // result.End(c.timeNow()) - - // // TODO: decide best timing heuristic here. - // // Maybe the server should return the tcpinfo_rtt? - // return result.ServerProcessing, ip, nil +async fn measure_https_latency( + dns_resolver: &DnsResolver, + node: &RelayNode, + certs: Option>>, +) -> Result<(Duration, IpAddr)> { + let url = node.url.join(RELAY_PROBE_PATH)?; + + // This should also use same connection establishment as relay client itself, which + // needs to be more configurable so users can do more crazy things: + // https://github.com/n0-computer/iroh/issues/2901 + let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + if let Some(Host::Domain(domain)) = url.host() { + // Use our own resolver rather than getaddrinfo + // + // Be careful, a non-zero port will override the port in the URI. + // + // The relay Client uses `.lookup_ipv4_ipv6` to connect, so use the same function + // but staggered for reliability. Ideally this tries to resolve **both** IPv4 and + // IPv6 though. But our resolver does not have a function for that yet. + let addrs: Vec<_> = dns_resolver + .lookup_ipv4_ipv6_staggered(domain, DNS_TIMEOUT, DNS_STAGGERING_MS) + .await? + .map(|ipaddr| SocketAddr::new(ipaddr, 0)) + .collect(); + builder = builder.resolve_to_addrs(domain, &addrs); + } + if let Some(certs) = certs { + for cert in certs { + let cert = reqwest::Certificate::from_der(&cert)?; + builder = builder.add_root_certificate(cert); + } + } + let client = builder.build()?; + + let start = Instant::now(); + let mut response = client.request(reqwest::Method::GET, url).send().await?; + let latency = start.elapsed(); + if response.status().is_success() { + // Drain the response body to be nice to the server, up to a limit. + const MAX_BODY_SIZE: usize = 8 << 10; // 8 KiB + let mut body_size = 0; + while let Some(chunk) = response.chunk().await? { + body_size += chunk.len(); + if body_size >= MAX_BODY_SIZE { + break; + } + } + + // Only `None` if a different hyper HttpConnector in the request. + let remote_ip = response + .remote_addr() + .context("missing HttpInfo from HttpConnector")? + .ip(); + Ok((latency, remote_ip)) + } else { + Err(anyhow!( + "Error response from server: '{}'", + response.status().canonical_reason().unwrap_or_default() + )) + } } /// Updates a netcheck [`Report`] with a new [`ProbeReport`]. @@ -1119,8 +1175,13 @@ fn update_report(report: &mut Report, probe_report: ProbeReport) { mod tests { use std::net::{Ipv4Addr, Ipv6Addr}; + use testresult::TestResult; + use super::*; - use crate::defaults::staging::{default_eu_relay_node, default_na_relay_node}; + use crate::{ + defaults::staging::{default_eu_relay_node, default_na_relay_node}, + test_utils, + }; #[test] fn test_update_report_stun_working() { @@ -1369,4 +1430,28 @@ mod tests { panic!("Ping error: {err:#}"); } } + + #[tokio::test] + async fn test_measure_https_latency() -> TestResult { + let _logging_guard = iroh_test::logging::setup(); + let (_relay_map, relay_url, server) = test_utils::run_relay_server().await?; + let dns_resolver = crate::dns::resolver(); + warn!(?relay_url, "RELAY_URL"); + let node = RelayNode { + stun_only: false, + stun_port: 0, + url: relay_url.clone(), + }; + let (latency, ip) = + measure_https_latency(dns_resolver, &node, server.certificates()).await?; + + assert!(latency > Duration::ZERO); + + let relay_url_ip = relay_url + .host_str() + .context("host")? + .parse::()?; + assert_eq!(ip, relay_url_ip); + Ok(()) + } } diff --git a/iroh-net/src/netcheck/reportgen/hairpin.rs b/iroh-net/src/netcheck/reportgen/hairpin.rs index eba5b202eb1..b2a3fc806c4 100644 --- a/iroh-net/src/netcheck/reportgen/hairpin.rs +++ b/iroh-net/src/netcheck/reportgen/hairpin.rs @@ -15,15 +15,15 @@ use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use anyhow::{bail, Context, Result}; +use iroh_relay::protos::stun; +use netwatch::UdpSocket; use tokio::{sync::oneshot, time::Instant}; use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, info_span, trace, warn, Instrument}; use crate::{ defaults::timeouts::HAIRPIN_CHECK_TIMEOUT, - net::UdpSocket, netcheck::{self, reportgen, Inflight}, - stun, }; /// Handle to the hairpin actor. diff --git a/iroh-net/src/netcheck/reportgen/probes.rs b/iroh-net/src/netcheck/reportgen/probes.rs index 5bf62deec9f..4850a0e49c5 100644 --- a/iroh-net/src/netcheck/reportgen/probes.rs +++ b/iroh-net/src/netcheck/reportgen/probes.rs @@ -7,13 +7,10 @@ use std::{collections::BTreeSet, fmt, sync::Arc}; use anyhow::{ensure, Result}; +use netwatch::interfaces; use tokio::time::Duration; -use crate::{ - net::interfaces, - netcheck::Report, - relay::{RelayMap, RelayNode, RelayUrl}, -}; +use crate::{netcheck::Report, RelayMap, RelayNode, RelayUrl}; /// The retransmit interval used when netcheck first runs. /// diff --git a/iroh-net/src/relay.rs b/iroh-net/src/relay.rs deleted file mode 100644 index b7557bcbfc1..00000000000 --- a/iroh-net/src/relay.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Package `relay` implements a revised version of the Designated Encrypted Relay for Packets (DERP) -//! protocol written by Tailscale. -// -//! The relay routes packets to clients using curve25519 keys as addresses. -// -//! The relay is used to proxy encrypted QUIC packets through the relay servers when -//! a direct path cannot be found or opened. The relay is a last resort. If both sides -//! have very aggressive NATs, or firewalls, or no IPv6, we use the relay connection. -//! Based on tailscale/derp/derp.go - -#![deny(missing_docs, rustdoc::broken_intra_doc_links)] - -pub(crate) mod client; -pub(crate) mod codec; -pub mod http; -mod map; -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub mod server; - -pub use iroh_base::node_addr::RelayUrl; - -/// Environment variable to force the use of staging relays. -#[cfg_attr(iroh_docsrs, doc(cfg(not(test))))] -pub const ENV_FORCE_STAGING_RELAYS: &str = "IROH_FORCE_STAGING_RELAYS"; - -/// Returns `true` if the use of staging relays is forced. -pub fn force_staging_infra() -> bool { - matches!(std::env::var(ENV_FORCE_STAGING_RELAYS), Ok(value) if !value.is_empty()) -} - -pub use self::{ - client::{ - conn::{Conn as RelayConn, ReceivedMessage}, - Client as HttpClient, ClientBuilder as HttpClientBuilder, ClientError as HttpClientError, - ClientReceiver as HttpClientReceiver, - }, - codec::MAX_PACKET_SIZE, - map::{RelayMap, RelayMode, RelayNode}, -}; diff --git a/iroh-net/src/relay/map.rs b/iroh-net/src/relay_map.rs similarity index 99% rename from iroh-net/src/relay/map.rs rename to iroh-net/src/relay_map.rs index 03f6a8a2a7e..bb48408086c 100644 --- a/iroh-net/src/relay/map.rs +++ b/iroh-net/src/relay_map.rs @@ -3,9 +3,9 @@ use std::{collections::BTreeMap, fmt, sync::Arc}; use anyhow::{ensure, Result}; +pub use iroh_relay::RelayUrl; use serde::{Deserialize, Serialize}; -use super::RelayUrl; use crate::defaults::DEFAULT_STUN_PORT; /// Configuration of the relay servers for an [`Endpoint`]. diff --git a/iroh-net/src/stun.rs b/iroh-net/src/stun.rs deleted file mode 100644 index 0327925f476..00000000000 --- a/iroh-net/src/stun.rs +++ /dev/null @@ -1,545 +0,0 @@ -//! STUN packets sending and receiving. - -use std::net::SocketAddr; - -use stun_rs::{ - attributes::stun::{Fingerprint, XorMappedAddress}, - DecoderContextBuilder, MessageDecoderBuilder, MessageEncoderBuilder, StunMessageBuilder, -}; -pub use stun_rs::{ - attributes::StunAttribute, error::StunDecodeError, methods, MessageClass, MessageDecoder, - TransactionId, -}; - -/// Errors that can occur when handling a STUN packet. -#[derive(Debug, thiserror::Error)] -pub enum Error { - /// The STUN message could not be parsed or is otherwise invalid. - #[error("invalid message")] - InvalidMessage, - /// STUN request is not a binding request when it should be. - #[error("not binding")] - NotBinding, - /// STUN packet is not a response when it should be. - #[error("not success response")] - NotSuccessResponse, - /// STUN response has malformed attributes. - #[error("malformed attributes")] - MalformedAttrs, - /// STUN request didn't end in fingerprint. - #[error("no fingerprint")] - NoFingerprint, - /// STUN request had bogus fingerprint. - #[error("invalid fingerprint")] - InvalidFingerprint, -} - -/// Generates a binding request STUN packet. -pub fn request(tx: TransactionId) -> Vec { - let fp = Fingerprint::default(); - let msg = StunMessageBuilder::new(methods::BINDING, MessageClass::Request) - .with_transaction_id(tx) - .with_attribute(fp) - .build(); - - let encoder = MessageEncoderBuilder::default().build(); - let mut buffer = vec![0u8; 150]; - let size = encoder.encode(&mut buffer, &msg).expect("invalid encoding"); - buffer.truncate(size); - buffer -} - -/// Generates a binding response. -pub fn response(tx: TransactionId, addr: SocketAddr) -> Vec { - let msg = StunMessageBuilder::new(methods::BINDING, MessageClass::SuccessResponse) - .with_transaction_id(tx) - .with_attribute(XorMappedAddress::from(addr)) - .build(); - - let encoder = MessageEncoderBuilder::default().build(); - let mut buffer = vec![0u8; 150]; - let size = encoder.encode(&mut buffer, &msg).expect("invalid encoding"); - buffer.truncate(size); - buffer -} - -// Copied from stun_rs -// const MAGIC_COOKIE: Cookie = Cookie(0x2112_A442); -const COOKIE: [u8; 4] = 0x2112_A442u32.to_be_bytes(); - -/// Reports whether b is a STUN message. -pub fn is(b: &[u8]) -> bool { - b.len() >= stun_rs::MESSAGE_HEADER_SIZE && - b[0]&0b11000000 == 0 && // top two bits must be zero - b[4..8] == COOKIE -} - -/// Parses a STUN binding request. -pub fn parse_binding_request(b: &[u8]) -> Result { - let ctx = DecoderContextBuilder::default() - .with_validation() // ensure fingerprint is validated - .build(); - let decoder = MessageDecoderBuilder::default().with_context(ctx).build(); - let (msg, _) = decoder.decode(b).map_err(|_| Error::InvalidMessage)?; - - let tx = *msg.transaction_id(); - if msg.method() != methods::BINDING { - return Err(Error::NotBinding); - } - - // TODO: Tailscale sets the software to tailscale, we should check if we want to do this too. - - if msg - .attributes() - .last() - .map(|attr| !attr.is_fingerprint()) - .unwrap_or_default() - { - return Err(Error::NoFingerprint); - } - - Ok(tx) -} - -/// Parses a successful binding response STUN packet. -/// The IP address is extracted from the XOR-MAPPED-ADDRESS attribute. -pub fn parse_response(b: &[u8]) -> Result<(TransactionId, SocketAddr), Error> { - let decoder = MessageDecoder::default(); - let (msg, _) = decoder.decode(b).map_err(|_| Error::InvalidMessage)?; - - let tx = *msg.transaction_id(); - if msg.class() != MessageClass::SuccessResponse { - return Err(Error::NotSuccessResponse); - } - - // Read through the attributes. - // The the addr+port reported by XOR-MAPPED-ADDRESS - // as the canonical value. If the attribute is not - // present but the STUN server responds with - // MAPPED-ADDRESS we fall back to it. - - let mut addr = None; - let mut fallback_addr = None; - for attr in msg.attributes() { - match attr { - StunAttribute::XorMappedAddress(a) => { - let mut a = *a.socket_address(); - a.set_ip(a.ip().to_canonical()); - addr = Some(a); - } - StunAttribute::MappedAddress(a) => { - let mut a = *a.socket_address(); - a.set_ip(a.ip().to_canonical()); - fallback_addr = Some(a); - } - _ => {} - } - } - - if let Some(addr) = addr { - return Ok((tx, addr)); - } - - if let Some(addr) = fallback_addr { - return Ok((tx, addr)); - } - - Err(Error::MalformedAttrs) -} - -#[cfg(test)] -pub(crate) mod tests { - use std::{ - net::{IpAddr, Ipv4Addr}, - sync::Arc, - }; - - use anyhow::Result; - use tokio::{ - net, - sync::{oneshot, Mutex}, - }; - use tracing::{debug, trace}; - - use super::*; - use crate::{ - relay::{RelayMap, RelayNode, RelayUrl}, - test_utils::CleanupDropGuard, - }; - - // TODO: make all this private - - // (read_ipv4, read_ipv5) - #[derive(Debug, Default, Clone)] - pub struct StunStats(Arc>); - - impl StunStats { - pub async fn total(&self) -> usize { - let s = self.0.lock().await; - s.0 + s.1 - } - } - - pub fn relay_map_of(stun: impl Iterator) -> RelayMap { - relay_map_of_opts(stun.map(|addr| (addr, true))) - } - - pub fn relay_map_of_opts(stun: impl Iterator) -> RelayMap { - let nodes = stun.map(|(addr, stun_only)| { - let host = addr.ip(); - let port = addr.port(); - - let url: RelayUrl = format!("http://{host}:{port}").parse().unwrap(); - RelayNode { - url, - stun_port: port, - stun_only, - } - }); - RelayMap::from_nodes(nodes).expect("generated invalid nodes") - } - - /// Sets up a simple STUN server binding to `0.0.0.0:0`. - /// - /// See [`serve`] for more details. - pub(crate) async fn serve_v4() -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { - serve(std::net::Ipv4Addr::UNSPECIFIED.into()).await - } - - /// Sets up a simple STUN server. - pub(crate) async fn serve(ip: IpAddr) -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { - let stats = StunStats::default(); - - let pc = net::UdpSocket::bind((ip, 0)).await?; - let mut addr = pc.local_addr()?; - match addr.ip() { - IpAddr::V4(ip) => { - if ip.octets() == [0, 0, 0, 0] { - addr.set_ip("127.0.0.1".parse().unwrap()); - } - } - _ => unreachable!("using ipv4"), - } - - println!("STUN listening on {}", addr); - let (s, r) = oneshot::channel(); - let stats_c = stats.clone(); - tokio::task::spawn(async move { - run_stun(pc, stats_c, r).await; - }); - - Ok((addr, stats, CleanupDropGuard(s))) - } - - async fn run_stun(pc: net::UdpSocket, stats: StunStats, mut done: oneshot::Receiver<()>) { - let mut buf = vec![0u8; 64 << 10]; - loop { - trace!("read loop"); - tokio::select! { - _ = &mut done => { - debug!("shutting down"); - break; - } - res = pc.recv_from(&mut buf) => match res { - Ok((n, addr)) => { - trace!("read packet {}bytes from {}", n, addr); - let pkt = &buf[..n]; - if !is(pkt) { - debug!("received non STUN pkt"); - continue; - } - if let Ok(txid) = parse_binding_request(pkt) { - debug!("received binding request"); - let mut s = stats.0.lock().await; - if addr.is_ipv4() { - s.0 += 1; - } else { - s.1 += 1; - } - drop(s); - - let res = response(txid, addr); - if let Err(err) = pc.send_to(&res, addr).await { - eprintln!("STUN server write failed: {:?}", err); - } - } - } - Err(err) => { - eprintln!("failed to read: {:?}", err); - } - } - } - } - } - - // Test to check if an existing stun server works - // #[tokio::test] - // async fn test_stun_server() { - // use tokio::net::UdpSocket; - // use std::sync::Arc; - // use hickory_resolver::TokioAsyncResolver; - - // let domain = "cert-test.iroh.computer"; - // let port = 3478; - - // let txid = TransactionId::default(); - // let req = request(txid); - // let socket = Arc::new(UdpSocket::bind("0.0.0.0:0").await.unwrap()); - - // let resolver = TokioAsyncResolver::tokio_from_system_conf().unwrap(); - // let response = resolver.lookup_ip(domain).await.unwrap(); - // dbg!(&response); - - // let server_socket = socket.clone(); - // let server_task = tokio::task::spawn(async move { - // let mut buf = vec![0u8; 64000]; - // let len = server_socket.recv(&mut buf).await.unwrap(); - // dbg!(len); - // buf.truncate(len); - // buf - // }); - - // for addr in response { - // let addr = SocketAddr::new(addr, port); - // println!("sending to {addr}"); - // socket.send_to(&req, addr).await.unwrap(); - // } - - // let response = server_task.await.unwrap(); - // let (txid_back, response_addr) = parse_response(&response).unwrap(); - // assert_eq!(txid, txid_back); - // println!("got {response_addr}"); - // } - - struct ResponseTestCase { - name: &'static str, - data: Vec, - want_tid: Vec, - want_addr: IpAddr, - want_port: u16, - } - - #[test] - fn test_parse_response() { - let cases = vec![ - ResponseTestCase { - name: "google-1", - data: vec![ - 0x01, 0x01, 0x00, 0x0c, 0x21, 0x12, 0xa4, 0x42, - 0x23, 0x60, 0xb1, 0x1e, 0x3e, 0xc6, 0x8f, 0xfa, - 0x93, 0xe0, 0x80, 0x07, 0x00, 0x20, 0x00, 0x08, - 0x00, 0x01, 0xc7, 0x86, 0x69, 0x57, 0x85, 0x6f, - ], - want_tid: vec![ - 0x23, 0x60, 0xb1, 0x1e, 0x3e, 0xc6, 0x8f, 0xfa, - 0x93, 0xe0, 0x80, 0x07, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), - want_port: 59028, - }, - ResponseTestCase { - name: "google-2", - data: vec![ - 0x01, 0x01, 0x00, 0x0c, 0x21, 0x12, 0xa4, 0x42, - 0xf9, 0xf1, 0x21, 0xcb, 0xde, 0x7d, 0x7c, 0x75, - 0x92, 0x3c, 0xe2, 0x71, 0x00, 0x20, 0x00, 0x08, - 0x00, 0x01, 0xc7, 0x87, 0x69, 0x57, 0x85, 0x6f, - ], - want_tid: vec![ - 0xf9, 0xf1, 0x21, 0xcb, 0xde, 0x7d, 0x7c, 0x75, - 0x92, 0x3c, 0xe2, 0x71, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), - want_port: 59029, - }, - ResponseTestCase{ - name: "stun.sipgate.net:10000", - data: vec![ - 0x01, 0x01, 0x00, 0x44, 0x21, 0x12, 0xa4, 0x42, - 0x48, 0x2e, 0xb6, 0x47, 0x15, 0xe8, 0xb2, 0x8e, - 0xae, 0xad, 0x64, 0x44, 0x00, 0x01, 0x00, 0x08, - 0x00, 0x01, 0xe4, 0xab, 0x48, 0x45, 0x21, 0x2d, - 0x00, 0x04, 0x00, 0x08, 0x00, 0x01, 0x27, 0x10, - 0xd9, 0x0a, 0x44, 0x98, 0x00, 0x05, 0x00, 0x08, - 0x00, 0x01, 0x27, 0x11, 0xd9, 0x74, 0x7a, 0x8a, - 0x80, 0x20, 0x00, 0x08, 0x00, 0x01, 0xc5, 0xb9, - 0x69, 0x57, 0x85, 0x6f, 0x80, 0x22, 0x00, 0x10, - 0x56, 0x6f, 0x76, 0x69, 0x64, 0x61, 0x2e, 0x6f, - 0x72, 0x67, 0x20, 0x30, 0x2e, 0x39, 0x36, 0x00, - ], - want_tid: vec![ - 0x48, 0x2e, 0xb6, 0x47, 0x15, 0xe8, 0xb2, 0x8e, - 0xae, 0xad, 0x64, 0x44, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), - want_port: 58539, - }, - ResponseTestCase{ - name: "stun.powervoip.com:3478", - data: vec![ - 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, - 0x7e, 0x57, 0x96, 0x68, 0x29, 0xf4, 0x44, 0x60, - 0x9d, 0x1d, 0xea, 0xa6, 0x00, 0x01, 0x00, 0x08, - 0x00, 0x01, 0xe9, 0xd3, 0x48, 0x45, 0x21, 0x2d, - 0x00, 0x04, 0x00, 0x08, 0x00, 0x01, 0x0d, 0x96, - 0x4d, 0x48, 0xa9, 0xd4, 0x00, 0x05, 0x00, 0x08, - 0x00, 0x01, 0x0d, 0x97, 0x4d, 0x48, 0xa9, 0xd5, - ], - want_tid: vec![ - 0x7e, 0x57, 0x96, 0x68, 0x29, 0xf4, 0x44, 0x60, - 0x9d, 0x1d, 0xea, 0xa6, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), - want_port: 59859, - }, - ResponseTestCase{ - name: "in-process pion server", - data: vec![ - 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x0a, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x00, 0x00, 0x00, 0x20, 0x00, 0x08, - 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, - 0x80, 0x28, 0x00, 0x04, 0xb6, 0x99, 0xbb, 0x02, - 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, - ], - want_tid: vec![ - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), - want_port: 61300, - }, - ResponseTestCase{ - name: "stuntman-server ipv6", - data: vec![ - 0x01, 0x01, 0x00, 0x48, 0x21, 0x12, 0xa4, 0x42, - 0x06, 0xf5, 0x66, 0x85, 0xd2, 0x8a, 0xf3, 0xe6, - 0x9c, 0xe3, 0x41, 0xe2, 0x00, 0x01, 0x00, 0x14, - 0x00, 0x02, 0x90, 0xce, 0x26, 0x02, 0x00, 0xd1, - 0xb4, 0xcf, 0xc1, 0x00, 0x38, 0xb2, 0x31, 0xff, - 0xfe, 0xef, 0x96, 0xf6, 0x80, 0x2b, 0x00, 0x14, - 0x00, 0x02, 0x0d, 0x96, 0x26, 0x04, 0xa8, 0x80, - 0x00, 0x02, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xc5, 0x70, 0x01, 0x00, 0x20, 0x00, 0x14, - 0x00, 0x02, 0xb1, 0xdc, 0x07, 0x10, 0xa4, 0x93, - 0xb2, 0x3a, 0xa7, 0x85, 0xea, 0x38, 0xc2, 0x19, - 0x62, 0x0c, 0xd7, 0x14, - ], - want_tid: vec![ - 6, 245, 102, 133, 210, 138, 243, 230, 156, 227, - 65, 226, - ], - want_addr: "2602:d1:b4cf:c100:38b2:31ff:feef:96f6".parse().unwrap(), - want_port: 37070, - }, - // Testing STUN attribute padding rules using STUN software attribute - // with values of 1 & 3 length respectively before the XorMappedAddress attribute - ResponseTestCase { - name: "software-a", - data: vec![ - 0x01, 0x01, 0x00, 0x14, 0x21, 0x12, 0xa4, 0x42, - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x01, - 0x61, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x08, - 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, - ], - want_tid: vec![ - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), - want_port: 61300, - }, - ResponseTestCase { - name: "software-abc", - data: vec![ - 0x01, 0x01, 0x00, 0x14, 0x21, 0x12, 0xa4, 0x42, - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x03, - 0x61, 0x62, 0x63, 0x00, 0x00, 0x20, 0x00, 0x08, - 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, - ], - want_tid: vec![ - 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, - 0x4f, 0x3e, 0x30, 0x8e, - ], - want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), - want_port: 61300, - }, - ResponseTestCase { - name: "no-4in6", - data: hex::decode("010100182112a4424fd5d202dcb37d31fc773306002000140002cd3d2112a4424fd5d202dcb382ce2dc3fcc7").unwrap(), - want_tid: vec![79, 213, 210, 2, 220, 179, 125, 49, 252, 119, 51, 6], - want_addr: IpAddr::V4(Ipv4Addr::from([209, 180, 207, 193])), - want_port: 60463, - }, - ]; - - for (i, test) in cases.into_iter().enumerate() { - println!("Case {i}: {}", test.name); - let (tx, addr_port) = parse_response(&test.data).unwrap(); - assert!(is(&test.data)); - assert_eq!(tx.as_bytes(), &test.want_tid[..]); - assert_eq!(addr_port.ip(), test.want_addr); - assert_eq!(addr_port.port(), test.want_port); - } - } - - #[test] - fn test_parse_binding_request() { - let tx = TransactionId::default(); - let req = request(tx); - assert!(is(&req)); - let got_tx = parse_binding_request(&req).unwrap(); - assert_eq!(got_tx, tx); - } - - #[test] - fn test_stun_cookie() { - assert_eq!(stun_rs::MAGIC_COOKIE, COOKIE); - } - - #[test] - fn test_response() { - let txn = |n| TransactionId::from([n; 12]); - - struct Case { - tx: TransactionId, - addr: IpAddr, - port: u16, - } - let tests = vec![ - Case { - tx: txn(1), - addr: "1.2.3.4".parse().unwrap(), - port: 254, - }, - Case { - tx: txn(2), - addr: "1.2.3.4".parse().unwrap(), - port: 257, - }, - Case { - tx: txn(3), - addr: "1::4".parse().unwrap(), - port: 254, - }, - Case { - tx: txn(4), - addr: "1::4".parse().unwrap(), - port: 257, - }, - ]; - - for tt in tests { - let res = response(tt.tx, SocketAddr::new(tt.addr, tt.port)); - assert!(is(&res)); - let (tx2, addr2) = parse_response(&res).unwrap(); - assert_eq!(tt.tx, tx2); - assert_eq!(tt.addr, addr2.ip()); - assert_eq!(tt.port, addr2.port()); - } - } -} diff --git a/iroh-net/src/test_utils.rs b/iroh-net/src/test_utils.rs index b7eefb338c3..0000505f40c 100644 --- a/iroh-net/src/test_utils.rs +++ b/iroh-net/src/test_utils.rs @@ -4,15 +4,10 @@ use std::net::Ipv4Addr; use anyhow::Result; pub use dns_and_pkarr_servers::DnsPkarrServer; pub use dns_server::create_dns_resolver; +use iroh_relay::server::{CertConfig, RelayConfig, Server, ServerConfig, StunConfig, TlsConfig}; use tokio::sync::oneshot; -use crate::{ - key::SecretKey, - relay::{ - server::{CertConfig, RelayConfig, Server, ServerConfig, StunConfig, TlsConfig}, - RelayMap, RelayNode, RelayUrl, - }, -}; +use crate::{defaults::DEFAULT_STUN_PORT, RelayMap, RelayNode, RelayUrl}; /// A drop guard to clean up test infrastructure. /// @@ -29,8 +24,24 @@ pub struct CleanupDropGuard(pub(crate) oneshot::Sender<()>); /// The returned `Url` is the url of the relay server in the returned [`RelayMap`]. /// When dropped, the returned [`Server`] does will stop running. pub async fn run_relay_server() -> Result<(RelayMap, RelayUrl, Server)> { - let secret_key = SecretKey::generate(); - let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap(); + run_relay_server_with(Some(StunConfig { + bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), + })) + .await +} + +/// Runs a relay server. +/// +/// `stun` can be set to `None` to disable stun, or set to `Some` `StunConfig`, +/// to enable stun on a specific socket. +/// +/// The return value is similar to [`run_relay_server`]. +pub async fn run_relay_server_with( + stun: Option, +) -> Result<(RelayMap, RelayUrl, Server)> { + let cert = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .expect("valid"); let rustls_cert = rustls::pki_types::CertificateDer::from(cert.serialize_der().unwrap()); let private_key = rustls::pki_types::PrivatePkcs8KeyDer::from(cert.get_key_pair().serialize_der()); @@ -39,7 +50,6 @@ pub async fn run_relay_server() -> Result<(RelayMap, RelayUrl, Server)> { let config = ServerConfig { relay: Some(RelayConfig { http_bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), - secret_key, tls: Some(TlsConfig { cert: CertConfig::<(), ()>::Manual { private_key, @@ -49,20 +59,18 @@ pub async fn run_relay_server() -> Result<(RelayMap, RelayUrl, Server)> { }), limits: Default::default(), }), - stun: Some(StunConfig { - bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), - }), + stun, #[cfg(feature = "metrics")] metrics_addr: None, }; let server = Server::spawn(config).await.unwrap(); - let url: RelayUrl = format!("https://localhost:{}", server.https_addr().unwrap().port()) + let url: RelayUrl = format!("https://{}", server.https_addr().expect("configured")) .parse() .unwrap(); let m = RelayMap::from_nodes([RelayNode { url: url.clone(), stun_only: false, - stun_port: server.stun_addr().unwrap().port(), + stun_port: server.stun_addr().map_or(DEFAULT_STUN_PORT, |s| s.port()), }]) .unwrap(); Ok((m, url, server)) diff --git a/iroh-net/src/util.rs b/iroh-net/src/util.rs index 64eeacdc564..84af5b85c51 100644 --- a/iroh-net/src/util.rs +++ b/iroh-net/src/util.rs @@ -6,8 +6,6 @@ use std::{ task::{Context, Poll}, }; -pub(crate) mod chain; - /// Resolves to pending if the inner is `None`. #[derive(Debug)] pub(crate) struct MaybeFuture { diff --git a/iroh-relay/Cargo.toml b/iroh-relay/Cargo.toml new file mode 100644 index 00000000000..0189cccd648 --- /dev/null +++ b/iroh-relay/Cargo.toml @@ -0,0 +1,103 @@ +[package] +name = "iroh-relay" +version = "0.28.0" +edition = "2021" +readme = "README.md" +description = "Iroh's relay server and client" +license = "MIT OR Apache-2.0" +authors = ["n0 team"] +repository = "https://github.com/n0-computer/iroh" +keywords = ["networking", "holepunching", "p2p"] +rust-version = "1.76" + +[lints] +workspace = true + +[dependencies] +anyhow = { version = "1" } +base64 = "0.22.1" +bytes = "1.7" +clap = { version = "4", features = ["derive"], optional = true } +derive_more = { version = "1.0.0", features = ["debug", "display", "from", "try_into", "deref"] } +futures-lite = "2.3" +futures-sink = "0.3.25" +futures-util = "0.3.25" +governor = "0.6.0" +hex = "0.4.3" +hickory-proto = "=0.25.0-alpha.2" +hickory-resolver = "=0.25.0-alpha.2" +hostname = "0.3.1" +http = "1" +http-body-util = "0.1.0" +hyper = { version = "1", features = ["server", "client", "http1"] } +hyper-util = "0.1.1" +iroh-base = { version = "0.28.0", features = ["key"] } +iroh-metrics = { version = "0.28.0", default-features = false} +libc = "0.2.139" +num_enum = "0.7" +once_cell = "1.18.0" +parking_lot = "0.12.1" +pin-project = "1" +postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } +rand = "0.8" +rcgen = { version = "0.12", optional = true} +regex = { version = "1.7.1", optional = true } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +ring = "0.17" +rustls = { version = "0.23", default-features = false, features = ["ring"] } +rustls-pemfile = { version = "2.1", optional = true } +serde = { version = "1", features = ["derive", "rc"] } +smallvec = "1.11.1" +socket2 = "0.5.3" +stun-rs = "0.1.5" +thiserror = "1" +time = "0.3.20" +tokio = { version = "1", features = ["io-util", "macros", "sync", "rt", "net", "fs", "io-std", "signal", "process"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring"] } +tokio-rustls-acme = { version = "0.4", optional = true } +tokio-tungstenite = "0.21" +tokio-tungstenite-wasm = "0.3" +tokio-util = { version = "0.7.12", features = ["io-util", "io", "codec", "rt"] } +toml = { version = "0.8", optional = true } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } +tungstenite = "0.21" +url = { version = "2.4", features = ["serde"] } +webpki = { package = "rustls-webpki", version = "0.102" } +webpki-roots = "0.26" + +[dev-dependencies] +clap = { version = "4", features = ["derive"] } +crypto_box = { version = "0.9.1", features = ["serde", "chacha20"] } +proptest = "1.2.0" +rand_chacha = "0.3.1" +tokio = { version = "1", features = ["io-util", "sync", "rt", "net", "fs", "macros", "time", "test-util"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +iroh-test = "0.28.0" +serde_json = "1.0.107" + +[build-dependencies] +duct = "0.13.6" + +[features] +default = ["metrics", "server"] +server = [ + "dep:tokio-rustls-acme", + "dep:clap", + "dep:toml", + "dep:rustls-pemfile", + "dep:regex", + "dep:tracing-subscriber", + "dep:rcgen", +] +metrics = ["iroh-metrics/metrics", "server"] +test-utils = [] + +[[bin]] +name = "iroh-relay" +path = "src/main.rs" +required-features = ["server"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "iroh_docsrs"] diff --git a/iroh-relay/README.md b/iroh-relay/README.md new file mode 100644 index 00000000000..234b9e0e1c5 --- /dev/null +++ b/iroh-relay/README.md @@ -0,0 +1,43 @@ +# Iroh Relay + +Iroh's relay is a feature within [iroh], a peer-to-peer networking system +designed to facilitate direct, encrypted connections between devices. Iroh aims +to simplify decentralized communication by automatically handling connections +through "relays" when direct connections aren't immediately possible. The relay +server helps establish connections by temporarily routing encrypted traffic +until a direct, P2P connection is feasible. Once this direct path is set up, +the relay server steps back, and the data flows directly between devices. This +approach allows Iroh to maintain a secure, low-latency connection, even in +challenging network situations. + +This crate provides a complete setup for creating and interacting with iroh +relays, including: +- Relay Protocol: The protocol used to communicate between relay servers and + clients +- Relay Server: A fully-fledged iroh-relay server over HTTP or HTTPS. + Optionally will also expose a stun endpoint and metrics. +- Relay Client: A client for establishing connections to the relay. +- Server Binary: A CLI for running your own relay server. It can be configured + to also offer STUN support and expose metrics. + + +Used in [iroh], created with love by the [n0 team](https://n0.computer/). + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. + +[iroh]: https://github.com/n0-computer/iroh diff --git a/iroh-net/src/relay/client.rs b/iroh-relay/src/client.rs similarity index 87% rename from iroh-net/src/relay/client.rs rename to iroh-relay/src/client.rs index d8ef25d58f8..a6d52427dcb 100644 --- a/iroh-net/src/relay/client.rs +++ b/iroh-relay/src/client.rs @@ -1,7 +1,10 @@ +//! Exposes [`Client`], which allows to establish connections to a relay server. +//! //! Based on tailscale/derp/derphttp/derphttp_client.go use std::{ collections::HashMap, + future, net::{IpAddr, SocketAddr}, sync::Arc, time::Duration, @@ -12,9 +15,16 @@ use bytes::Bytes; use conn::{Conn, ConnBuilder, ConnReader, ConnReceiver, ConnWriter, ReceivedMessage}; use futures_lite::future::Boxed as BoxFuture; use futures_util::StreamExt; +use hickory_resolver::TokioAsyncResolver as DnsResolver; use http_body_util::Empty; -use hyper::{body::Incoming, header::UPGRADE, upgrade::Parts, Request}; +use hyper::{ + body::Incoming, + header::{HOST, UPGRADE}, + upgrade::Parts, + Request, +}; use hyper_util::rt::TokioIo; +use iroh_base::key::{NodeId, PublicKey, SecretKey}; use rand::Rng; use rustls::client::Resumption; use streams::{downcast_upgrade, MaybeTlsStream, ProxyStream}; @@ -33,19 +43,15 @@ use tracing::{debug, error, event, info_span, trace, warn, Instrument, Level}; use url::Url; use crate::{ - defaults::timeouts::relay::*, - dns::{DnsResolver, ResolverExt}, - key::{NodeId, PublicKey, SecretKey}, - relay::{ - codec::DerpCodec, - http::{Protocol, RELAY_PATH}, - RelayUrl, - }, - util::chain, + defaults::timeouts::*, + http::{Protocol, RELAY_PATH}, + protos::relay::DerpCodec, + RelayUrl, }; pub(crate) mod conn; pub(crate) mod streams; +mod util; /// Possible connection errors on the [`Client`] #[derive(Debug, thiserror::Error)] @@ -404,7 +410,7 @@ impl Client { /// Returns [`ClientError::Closed`] if the [`Client`] is closed. /// /// If there is already an active relay connection, returns the already - /// connected [`crate::relay::RelayConn`]. + /// connected [`crate::RelayConn`]. pub async fn connect(&self) -> Result { self.send_actor(ActorMessage::Connect).await } @@ -643,6 +649,7 @@ impl Actor { } async fn connect_derp(&self) -> Result<(ConnReader, ConnWriter, SocketAddr), ClientError> { + let url = self.url.clone(); let tcp_stream = self.dial_url().await?; let local_addr = tcp_stream @@ -659,10 +666,10 @@ impl Actor { let hostname = hostname.to_owned(); let tls_stream = self.tls_connector.connect(hostname, tcp_stream).await?; debug!("tls_connector connect success"); - Self::start_upgrade(tls_stream).await? + Self::start_upgrade(tls_stream, url).await? } else { debug!("Starting handshake"); - Self::start_upgrade(tcp_stream).await? + Self::start_upgrade(tcp_stream, url).await? }; if response.status() != hyper::StatusCode::SWITCHING_PROTOCOLS { @@ -696,10 +703,15 @@ impl Actor { } /// Sends the HTTP upgrade request to the relay server. - async fn start_upgrade(io: T) -> Result, ClientError> + async fn start_upgrade( + io: T, + relay_url: RelayUrl, + ) -> Result, ClientError> where T: AsyncRead + AsyncWrite + Send + Unpin + 'static, { + let host_header_value = host_header_value(relay_url)?; + let io = hyper_util::rt::TokioIo::new(io); let (mut request_sender, connection) = hyper::client::conn::http1::Builder::new() .handshake(io) @@ -719,6 +731,10 @@ impl Actor { let req = Request::builder() .uri(RELAY_PATH) .header(UPGRADE, Protocol::Relay.upgrade_header()) + // https://datatracker.ietf.org/doc/html/rfc2616#section-14.23 + // > A client MUST include a Host header field in all HTTP/1.1 request messages. + // This header value helps reverse proxies identify how to forward requests. + .header(HOST, host_header_value) .body(http_body_util::Empty::::new())?; request_sender.send_request(req).await.map_err(From::from) } @@ -849,7 +865,10 @@ impl Actor { async fn dial_url_direct(&self) -> Result { debug!(%self.url, "dial url"); let prefer_ipv6 = self.prefer_ipv6().await; - let dst_ip = resolve_host(&self.dns_resolver, &self.url, prefer_ipv6).await?; + let dst_ip = self + .dns_resolver + .resolve_host(&self.url, prefer_ipv6) + .await?; let port = url_port(&self.url) .ok_or_else(|| ClientError::InvalidUrl("missing url port".into()))?; @@ -873,12 +892,15 @@ impl Actor { async fn dial_url_proxy( &self, proxy_url: Url, - ) -> Result, MaybeTlsStream>, ClientError> { + ) -> Result, MaybeTlsStream>, ClientError> { debug!(%self.url, %proxy_url, "dial url via proxy"); // Resolve proxy DNS let prefer_ipv6 = self.prefer_ipv6().await; - let proxy_ip = resolve_host(&self.dns_resolver, &proxy_url, prefer_ipv6).await?; + let proxy_ip = self + .dns_resolver + .resolve_host(&proxy_url, prefer_ipv6) + .await?; let proxy_port = url_port(&proxy_url) .ok_or_else(|| ClientError::Proxy("missing proxy url port".into()))?; @@ -961,7 +983,7 @@ impl Actor { return Err(ClientError::Proxy("invalid upgrade".to_string())); }; - let res = chain::chain(std::io::Cursor::new(read_buf), io.into_inner()); + let res = util::chain(std::io::Cursor::new(read_buf), io.into_inner()); Ok(res) } @@ -995,7 +1017,7 @@ impl Actor { } } } - std::future::pending().await + future::pending().await } /// Close the underlying relay connection. The next time the client takes some action that @@ -1008,34 +1030,81 @@ impl Actor { } } -async fn resolve_host( - resolver: &DnsResolver, - url: &Url, - prefer_ipv6: bool, -) -> Result { - let host = url - .host() - .ok_or_else(|| ClientError::InvalidUrl("missing host".into()))?; - match host { - url::Host::Domain(domain) => { - // Need to do a DNS lookup - let mut addrs = resolver - .lookup_ipv4_ipv6(domain, DNS_TIMEOUT) - .await - .map_err(|e| ClientError::Dns(Some(e)))? - .peekable(); +fn host_header_value(relay_url: RelayUrl) -> Result { + // grab the host, turns e.g. https://example.com:8080/xyz -> example.com. + let relay_url_host = relay_url + .host_str() + .ok_or_else(|| ClientError::InvalidUrl(relay_url.to_string()))?; + // strip the trailing dot, if present: example.com. -> example.com + let relay_url_host = relay_url_host.strip_suffix('.').unwrap_or(relay_url_host); + // build the host header value (reserve up to 6 chars for the ":" and port digits): + let mut host_header_value = String::with_capacity(relay_url_host.len() + 6); + host_header_value += relay_url_host; + if let Some(port) = relay_url.port() { + host_header_value += ":"; + host_header_value += &port.to_string(); + } + Ok(host_header_value) +} - let found = if prefer_ipv6 { - let first = addrs.peek().copied(); - addrs.find(IpAddr::is_ipv6).or(first) - } else { - addrs.next() - }; +trait DnsExt { + fn lookup_ipv4( + &self, + host: N, + ) -> impl future::Future>>; + + fn lookup_ipv6( + &self, + host: N, + ) -> impl future::Future>>; + + fn resolve_host( + &self, + url: &Url, + prefer_ipv6: bool, + ) -> impl future::Future>; +} - found.ok_or_else(|| ClientError::Dns(None)) +impl DnsExt for DnsResolver { + async fn lookup_ipv4( + &self, + host: N, + ) -> anyhow::Result> { + let addrs = tokio::time::timeout(DNS_TIMEOUT, self.ipv4_lookup(host)).await??; + Ok(addrs.into_iter().next().map(|ip| IpAddr::V4(ip.0))) + } + + async fn lookup_ipv6( + &self, + host: N, + ) -> anyhow::Result> { + let addrs = tokio::time::timeout(DNS_TIMEOUT, self.ipv6_lookup(host)).await??; + Ok(addrs.into_iter().next().map(|ip| IpAddr::V6(ip.0))) + } + + async fn resolve_host(&self, url: &Url, prefer_ipv6: bool) -> Result { + let host = url + .host() + .ok_or_else(|| ClientError::InvalidUrl("missing host".into()))?; + match host { + url::Host::Domain(domain) => { + // Need to do a DNS lookup + let lookup = tokio::join!(self.lookup_ipv4(domain), self.lookup_ipv6(domain)); + let (v4, v6) = match lookup { + (Err(ipv4_err), Err(ipv6_err)) => { + let err = anyhow::anyhow!("Ipv4: {:?}, Ipv6: {:?}", ipv4_err, ipv6_err); + return Err(ClientError::Dns(Some(err))); + } + (Err(_), Ok(v6)) => (None, v6), + (Ok(v4), Err(_)) => (v4, None), + (Ok(v4), Ok(v6)) => (v4, v6), + }; + if prefer_ipv6 { v6.or(v4) } else { v4.or(v6) } + .ok_or_else(|| ClientError::Dns(None)) + } + url::Host::Ipv4(ip) => Ok(IpAddr::V4(ip)), + url::Host::Ipv6(ip) => Ok(IpAddr::V6(ip)), } - url::Host::Ipv4(ip) => Ok(IpAddr::V4(ip)), - url::Host::Ipv6(ip) => Ok(IpAddr::V6(ip)), } } @@ -1096,6 +1165,8 @@ fn url_port(url: &Url) -> Option { #[cfg(test)] mod tests { + use std::str::FromStr; + use anyhow::{bail, Result}; use super::*; @@ -1119,4 +1190,25 @@ mod tests { } Ok(()) } + + #[test] + fn test_host_header_value() -> Result<()> { + let _guard = iroh_test::logging::setup(); + + let cases = [ + ( + "https://euw1-1.relay.iroh.network.", + "euw1-1.relay.iroh.network", + ), + ("http://localhost:8080", "localhost:8080"), + ]; + + for (url, expected_host) in cases { + let relay_url = RelayUrl::from_str(url)?; + let host = host_header_value(relay_url)?; + assert_eq!(host, expected_host); + } + + Ok(()) + } } diff --git a/iroh-net/src/relay/client/conn.rs b/iroh-relay/src/client/conn.rs similarity index 97% rename from iroh-net/src/relay/client/conn.rs rename to iroh-relay/src/client/conn.rs index 70a8a182ad1..d9de3810cbd 100644 --- a/iroh-net/src/relay/client/conn.rs +++ b/iroh-relay/src/client/conn.rs @@ -19,6 +19,7 @@ use futures_util::{ stream::{SplitSink, SplitStream, StreamExt}, SinkExt, }; +use iroh_base::key::{PublicKey, SecretKey}; use tokio::sync::mpsc; use tokio_tungstenite_wasm::WebSocketStream; use tokio_util::{ @@ -28,14 +29,11 @@ use tokio_util::{ use tracing::{debug, info_span, trace, Instrument}; use crate::{ - defaults::timeouts::relay::CLIENT_RECV_TIMEOUT, - key::{PublicKey, SecretKey}, - relay::{ - client::streams::{MaybeTlsStreamReader, MaybeTlsStreamWriter}, - codec::{ - write_frame, ClientInfo, DerpCodec, Frame, MAX_PACKET_SIZE, - PER_CLIENT_READ_QUEUE_DEPTH, PER_CLIENT_SEND_QUEUE_DEPTH, PROTOCOL_VERSION, - }, + client::streams::{MaybeTlsStreamReader, MaybeTlsStreamWriter}, + defaults::timeouts::CLIENT_RECV_TIMEOUT, + protos::relay::{ + write_frame, ClientInfo, DerpCodec, Frame, MAX_PACKET_SIZE, PER_CLIENT_READ_QUEUE_DEPTH, + PER_CLIENT_SEND_QUEUE_DEPTH, PROTOCOL_VERSION, }, }; @@ -208,7 +206,7 @@ fn process_incoming_frame(frame: Frame) -> Result { } } -/// The kinds of messages we can send to the [`Server`](crate::relay::server::Server) +/// The kinds of messages we can send to the [`Server`](crate::server::Server) #[derive(Debug)] enum ConnWriterMessage { /// Send a packet (addressed to the [`PublicKey`]) to the server @@ -368,7 +366,7 @@ impl ConnBuilder { version: PROTOCOL_VERSION, }; debug!("server_handshake: sending client_key: {:?}", &client_info); - crate::relay::codec::send_client_key(&mut self.writer, &self.secret_key, &client_info) + crate::protos::relay::send_client_key(&mut self.writer, &self.secret_key, &client_info) .await?; // TODO: add some actual configuration diff --git a/iroh-net/src/relay/client/streams.rs b/iroh-relay/src/client/streams.rs similarity index 92% rename from iroh-net/src/relay/client/streams.rs rename to iroh-relay/src/client/streams.rs index 7910049683b..6e07103e839 100644 --- a/iroh-net/src/relay/client/streams.rs +++ b/iroh-relay/src/client/streams.rs @@ -13,17 +13,17 @@ use tokio::{ net::TcpStream, }; -use crate::util::chain; +use super::util; pub enum MaybeTlsStreamReader { - Raw(chain::Chain, tokio::io::ReadHalf>), + Raw(util::Chain, tokio::io::ReadHalf>), Tls( - chain::Chain< + util::Chain< std::io::Cursor, tokio::io::ReadHalf>, >, ), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Mem(tokio::io::ReadHalf), } @@ -36,7 +36,7 @@ impl AsyncRead for MaybeTlsStreamReader { match &mut *self { Self::Raw(stream) => Pin::new(stream).poll_read(cx, buf), Self::Tls(stream) => Pin::new(stream).poll_read(cx, buf), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Self::Mem(stream) => Pin::new(stream).poll_read(cx, buf), } } @@ -45,7 +45,7 @@ impl AsyncRead for MaybeTlsStreamReader { pub enum MaybeTlsStreamWriter { Raw(tokio::io::WriteHalf), Tls(tokio::io::WriteHalf>), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Mem(tokio::io::WriteHalf), } @@ -58,7 +58,7 @@ impl AsyncWrite for MaybeTlsStreamWriter { match &mut *self { Self::Raw(stream) => Pin::new(stream).poll_write(cx, buf), Self::Tls(stream) => Pin::new(stream).poll_write(cx, buf), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Self::Mem(stream) => Pin::new(stream).poll_write(cx, buf), } } @@ -70,7 +70,7 @@ impl AsyncWrite for MaybeTlsStreamWriter { match &mut *self { Self::Raw(stream) => Pin::new(stream).poll_flush(cx), Self::Tls(stream) => Pin::new(stream).poll_flush(cx), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Self::Mem(stream) => Pin::new(stream).poll_flush(cx), } } @@ -82,7 +82,7 @@ impl AsyncWrite for MaybeTlsStreamWriter { match &mut *self { Self::Raw(stream) => Pin::new(stream).poll_shutdown(cx), Self::Tls(stream) => Pin::new(stream).poll_shutdown(cx), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Self::Mem(stream) => Pin::new(stream).poll_shutdown(cx), } } @@ -95,7 +95,7 @@ impl AsyncWrite for MaybeTlsStreamWriter { match &mut *self { Self::Raw(stream) => Pin::new(stream).poll_write_vectored(cx, bufs), Self::Tls(stream) => Pin::new(stream).poll_write_vectored(cx, bufs), - #[cfg(test)] + #[cfg(all(test, feature = "server"))] Self::Mem(stream) => Pin::new(stream).poll_write_vectored(cx, bufs), } } @@ -109,7 +109,7 @@ pub fn downcast_upgrade( let inner = io.into_inner(); let (reader, writer) = tokio::io::split(inner); // Prepend data to the reader to avoid data loss - let reader = chain::chain(std::io::Cursor::new(read_buf), reader); + let reader = util::chain(std::io::Cursor::new(read_buf), reader); Ok(( MaybeTlsStreamReader::Raw(reader), MaybeTlsStreamWriter::Raw(writer), @@ -122,7 +122,7 @@ pub fn downcast_upgrade( let inner = io.into_inner(); let (reader, writer) = tokio::io::split(inner); // Prepend data to the reader to avoid data loss - let reader = chain::chain(std::io::Cursor::new(read_buf), reader); + let reader = util::chain(std::io::Cursor::new(read_buf), reader); return Ok(( MaybeTlsStreamReader::Tls(reader), @@ -139,7 +139,7 @@ pub fn downcast_upgrade( pub enum ProxyStream { Raw(TcpStream), - Proxied(chain::Chain, MaybeTlsStream>), + Proxied(util::Chain, MaybeTlsStream>), } impl AsyncRead for ProxyStream { diff --git a/iroh-net/src/util/chain.rs b/iroh-relay/src/client/util.rs similarity index 100% rename from iroh-net/src/util/chain.rs rename to iroh-relay/src/client/util.rs diff --git a/iroh-relay/src/defaults.rs b/iroh-relay/src/defaults.rs new file mode 100644 index 00000000000..f5446f8e02a --- /dev/null +++ b/iroh-relay/src/defaults.rs @@ -0,0 +1,42 @@ +//! Default values used in the relay. + +/// The efault STUN port used by the Relay server. +/// +/// The STUN port as defined by [RFC +/// 8489]() +pub const DEFAULT_STUN_PORT: u16 = 3478; + +/// The default HTTP port used by the Relay server. +pub const DEFAULT_HTTP_PORT: u16 = 80; + +/// The default HTTPS port used by the Relay server. +pub const DEFAULT_HTTPS_PORT: u16 = 443; + +/// The default metrics port used by the Relay server. +pub const DEFAULT_METRICS_PORT: u16 = 9090; + +/// Contains all timeouts that we use in `iroh-net`. +pub(crate) mod timeouts { + use std::time::Duration; + + /// Timeout used by the relay client while connecting to the relay server, + /// using `TcpStream::connect` + pub(crate) const DIAL_NODE_TIMEOUT: Duration = Duration::from_millis(1500); + /// Timeout for expecting a pong from the relay server + pub(crate) const PING_TIMEOUT: Duration = Duration::from_secs(5); + /// Timeout for the entire relay connection, which includes dns, dialing + /// the server, upgrading the connection, and completing the handshake + pub(crate) const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + /// Timeout for our async dns resolver + pub(crate) const DNS_TIMEOUT: Duration = Duration::from_secs(1); + + /// Maximum time the client will wait to receive on the connection, since + /// the last message. Longer than this time and the client will consider + /// the connection dead. + pub(crate) const CLIENT_RECV_TIMEOUT: Duration = Duration::from_secs(120); + + /// Maximum time the server will attempt to get a successful write to the connection. + #[cfg(feature = "server")] + #[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] + pub(crate) const SERVER_WRITE_TIMEOUT: Duration = Duration::from_secs(2); +} diff --git a/iroh-relay/src/dns.rs b/iroh-relay/src/dns.rs new file mode 100644 index 00000000000..afd2d1c77ea --- /dev/null +++ b/iroh-relay/src/dns.rs @@ -0,0 +1,64 @@ +use std::net::{IpAddr, Ipv6Addr}; + +use anyhow::Result; +use hickory_resolver::{AsyncResolver, TokioAsyncResolver}; +use once_cell::sync::Lazy; + +/// The DNS resolver type used throughout `iroh-net`. +pub(crate) type DnsResolver = TokioAsyncResolver; + +static DNS_RESOLVER: Lazy = + Lazy::new(|| create_default_resolver().expect("unable to create DNS resolver")); + +/// Get a reference to the default DNS resolver. +/// +/// The default resolver can be cheaply cloned and is shared throughout the running process. +/// It is configured to use the system's DNS configuration. +pub fn default_resolver() -> &'static DnsResolver { + &DNS_RESOLVER +} + +/// Deprecated IPv6 site-local anycast addresses still configured by windows. +/// +/// Windows still configures these site-local addresses as soon even as an IPv6 loopback +/// interface is configured. We do not want to use these DNS servers, the chances of them +/// being usable are almost always close to zero, while the chance of DNS configuration +/// **only** relying on these servers and not also being configured normally are also almost +/// zero. The chance of the DNS resolver accidentally trying one of these and taking a +/// bunch of timeouts to figure out they're no good are on the other hand very high. +const WINDOWS_BAD_SITE_LOCAL_DNS_SERVERS: [IpAddr; 3] = [ + IpAddr::V6(Ipv6Addr::new(0xfec0, 0, 0, 0xffff, 0, 0, 0, 1)), + IpAddr::V6(Ipv6Addr::new(0xfec0, 0, 0, 0xffff, 0, 0, 0, 2)), + IpAddr::V6(Ipv6Addr::new(0xfec0, 0, 0, 0xffff, 0, 0, 0, 3)), +]; + +/// Get resolver to query MX records. +/// +/// We first try to read the system's resolver from `/etc/resolv.conf`. +/// This does not work at least on some Androids, therefore we fallback +/// to the default `ResolverConfig` which uses eg. to google's `8.8.8.8` or `8.8.4.4`. +fn create_default_resolver() -> Result { + let (system_config, mut options) = + hickory_resolver::system_conf::read_system_conf().unwrap_or_default(); + + // Copy all of the system config, but strip the bad windows nameservers. Unfortunately + // there is no easy way to do this. + let mut config = hickory_resolver::config::ResolverConfig::new(); + if let Some(name) = system_config.domain() { + config.set_domain(name.clone()); + } + for name in system_config.search() { + config.add_search(name.clone()); + } + for nameserver_cfg in system_config.name_servers() { + if !WINDOWS_BAD_SITE_LOCAL_DNS_SERVERS.contains(&nameserver_cfg.socket_addr.ip()) { + config.add_name_server(nameserver_cfg.clone()); + } + } + + // see [`ResolverExt::lookup_ipv4_ipv6`] for info on why we avoid `LookupIpStrategy::Ipv4AndIpv6` + options.ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6; + + let resolver = AsyncResolver::tokio(config, options); + Ok(resolver) +} diff --git a/iroh-net/src/relay/http.rs b/iroh-relay/src/http.rs similarity index 72% rename from iroh-net/src/relay/http.rs rename to iroh-relay/src/http.rs index 25474e8567c..13b19e71e01 100644 --- a/iroh-net/src/relay/http.rs +++ b/iroh-relay/src/http.rs @@ -2,25 +2,20 @@ pub(crate) const HTTP_UPGRADE_PROTOCOL: &str = "iroh derp http"; pub(crate) const WEBSOCKET_UPGRADE_PROTOCOL: &str = "websocket"; -#[cfg(feature = "iroh-relay")] // only used in the server for now -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] +#[cfg(feature = "server")] // only used in the server for now +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] pub(crate) const SUPPORTED_WEBSOCKET_VERSION: &str = "13"; /// The HTTP path under which the relay accepts relaying connections /// (over websockets and a custom upgrade protocol). pub const RELAY_PATH: &str = "/relay"; /// The HTTP path under which the relay allows doing latency queries for testing. -pub const RELAY_PROBE_PATH: &str = "/relay/probe"; +pub const RELAY_PROBE_PATH: &str = "/ping"; /// The legacy HTTP path under which the relay used to accept relaying connections. /// We keep this for backwards compatibility. -#[cfg(feature = "iroh-relay")] // legacy paths only used on server-side for backwards compat -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] +#[cfg(feature = "server")] // legacy paths only used on server-side for backwards compat +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] pub(crate) const LEGACY_RELAY_PATH: &str = "/derp"; -/// The legacy HTTP path under which the relay used to allow latency queries. -/// We keep this for backwards compatibility. -#[cfg(feature = "iroh-relay")] // legacy paths only used on server-side for backwards compat -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub(crate) const LEGACY_RELAY_PROBE_PATH: &str = "/derp/probe"; /// The HTTP upgrade protocol used for relaying. #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/iroh-relay/src/lib.rs b/iroh-relay/src/lib.rs new file mode 100644 index 00000000000..406a34d1b89 --- /dev/null +++ b/iroh-relay/src/lib.rs @@ -0,0 +1,42 @@ +//! Iroh's relay is a feature within [iroh](https://github.com/n0-computer/iroh), a peer-to-peer +//! networking system designed to facilitate direct, encrypted connections between devices. Iroh +//! aims to simplify decentralized communication by automatically handling connections through +//! "relays" when direct connections aren't immediately possible. The relay server helps establish +//! connections by temporarily routing encrypted traffic until a direct, P2P connection is +//! feasible. Once this direct path is set up, the relay server steps back, and the data flows +//! directly between devices. This approach allows Iroh to maintain a secure, low-latency +//! connection, even in challenging network situations. +//! +//! This crate provides a complete setup for creating and interacting with iroh relays, including: +//! - [`protos::relay`]: The protocol used to communicate between relay servers and clients. It's a +//! revised version of the Designated Encrypted Relay for Packets (DERP) protocol written by +//! Tailscale. +//! - [`server`]: A fully-fledged iroh-relay server over HTTP or HTTPS. Optionally will also +//! expose a stun endpoint and metrics. +//! - [`client`]: A client for establishing connections to the relay. +//! - *Server Binary*: A CLI for running your own relay server. It can be configured to also offer +//! STUN support and expose metrics. +// Based on tailscale/derp/derp.go + +#![cfg_attr(iroh_docsrs, feature(doc_cfg))] +#![deny(missing_docs, rustdoc::broken_intra_doc_links)] + +pub mod client; +pub mod defaults; +pub mod http; +pub mod protos; +#[cfg(feature = "server")] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] +pub mod server; + +#[cfg(test)] +mod dns; + +pub use iroh_base::node_addr::RelayUrl; +pub use protos::relay::MAX_PACKET_SIZE; + +pub use self::client::{ + conn::{Conn as RelayConn, ReceivedMessage}, + Client as HttpClient, ClientBuilder as HttpClientBuilder, ClientError as HttpClientError, + ClientReceiver as HttpClientReceiver, +}; diff --git a/iroh-net/src/bin/iroh-relay.rs b/iroh-relay/src/main.rs similarity index 91% rename from iroh-net/src/bin/iroh-relay.rs rename to iroh-relay/src/main.rs index 1c9cffb818a..aad6b953527 100644 --- a/iroh-net/src/bin/iroh-relay.rs +++ b/iroh-relay/src/main.rs @@ -10,15 +10,13 @@ use std::{ use anyhow::{anyhow, bail, Context as _, Result}; use clap::Parser; -use iroh_net::{ +use iroh_relay::{ defaults::{DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT, DEFAULT_METRICS_PORT, DEFAULT_STUN_PORT}, - key::SecretKey, - relay::server as iroh_relay, + server as relay, }; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tokio_rustls_acme::{caches::DirCache, AcmeConfig}; -use tracing::{debug, info}; +use tracing::debug; use tracing_subscriber::{prelude::*, EnvFilter}; /// The default `http_bind_port` when using `--dev`. @@ -94,16 +92,8 @@ fn load_secret_key( /// Configuration for the relay-server. /// /// This is (de)serialised to/from a TOML config file. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] struct Config { - /// The iroh [`SecretKey`] for this relay server. - /// - /// If not specified a new key will be generated and the config file will be re-written - /// using it. - #[serde_as(as = "DisplayFromStr")] - #[serde(default = "SecretKey::generate")] - secret_key: SecretKey, /// Whether to enable the Relay server. /// /// Defaults to `true`. @@ -178,7 +168,6 @@ impl Config { impl Default for Config { fn default() -> Self { Self { - secret_key: SecretKey::generate(), enable_relay: true, http_bind_addr: None, tls: None, @@ -321,10 +310,6 @@ impl Config { .await .context("unable to read config")?; let config: Self = toml::from_str(&config_ser).context("config file must be valid toml")?; - if !config_ser.contains("secret_key") { - info!("generating new secret key and updating config file"); - config.write_to_file(path).await?; - } Ok(config) } @@ -366,7 +351,7 @@ async fn main() -> Result<()> { let relay_config = build_relay_config(cfg).await?; debug!("{relay_config:#?}"); - let mut relay = iroh_relay::Server::spawn(relay_config).await?; + let mut relay = relay::Server::spawn(relay_config).await?; tokio::select! { biased; @@ -377,8 +362,8 @@ async fn main() -> Result<()> { relay.shutdown().await } -/// Convert the TOML-loaded config to the [`iroh_relay::RelayConfig`] format. -async fn build_relay_config(cfg: Config) -> Result> { +/// Convert the TOML-loaded config to the [`relay::RelayConfig`] format. +async fn build_relay_config(cfg: Config) -> Result> { let tls = match cfg.tls { Some(ref tls) => { let cert_config = match tls.cert_mode { @@ -392,7 +377,7 @@ async fn build_relay_config(cfg: Config) -> Result { let hostname = tls @@ -407,17 +392,17 @@ async fn build_relay_config(cfg: Config) -> Result None, }; - let limits = iroh_relay::Limits { + let limits = relay::Limits { accept_conn_limit: cfg .limits .as_ref() @@ -429,24 +414,19 @@ async fn build_relay_config(cfg: Config) -> Result bool { + if p.len() < MESSAGE_HEADER_LEN { + return false; + } + + &p[..MAGIC_LEN] == MAGIC.as_bytes() +} diff --git a/iroh-net/src/relay/codec.rs b/iroh-relay/src/protos/relay.rs similarity index 96% rename from iroh-net/src/relay/codec.rs rename to iroh-relay/src/protos/relay.rs index f6de44142c3..c073e5524d3 100644 --- a/iroh-net/src/relay/codec.rs +++ b/iroh-relay/src/protos/relay.rs @@ -1,18 +1,18 @@ +//! This module implements the relaying protocol used the [`crate::server`] and [`crate::client`]. + use std::time::Duration; use anyhow::{bail, ensure}; use bytes::{Buf, BufMut, Bytes, BytesMut}; -#[cfg(feature = "iroh-relay")] +#[cfg(feature = "server")] use futures_lite::{Stream, StreamExt}; use futures_sink::Sink; use futures_util::SinkExt; -use iroh_base::key::{Signature, PUBLIC_KEY_LENGTH}; +use iroh_base::key::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH}; use postcard::experimental::max_size::MaxSize; use serde::{Deserialize, Serialize}; use tokio_util::codec::{Decoder, Encoder}; -use crate::key::{PublicKey, SecretKey}; - /// The maximum size of a packet sent over relay. /// (This only includes the data bytes visible to magicsock, not /// including its on-wire framing overhead) @@ -23,16 +23,16 @@ const MAX_FRAME_SIZE: usize = 1024 * 1024; /// The Relay magic number, sent in the FrameType::ClientInfo frame upon initial connection. const MAGIC: &str = "RELAY🔑"; -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub(super) const KEEP_ALIVE: Duration = Duration::from_secs(60); +#[cfg(feature = "server")] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] +pub(crate) const KEEP_ALIVE: Duration = Duration::from_secs(60); // TODO: what should this be? -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub(super) const SERVER_CHANNEL_SIZE: usize = 1024 * 100; +#[cfg(feature = "server")] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] +pub(crate) const SERVER_CHANNEL_SIZE: usize = 1024 * 100; /// The number of packets buffered for sending per client -pub(super) const PER_CLIENT_SEND_QUEUE_DEPTH: usize = 512; //32; -pub(super) const PER_CLIENT_READ_QUEUE_DEPTH: usize = 512; +pub(crate) const PER_CLIENT_SEND_QUEUE_DEPTH: usize = 512; //32; +pub(crate) const PER_CLIENT_READ_QUEUE_DEPTH: usize = 512; /// ProtocolVersion is bumped whenever there's a wire-incompatible change. /// - version 1 (zero on wire): consistent box headers, in use by employee dev nodes a bit @@ -46,7 +46,7 @@ pub(super) const PER_CLIENT_READ_QUEUE_DEPTH: usize = 512; /// The server will error on that connection if a client sends one of these frames. /// This materially affects the handshake protocol, and so relay nodes on version 3 will be unable to communicate /// with nodes running earlier protocol versions. -pub(super) const PROTOCOL_VERSION: usize = 3; +pub(crate) const PROTOCOL_VERSION: usize = 3; /// /// Protocol flow: @@ -60,7 +60,6 @@ pub(super) const PROTOCOL_VERSION: usize = 3; /// * client responds to any FrameType::Ping with a FrameType::Pong /// * clients sends FrameType::SendPacket /// * server then sends FrameType::RecvPacket to recipient -/// const PREFERRED: u8 = 1u8; /// indicates this is NOT the client's home node @@ -128,7 +127,7 @@ pub(crate) struct ClientInfo { /// Ignores the timeout if `None` /// /// Does not flush. -pub(super) async fn write_frame + Unpin>( +pub(crate) async fn write_frame + Unpin>( mut writer: S, frame: Frame, timeout: Option, @@ -167,9 +166,9 @@ pub(crate) async fn send_client_key + Unp /// Reads the `FrameType::ClientInfo` frame from the client (its proof of identity) /// upon it's initial connection. -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub(super) async fn recv_client_key> + Unpin>( +#[cfg(any(test, feature = "server"))] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] +pub(crate) async fn recv_client_key> + Unpin>( stream: S, ) -> anyhow::Result<(PublicKey, ClientInfo)> { use anyhow::Context; @@ -242,7 +241,7 @@ pub(crate) enum Frame { } impl Frame { - pub(super) fn typ(&self) -> FrameType { + pub(crate) fn typ(&self) -> FrameType { match self { Frame::ClientInfo { .. } => FrameType::ClientInfo, Frame::SendPacket { .. } => FrameType::SendPacket, @@ -258,7 +257,7 @@ impl Frame { } /// Serialized length (without the frame header) - pub(super) fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { match self { Frame::ClientInfo { client_public_key: _, @@ -538,9 +537,9 @@ impl Encoder for DerpCodec { /// Receives the next frame and matches the frame type. If the correct type is found returns the content, /// otherwise an error. -#[cfg(feature = "iroh-relay")] -#[cfg_attr(iroh_docsrs, doc(cfg(feature = "iroh-relay")))] -pub(super) async fn recv_frame> + Unpin>( +#[cfg(any(test, feature = "server"))] +#[cfg_attr(iroh_docsrs, doc(cfg(feature = "server")))] +pub(crate) async fn recv_frame> + Unpin>( frame_type: FrameType, mut stream: S, ) -> anyhow::Result { diff --git a/iroh-relay/src/protos/stun.rs b/iroh-relay/src/protos/stun.rs new file mode 100644 index 00000000000..67c31b62225 --- /dev/null +++ b/iroh-relay/src/protos/stun.rs @@ -0,0 +1,388 @@ +//! STUN packets sending and receiving. + +use std::net::SocketAddr; + +use stun_rs::{ + attributes::stun::{Fingerprint, XorMappedAddress}, + DecoderContextBuilder, MessageDecoderBuilder, MessageEncoderBuilder, StunMessageBuilder, +}; +pub use stun_rs::{ + attributes::StunAttribute, error::StunDecodeError, methods, MessageClass, MessageDecoder, + TransactionId, +}; + +/// Errors that can occur when handling a STUN packet. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The STUN message could not be parsed or is otherwise invalid. + #[error("invalid message")] + InvalidMessage, + /// STUN request is not a binding request when it should be. + #[error("not binding")] + NotBinding, + /// STUN packet is not a response when it should be. + #[error("not success response")] + NotSuccessResponse, + /// STUN response has malformed attributes. + #[error("malformed attributes")] + MalformedAttrs, + /// STUN request didn't end in fingerprint. + #[error("no fingerprint")] + NoFingerprint, + /// STUN request had bogus fingerprint. + #[error("invalid fingerprint")] + InvalidFingerprint, +} + +/// Generates a binding request STUN packet. +pub fn request(tx: TransactionId) -> Vec { + let fp = Fingerprint::default(); + let msg = StunMessageBuilder::new(methods::BINDING, MessageClass::Request) + .with_transaction_id(tx) + .with_attribute(fp) + .build(); + + let encoder = MessageEncoderBuilder::default().build(); + let mut buffer = vec![0u8; 150]; + let size = encoder.encode(&mut buffer, &msg).expect("invalid encoding"); + buffer.truncate(size); + buffer +} + +/// Generates a binding response. +pub fn response(tx: TransactionId, addr: SocketAddr) -> Vec { + let msg = StunMessageBuilder::new(methods::BINDING, MessageClass::SuccessResponse) + .with_transaction_id(tx) + .with_attribute(XorMappedAddress::from(addr)) + .build(); + + let encoder = MessageEncoderBuilder::default().build(); + let mut buffer = vec![0u8; 150]; + let size = encoder.encode(&mut buffer, &msg).expect("invalid encoding"); + buffer.truncate(size); + buffer +} + +// Copied from stun_rs +// const MAGIC_COOKIE: Cookie = Cookie(0x2112_A442); +const COOKIE: [u8; 4] = 0x2112_A442u32.to_be_bytes(); + +/// Reports whether b is a STUN message. +pub fn is(b: &[u8]) -> bool { + b.len() >= stun_rs::MESSAGE_HEADER_SIZE && + b[0]&0b11000000 == 0 && // top two bits must be zero + b[4..8] == COOKIE +} + +/// Parses a STUN binding request. +pub fn parse_binding_request(b: &[u8]) -> Result { + let ctx = DecoderContextBuilder::default() + .with_validation() // ensure fingerprint is validated + .build(); + let decoder = MessageDecoderBuilder::default().with_context(ctx).build(); + let (msg, _) = decoder.decode(b).map_err(|_| Error::InvalidMessage)?; + + let tx = *msg.transaction_id(); + if msg.method() != methods::BINDING { + return Err(Error::NotBinding); + } + + // TODO: Tailscale sets the software to tailscale, we should check if we want to do this too. + + if msg + .attributes() + .last() + .map(|attr| !attr.is_fingerprint()) + .unwrap_or_default() + { + return Err(Error::NoFingerprint); + } + + Ok(tx) +} + +/// Parses a successful binding response STUN packet. +/// The IP address is extracted from the XOR-MAPPED-ADDRESS attribute. +pub fn parse_response(b: &[u8]) -> Result<(TransactionId, SocketAddr), Error> { + let decoder = MessageDecoder::default(); + let (msg, _) = decoder.decode(b).map_err(|_| Error::InvalidMessage)?; + + let tx = *msg.transaction_id(); + if msg.class() != MessageClass::SuccessResponse { + return Err(Error::NotSuccessResponse); + } + + // Read through the attributes. + // The the addr+port reported by XOR-MAPPED-ADDRESS + // as the canonical value. If the attribute is not + // present but the STUN server responds with + // MAPPED-ADDRESS we fall back to it. + + let mut addr = None; + let mut fallback_addr = None; + for attr in msg.attributes() { + match attr { + StunAttribute::XorMappedAddress(a) => { + let mut a = *a.socket_address(); + a.set_ip(a.ip().to_canonical()); + addr = Some(a); + } + StunAttribute::MappedAddress(a) => { + let mut a = *a.socket_address(); + a.set_ip(a.ip().to_canonical()); + fallback_addr = Some(a); + } + _ => {} + } + } + + if let Some(addr) = addr { + return Ok((tx, addr)); + } + + if let Some(addr) = fallback_addr { + return Ok((tx, addr)); + } + + Err(Error::MalformedAttrs) +} + +#[cfg(test)] +mod tests { + + use std::net::{IpAddr, Ipv4Addr}; + + use super::*; + + struct ResponseTestCase { + name: &'static str, + data: Vec, + want_tid: Vec, + want_addr: IpAddr, + want_port: u16, + } + + #[test] + fn test_parse_response() { + let cases = vec![ + ResponseTestCase { + name: "google-1", + data: vec![ + 0x01, 0x01, 0x00, 0x0c, 0x21, 0x12, 0xa4, 0x42, + 0x23, 0x60, 0xb1, 0x1e, 0x3e, 0xc6, 0x8f, 0xfa, + 0x93, 0xe0, 0x80, 0x07, 0x00, 0x20, 0x00, 0x08, + 0x00, 0x01, 0xc7, 0x86, 0x69, 0x57, 0x85, 0x6f, + ], + want_tid: vec![ + 0x23, 0x60, 0xb1, 0x1e, 0x3e, 0xc6, 0x8f, 0xfa, + 0x93, 0xe0, 0x80, 0x07, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), + want_port: 59028, + }, + ResponseTestCase { + name: "google-2", + data: vec![ + 0x01, 0x01, 0x00, 0x0c, 0x21, 0x12, 0xa4, 0x42, + 0xf9, 0xf1, 0x21, 0xcb, 0xde, 0x7d, 0x7c, 0x75, + 0x92, 0x3c, 0xe2, 0x71, 0x00, 0x20, 0x00, 0x08, + 0x00, 0x01, 0xc7, 0x87, 0x69, 0x57, 0x85, 0x6f, + ], + want_tid: vec![ + 0xf9, 0xf1, 0x21, 0xcb, 0xde, 0x7d, 0x7c, 0x75, + 0x92, 0x3c, 0xe2, 0x71, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), + want_port: 59029, + }, + ResponseTestCase{ + name: "stun.sipgate.net:10000", + data: vec![ + 0x01, 0x01, 0x00, 0x44, 0x21, 0x12, 0xa4, 0x42, + 0x48, 0x2e, 0xb6, 0x47, 0x15, 0xe8, 0xb2, 0x8e, + 0xae, 0xad, 0x64, 0x44, 0x00, 0x01, 0x00, 0x08, + 0x00, 0x01, 0xe4, 0xab, 0x48, 0x45, 0x21, 0x2d, + 0x00, 0x04, 0x00, 0x08, 0x00, 0x01, 0x27, 0x10, + 0xd9, 0x0a, 0x44, 0x98, 0x00, 0x05, 0x00, 0x08, + 0x00, 0x01, 0x27, 0x11, 0xd9, 0x74, 0x7a, 0x8a, + 0x80, 0x20, 0x00, 0x08, 0x00, 0x01, 0xc5, 0xb9, + 0x69, 0x57, 0x85, 0x6f, 0x80, 0x22, 0x00, 0x10, + 0x56, 0x6f, 0x76, 0x69, 0x64, 0x61, 0x2e, 0x6f, + 0x72, 0x67, 0x20, 0x30, 0x2e, 0x39, 0x36, 0x00, + ], + want_tid: vec![ + 0x48, 0x2e, 0xb6, 0x47, 0x15, 0xe8, 0xb2, 0x8e, + 0xae, 0xad, 0x64, 0x44, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), + want_port: 58539, + }, + ResponseTestCase{ + name: "stun.powervoip.com:3478", + data: vec![ + 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, + 0x7e, 0x57, 0x96, 0x68, 0x29, 0xf4, 0x44, 0x60, + 0x9d, 0x1d, 0xea, 0xa6, 0x00, 0x01, 0x00, 0x08, + 0x00, 0x01, 0xe9, 0xd3, 0x48, 0x45, 0x21, 0x2d, + 0x00, 0x04, 0x00, 0x08, 0x00, 0x01, 0x0d, 0x96, + 0x4d, 0x48, 0xa9, 0xd4, 0x00, 0x05, 0x00, 0x08, + 0x00, 0x01, 0x0d, 0x97, 0x4d, 0x48, 0xa9, 0xd5, + ], + want_tid: vec![ + 0x7e, 0x57, 0x96, 0x68, 0x29, 0xf4, 0x44, 0x60, + 0x9d, 0x1d, 0xea, 0xa6, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([72, 69, 33, 45])), + want_port: 59859, + }, + ResponseTestCase{ + name: "in-process pion server", + data: vec![ + 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x0a, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x00, 0x00, 0x00, 0x20, 0x00, 0x08, + 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, + 0x80, 0x28, 0x00, 0x04, 0xb6, 0x99, 0xbb, 0x02, + 0x01, 0x01, 0x00, 0x24, 0x21, 0x12, 0xa4, 0x42, + ], + want_tid: vec![ + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), + want_port: 61300, + }, + ResponseTestCase{ + name: "stuntman-server ipv6", + data: vec![ + 0x01, 0x01, 0x00, 0x48, 0x21, 0x12, 0xa4, 0x42, + 0x06, 0xf5, 0x66, 0x85, 0xd2, 0x8a, 0xf3, 0xe6, + 0x9c, 0xe3, 0x41, 0xe2, 0x00, 0x01, 0x00, 0x14, + 0x00, 0x02, 0x90, 0xce, 0x26, 0x02, 0x00, 0xd1, + 0xb4, 0xcf, 0xc1, 0x00, 0x38, 0xb2, 0x31, 0xff, + 0xfe, 0xef, 0x96, 0xf6, 0x80, 0x2b, 0x00, 0x14, + 0x00, 0x02, 0x0d, 0x96, 0x26, 0x04, 0xa8, 0x80, + 0x00, 0x02, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc5, 0x70, 0x01, 0x00, 0x20, 0x00, 0x14, + 0x00, 0x02, 0xb1, 0xdc, 0x07, 0x10, 0xa4, 0x93, + 0xb2, 0x3a, 0xa7, 0x85, 0xea, 0x38, 0xc2, 0x19, + 0x62, 0x0c, 0xd7, 0x14, + ], + want_tid: vec![ + 6, 245, 102, 133, 210, 138, 243, 230, 156, 227, + 65, 226, + ], + want_addr: "2602:d1:b4cf:c100:38b2:31ff:feef:96f6".parse().unwrap(), + want_port: 37070, + }, + // Testing STUN attribute padding rules using STUN software attribute + // with values of 1 & 3 length respectively before the XorMappedAddress attribute + ResponseTestCase { + name: "software-a", + data: vec![ + 0x01, 0x01, 0x00, 0x14, 0x21, 0x12, 0xa4, 0x42, + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x01, + 0x61, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x08, + 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, + ], + want_tid: vec![ + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), + want_port: 61300, + }, + ResponseTestCase { + name: "software-abc", + data: vec![ + 0x01, 0x01, 0x00, 0x14, 0x21, 0x12, 0xa4, 0x42, + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, 0x80, 0x22, 0x00, 0x03, + 0x61, 0x62, 0x63, 0x00, 0x00, 0x20, 0x00, 0x08, + 0x00, 0x01, 0xce, 0x66, 0x5e, 0x12, 0xa4, 0x43, + ], + want_tid: vec![ + 0xeb, 0xc2, 0xd3, 0x6e, 0xf4, 0x71, 0x21, 0x7c, + 0x4f, 0x3e, 0x30, 0x8e, + ], + want_addr: IpAddr::V4(Ipv4Addr::from([127, 0, 0, 1])), + want_port: 61300, + }, + ResponseTestCase { + name: "no-4in6", + data: hex::decode("010100182112a4424fd5d202dcb37d31fc773306002000140002cd3d2112a4424fd5d202dcb382ce2dc3fcc7").unwrap(), + want_tid: vec![79, 213, 210, 2, 220, 179, 125, 49, 252, 119, 51, 6], + want_addr: IpAddr::V4(Ipv4Addr::from([209, 180, 207, 193])), + want_port: 60463, + }, + ]; + + for (i, test) in cases.into_iter().enumerate() { + println!("Case {i}: {}", test.name); + let (tx, addr_port) = parse_response(&test.data).unwrap(); + assert!(is(&test.data)); + assert_eq!(tx.as_bytes(), &test.want_tid[..]); + assert_eq!(addr_port.ip(), test.want_addr); + assert_eq!(addr_port.port(), test.want_port); + } + } + + #[test] + fn test_parse_binding_request() { + let tx = TransactionId::default(); + let req = request(tx); + assert!(is(&req)); + let got_tx = parse_binding_request(&req).unwrap(); + assert_eq!(got_tx, tx); + } + + #[test] + fn test_stun_cookie() { + assert_eq!(stun_rs::MAGIC_COOKIE, COOKIE); + } + + #[test] + fn test_response() { + let txn = |n| TransactionId::from([n; 12]); + + struct Case { + tx: TransactionId, + addr: IpAddr, + port: u16, + } + let tests = vec![ + Case { + tx: txn(1), + addr: "1.2.3.4".parse().unwrap(), + port: 254, + }, + Case { + tx: txn(2), + addr: "1.2.3.4".parse().unwrap(), + port: 257, + }, + Case { + tx: txn(3), + addr: "1::4".parse().unwrap(), + port: 254, + }, + Case { + tx: txn(4), + addr: "1::4".parse().unwrap(), + port: 257, + }, + ]; + + for tt in tests { + let res = response(tt.tx, SocketAddr::new(tt.addr, tt.port)); + assert!(is(&res)); + let (tx2, addr2) = parse_response(&res).unwrap(); + assert_eq!(tt.tx, tx2); + assert_eq!(tt.addr, addr2.ip()); + assert_eq!(tt.port, addr2.port()); + } + } +} diff --git a/iroh-net/src/relay/server.rs b/iroh-relay/src/server.rs similarity index 92% rename from iroh-net/src/relay/server.rs rename to iroh-relay/src/server.rs index 74242c59db0..400abce535c 100644 --- a/iroh-net/src/relay/server.rs +++ b/iroh-relay/src/server.rs @@ -8,6 +8,13 @@ //! always attached to a handle and when the handle is dropped the tasks abort. So tasks //! can not outlive their handle. It is also always possible to await for completion of a //! task. Some tasks additionally have a method to do graceful shutdown. +//! +//! The relay server hosts the following services: +//! +//! - HTTPS `/relay`: The main URL endpoint to which clients connect and sends traffic over. +//! - HTTPS `/ping`: Used for netcheck probes. +//! - HTTPS `/generate_204`: Used for netcheck probes. +//! - STUN: UDP port for STUN requests/responses. use std::{fmt, future::Future, net::SocketAddr, pin::Pin, sync::Arc}; @@ -27,11 +34,7 @@ use tokio::{ use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, info, info_span, instrument, trace, warn, Instrument}; -use crate::{ - key::SecretKey, - relay::http::{LEGACY_RELAY_PROBE_PATH, RELAY_PROBE_PATH}, - stun, -}; +use crate::{http::RELAY_PROBE_PATH, protos::stun}; pub(crate) mod actor; pub(crate) mod client_conn; @@ -50,7 +53,6 @@ pub use self::{ const NO_CONTENT_CHALLENGE_HEADER: &str = "X-Tailscale-Challenge"; const NO_CONTENT_RESPONSE_HEADER: &str = "X-Tailscale-Response"; const NOTFOUND: &[u8] = b"Not Found"; -const RELAY_DISABLED: &[u8] = b"relay server disabled"; const ROBOTS_TXT: &[u8] = b"User-agent: *\nDisallow: /\n"; const INDEX: &[u8] = br#"

Iroh Relay

@@ -94,8 +96,6 @@ pub struct ServerConfig { /// endpoint is only one of the services served. #[derive(Debug)] pub struct RelayConfig { - /// The iroh secret key of the Relay server. - pub secret_key: SecretKey, /// The socket address on which the Relay HTTP server should bind. /// /// Normally you'd choose port `80`. The bind address for the HTTPS server is @@ -186,6 +186,11 @@ pub struct Server { relay_handle: Option, /// The main task running the server. supervisor: AbortOnDropHandle>, + /// The certificate for the server. + /// + /// If the server has manual certificates configured the certificate chain will be + /// available here, this can be used by a client to authenticate the server. + certificates: Option>>, } impl Server { @@ -203,7 +208,7 @@ impl Server { use iroh_metrics::core::Metric; iroh_metrics::core::Core::init(|reg, metrics| { - metrics.insert(crate::metrics::RelayMetrics::new(reg)); + metrics.insert(metrics::Metrics::new(reg)); metrics.insert(StunMetrics::new(reg)); }); tasks.spawn( @@ -231,7 +236,13 @@ impl Server { None => None, }; - // Start the Relay server. + // Start the Relay server, but first clone the certs out. + let certificates = config.relay.as_ref().and_then(|relay| { + relay.tls.as_ref().and_then(|tls| match tls.cert { + CertConfig::LetsEncrypt { .. } => None, + CertConfig::Manual { ref certs, .. } => Some(certs.clone()), + }) + }); let (relay_server, http_addr) = match config.relay { Some(relay_config) => { debug!("Starting Relay server"); @@ -244,16 +255,9 @@ impl Server { None => relay_config.http_bind_addr, }; let mut builder = http_server::ServerBuilder::new(relay_bind_addr) - .secret_key(Some(relay_config.secret_key)) .headers(headers) - .relay_override(Box::new(relay_disabled_handler)) .request_handler(Method::GET, "/", Box::new(root_handler)) .request_handler(Method::GET, "/index.html", Box::new(root_handler)) - .request_handler( - Method::GET, - LEGACY_RELAY_PROBE_PATH, - Box::new(probe_handler), - ) // backwards compat .request_handler(Method::GET, RELAY_PROBE_PATH, Box::new(probe_handler)) .request_handler(Method::GET, "/robots.txt", Box::new(robots_handler)); let http_addr = match relay_config.tls { @@ -290,7 +294,7 @@ impl Server { } CertConfig::Manual { private_key, certs } => { let server_config = - server_config.with_single_cert(certs.clone(), private_key)?; + server_config.with_single_cert(certs, private_key)?; let server_config = Arc::new(server_config); let acceptor = tokio_rustls::TlsAcceptor::from(server_config.clone()); @@ -342,6 +346,7 @@ impl Server { https_addr: http_addr.and(relay_addr), relay_handle, supervisor: AbortOnDropHandle::new(task), + certificates, }) } @@ -379,6 +384,11 @@ impl Server { pub fn stun_addr(&self) -> Option { self.stun_addr } + + /// The certificates chain if configured with manual TLS certificates. + pub fn certificates(&self) -> Option>> { + self.certificates.clone() + } } /// Supervisor for the relay server tasks. @@ -446,7 +456,14 @@ async fn server_stun_listener(sock: UdpSocket) -> Result<()> { loop { tokio::select! { biased; - _ = tasks.join_next(), if !tasks.is_empty() => (), + + Some(res) = tasks.join_next(), if !tasks.is_empty() => { + if let Err(err) = res { + if err.is_panic() { + panic!("task panicked: {:#?}", err); + } + } + } res = sock.recv_from(&mut buffer) => { match res { Ok((n, src_addr)) => { @@ -518,16 +535,6 @@ async fn handle_stun_request(src_addr: SocketAddr, pkt: Vec, sock: Arc, - response: ResponseBuilder, -) -> HyperResult> { - response - .status(StatusCode::NOT_FOUND) - .body(RELAY_DISABLED.into()) - .map_err(|err| Box::new(err) as HyperError) -} - fn root_handler( _r: Request, response: ResponseBuilder, @@ -605,28 +612,42 @@ async fn run_captive_portal_service(http_listener: TcpListener) -> Result<()> { let mut tasks = JoinSet::new(); loop { - match http_listener.accept().await { - Ok((stream, peer_addr)) => { - debug!(%peer_addr, "Connection opened",); - let handler = CaptivePortalService; - - tasks.spawn(async move { - let stream = crate::relay::server::streams::MaybeTlsStream::Plain(stream); - let stream = hyper_util::rt::TokioIo::new(stream); - if let Err(err) = hyper::server::conn::http1::Builder::new() - .serve_connection(stream, handler) - .with_upgrades() - .await - { - error!("Failed to serve connection: {err:?}"); + tokio::select! { + biased; + + Some(res) = tasks.join_next(), if !tasks.is_empty() => { + if let Err(err) = res { + if err.is_panic() { + panic!("task panicked: {:#?}", err); } - }); + } } - Err(err) => { - error!( - "[CaptivePortalService] failed to accept connection: {:#?}", - err - ); + + res = http_listener.accept() => { + match res { + Ok((stream, peer_addr)) => { + debug!(%peer_addr, "Connection opened",); + let handler = CaptivePortalService; + + tasks.spawn(async move { + let stream = crate::server::streams::MaybeTlsStream::Plain(stream); + let stream = hyper_util::rt::TokioIo::new(stream); + if let Err(err) = hyper::server::conn::http1::Builder::new() + .serve_connection(stream, handler) + .with_upgrades() + .await + { + error!("Failed to serve connection: {err:?}"); + } + }); + } + Err(err) => { + error!( + "[CaptivePortalService] failed to accept connection: {:#?}", + err + ); + } + } } } } @@ -712,10 +733,10 @@ mod tests { use bytes::Bytes; use http::header::UPGRADE; - use iroh_base::node_addr::RelayUrl; + use iroh_base::{key::SecretKey, node_addr::RelayUrl}; use super::*; - use crate::relay::{ + use crate::{ client::{conn::ReceivedMessage, ClientBuilder}, http::{Protocol, HTTP_UPGRADE_PROTOCOL}, }; @@ -723,7 +744,6 @@ mod tests { async fn spawn_local_relay() -> Result { Server::spawn(ServerConfig::<(), ()> { relay: Some(RelayConfig { - secret_key: SecretKey::generate(), http_bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), tls: None, limits: Default::default(), @@ -752,7 +772,6 @@ mod tests { let _guard = iroh_test::logging::setup(); let mut server = Server::spawn(ServerConfig::<(), ()> { relay: Some(RelayConfig { - secret_key: SecretKey::generate(), http_bind_addr: (Ipv4Addr::LOCALHOST, 1234).into(), tls: None, limits: Default::default(), diff --git a/iroh-net/src/relay/server/actor.rs b/iroh-relay/src/server/actor.rs similarity index 85% rename from iroh-net/src/relay/server/actor.rs rename to iroh-relay/src/server/actor.rs index 97ee3c3d14d..e07ec6c9def 100644 --- a/iroh-net/src/relay/server/actor.rs +++ b/iroh-relay/src/server/actor.rs @@ -13,6 +13,7 @@ use std::{ use anyhow::{bail, Context as _, Result}; use hyper::HeaderMap; +use iroh_base::key::PublicKey; use iroh_metrics::{core::UsageStatsReport, inc, inc_by, report_usage_stats}; use time::{Date, OffsetDateTime}; use tokio::sync::mpsc; @@ -22,21 +23,18 @@ use tracing::{info_span, trace, Instrument}; use tungstenite::protocol::Role; use crate::{ - defaults::timeouts::relay::SERVER_WRITE_TIMEOUT as WRITE_TIMEOUT, - key::{PublicKey, SecretKey}, - relay::{ - codec::{ - recv_client_key, DerpCodec, PER_CLIENT_SEND_QUEUE_DEPTH, PROTOCOL_VERSION, - SERVER_CHANNEL_SIZE, - }, - http::Protocol, - server::{ - client_conn::ClientConnBuilder, - clients::Clients, - metrics::Metrics, - streams::{MaybeTlsStream, RelayIo}, - types::ServerMessage, - }, + defaults::timeouts::SERVER_WRITE_TIMEOUT as WRITE_TIMEOUT, + http::Protocol, + protos::relay::{ + recv_client_key, DerpCodec, PER_CLIENT_SEND_QUEUE_DEPTH, PROTOCOL_VERSION, + SERVER_CHANNEL_SIZE, + }, + server::{ + client_conn::ClientConnBuilder, + clients::Clients, + metrics::Metrics, + streams::{MaybeTlsStream, RelayIo}, + types::ServerMessage, }, }; @@ -52,16 +50,12 @@ fn new_conn_num() -> usize { /// Will forcefully abort the server actor loop when dropped. /// For stopping gracefully, use [`ServerActorTask::close`]. /// -/// Responsible for managing connections to relay [`Conn`](crate::relay::RelayConn)s, sending packets from one client to another. +/// Responsible for managing connections to relay [`Conn`](crate::RelayConn)s, sending packets from one client to another. #[derive(Debug)] pub struct ServerActorTask { /// Optionally specifies how long to wait before failing when writing /// to a client write_timeout: Option, - /// secret_key of the client - secret_key: SecretKey, - /// The DER encoded x509 cert to send after `LetsEncrypt` cert+intermediate. - meta_cert: Vec, /// Channel on which to communicate to the [`ServerActor`] server_channel: mpsc::Sender, /// When true, the server has been shutdown. @@ -73,37 +67,30 @@ pub struct ServerActorTask { // TODO: stats collection } -impl ServerActorTask { - /// TODO: replace with builder - pub fn new(key: SecretKey) -> Self { +impl Default for ServerActorTask { + fn default() -> Self { let (server_channel_s, server_channel_r) = mpsc::channel(SERVER_CHANNEL_SIZE); - let server_actor = ServerActor::new(key.public(), server_channel_r); + let server_actor = ServerActor::new(server_channel_r); let cancel_token = CancellationToken::new(); let done = cancel_token.clone(); let server_task = AbortOnDropHandle::new(tokio::spawn( - async move { server_actor.run(done).await } - .instrument(info_span!("relay.server", me = %key.public().fmt_short())), + async move { server_actor.run(done).await }.instrument(info_span!("relay.server")), )); - let meta_cert = init_meta_cert(&key.public()); + Self { write_timeout: Some(WRITE_TIMEOUT), - secret_key: key, - meta_cert, server_channel: server_channel_s, closed: false, loop_handler: server_task, cancel: cancel_token, } } +} - /// Returns the server's secret key. - pub fn secret_key(&self) -> &SecretKey { - &self.secret_key - } - - /// Returns the server's public key. - pub fn public_key(&self) -> PublicKey { - self.secret_key.public() +impl ServerActorTask { + /// Creates a new default `ServerActorTask`. + pub fn new() -> Self { + Self::default() } /// Closes the server and waits for the connections to disconnect. @@ -142,17 +129,10 @@ impl ServerActorTask { pub fn client_conn_handler(&self, default_headers: HeaderMap) -> ClientConnHandler { ClientConnHandler { server_channel: self.server_channel.clone(), - secret_key: self.secret_key.clone(), write_timeout: self.write_timeout, default_headers: Arc::new(default_headers), } } - - /// Returns the server metadata cert that can be sent by the TLS server to - /// let the client skip a round trip during start-up. - pub fn meta_cert(&self) -> &[u8] { - &self.meta_cert - } } /// Handle incoming connections to the Server. @@ -163,7 +143,6 @@ impl ServerActorTask { #[derive(Debug)] pub struct ClientConnHandler { server_channel: mpsc::Sender, - secret_key: SecretKey, write_timeout: Option, pub(crate) default_headers: Arc, } @@ -172,7 +151,6 @@ impl Clone for ClientConnHandler { fn clone(&self) -> Self { Self { server_channel: self.server_channel.clone(), - secret_key: self.secret_key.clone(), write_timeout: self.write_timeout, default_headers: Arc::clone(&self.default_headers), } @@ -236,7 +214,6 @@ impl ClientConnHandler { } struct ServerActor { - key: PublicKey, receiver: mpsc::Receiver, /// All clients connected to this server clients: Clients, @@ -244,9 +221,8 @@ struct ServerActor { } impl ServerActor { - fn new(key: PublicKey, receiver: mpsc::Receiver) -> Self { + fn new(receiver: mpsc::Receiver) -> Self { Self { - key, receiver, clients: Clients::new(), client_counter: ClientCounter::default(), @@ -310,7 +286,7 @@ impl ServerActor { report_usage_stats(&UsageStatsReport::new( "relay_accepts".to_string(), - self.key.to_string(), + "relay_server".to_string(), // TODO: other id? 1, None, // TODO(arqu): attribute to user id; possibly with the re-introduction of request tokens or other auth Some(key.to_string()), @@ -346,36 +322,6 @@ impl ServerActor { } } -/// Initializes the [`ServerActor`] with a self-signed x509 cert -/// encoding this server's public key and protocol version. "cmd/relay_server -/// then sends this after the Let's Encrypt leaf + intermediate certs after -/// the ServerHello (encrypted in TLS 1.3, not that is matters much). -/// -/// Then the client can save a round trime getting that and can start speaking -/// relay right away. (we don't use ALPN because that's sent in the clear and -/// we're being paranoid to not look too weird to any middleboxes, given that -/// relay is an ultimate fallback path). But since the post-ServerHello certs -/// are encrypted we can have the client also use them as a signal to be able -/// to start speaking relay right away, starting with its identity proof, -/// encrypted to the server's public key. -/// -/// This RTT optimization fails where there's a corp-mandated TLS proxy with -/// corp-mandated root certs on employee machines and TLS proxy cleans up -/// unnecessary certs. In that case we just fall back to the extra RTT. -fn init_meta_cert(server_key: &PublicKey) -> Vec { - let mut params = - rcgen::CertificateParams::new([format!("derpkey{}", hex::encode(server_key.as_bytes()))]); - params.serial_number = Some((PROTOCOL_VERSION as u64).into()); - // Windows requires not_after and not_before set: - params.not_after = time::OffsetDateTime::now_utc().saturating_add(30 * time::Duration::DAY); - params.not_before = time::OffsetDateTime::now_utc().saturating_sub(30 * time::Duration::DAY); - - rcgen::Certificate::from_params(params) - .expect("fixed inputs") - .serialize_der() - .expect("fixed allocations") -} - struct ClientCounter { clients: HashMap, last_clear_date: Date, @@ -412,17 +358,18 @@ impl ClientCounter { #[cfg(test)] mod tests { use bytes::Bytes; + use iroh_base::key::SecretKey; use tokio::io::DuplexStream; use tokio_util::codec::{FramedRead, FramedWrite}; use tracing_subscriber::{prelude::*, EnvFilter}; use super::*; - use crate::relay::{ + use crate::{ client::{ conn::{ConnBuilder, ConnReader, ConnWriter, ReceivedMessage}, streams::{MaybeTlsStreamReader, MaybeTlsStreamWriter}, }, - codec::{recv_frame, ClientInfo, Frame, FrameType}, + protos::relay::{recv_frame, ClientInfo, Frame, FrameType}, }; fn test_client_builder( @@ -446,11 +393,9 @@ mod tests { #[tokio::test] async fn test_server_actor() -> Result<()> { - let server_key = SecretKey::generate().public(); - // make server actor let (server_channel, server_channel_r) = mpsc::channel(20); - let server_actor: ServerActor = ServerActor::new(server_key, server_channel_r); + let server_actor: ServerActor = ServerActor::new(server_channel_r); let done = CancellationToken::new(); let server_done = done.clone(); @@ -479,8 +424,7 @@ mod tests { // write message from b to a let msg = b"hello world!"; - crate::relay::client::conn::send_packet(&mut b_io, &None, key_a, Bytes::from_static(msg)) - .await?; + crate::client::conn::send_packet(&mut b_io, &None, key_a, Bytes::from_static(msg)).await?; // get message on a's reader let frame = recv_frame(FrameType::RecvPacket, &mut a_io).await?; @@ -518,7 +462,6 @@ mod tests { let (server_channel_s, mut server_channel_r) = mpsc::channel(10); let client_key = SecretKey::generate(); let handler = ClientConnHandler { - secret_key: client_key.clone(), write_timeout: None, server_channel: server_channel_s, default_headers: Default::default(), @@ -537,7 +480,7 @@ mod tests { let client_info = ClientInfo { version: PROTOCOL_VERSION, }; - crate::relay::codec::send_client_key(&mut client_writer, &client_key, &client_info) + crate::protos::relay::send_client_key(&mut client_writer, &client_key, &client_info) .await?; Ok(()) @@ -580,8 +523,7 @@ mod tests { let _guard = iroh_test::logging::setup(); // create the server! - let server_key = SecretKey::generate(); - let server: ServerActorTask = ServerActorTask::new(server_key); + let server: ServerActorTask = ServerActorTask::new(); // create client a and connect it to the server let key_a = SecretKey::generate(); @@ -656,8 +598,7 @@ mod tests { .ok(); // create the server! - let server_key = SecretKey::generate(); - let server: ServerActorTask = ServerActorTask::new(server_key); + let server: ServerActorTask = ServerActorTask::new(); // create client a and connect it to the server let key_a = SecretKey::generate(); diff --git a/iroh-net/src/relay/server/client_conn.rs b/iroh-relay/src/server/client_conn.rs similarity index 97% rename from iroh-net/src/relay/server/client_conn.rs rename to iroh-relay/src/server/client_conn.rs index 5d5bf4e6938..e4872c31903 100644 --- a/iroh-net/src/relay/server/client_conn.rs +++ b/iroh-relay/src/server/client_conn.rs @@ -12,28 +12,28 @@ use anyhow::{Context, Result}; use bytes::Bytes; use futures_lite::StreamExt; use futures_util::SinkExt; +use iroh_base::key::PublicKey; use iroh_metrics::{inc, inc_by}; use tokio::sync::mpsc; use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{trace, Instrument}; use crate::{ - disco::looks_like_disco_wrapper, - key::PublicKey, - relay::{ - codec::{write_frame, Frame, KEEP_ALIVE}, - server::{ - metrics::Metrics, - streams::RelayIo, - types::{Packet, ServerMessage}, - }, + protos::{ + disco, + relay::{write_frame, Frame, KEEP_ALIVE}, + }, + server::{ + metrics::Metrics, + streams::RelayIo, + types::{Packet, ServerMessage}, }, }; /// The [`Server`] side representation of a [`Client`]'s connection. /// -/// [`Server`]: crate::relay::server::Server -/// [`Client`]: crate::relay::client::Client +/// [`Server`]: crate::server::Server +/// [`Client`]: crate::client::Client #[derive(Debug)] pub(crate) struct ClientConnManager { /// Static after construction, process-wide unique counter, incremented each time we accept @@ -446,7 +446,7 @@ impl ClientConnIo { /// destination is not connected, or if the destination client can /// not fit any more messages in its queue. async fn transfer_packet(&self, dstkey: PublicKey, packet: Packet) -> Result<()> { - if looks_like_disco_wrapper(&packet.bytes) { + if disco::looks_like_disco_wrapper(&packet.bytes) { inc!(Metrics, disco_packets_recv); self.send_server(ServerMessage::SendDiscoPacket((dstkey, packet))) .await?; @@ -462,16 +462,14 @@ impl ClientConnIo { #[cfg(test)] mod tests { use anyhow::bail; + use iroh_base::key::SecretKey; use tokio_util::codec::Framed; use super::*; use crate::{ - key::SecretKey, - relay::{ - client::conn, - codec::{recv_frame, DerpCodec, FrameType}, - server::streams::MaybeTlsStream, - }, + client::conn, + protos::relay::{recv_frame, DerpCodec, FrameType}, + server::streams::MaybeTlsStream, }; #[tokio::test] @@ -585,7 +583,7 @@ mod tests { // send disco packet println!(" send disco packet"); // starts with `MAGIC` & key, then data - let mut disco_data = crate::disco::MAGIC.as_bytes().to_vec(); + let mut disco_data = disco::MAGIC.as_bytes().to_vec(); disco_data.extend_from_slice(target.as_bytes()); disco_data.extend_from_slice(data); conn::send_packet(&mut io_rw, &None, target, disco_data.clone().into()).await?; diff --git a/iroh-net/src/relay/server/clients.rs b/iroh-relay/src/server/clients.rs similarity index 98% rename from iroh-net/src/relay/server/clients.rs rename to iroh-relay/src/server/clients.rs index 755c47901df..2eb7ec6888e 100644 --- a/iroh-net/src/relay/server/clients.rs +++ b/iroh-relay/src/server/clients.rs @@ -3,6 +3,7 @@ //! The "Server" side of the client. Uses the `ClientConnManager`. use std::collections::{HashMap, HashSet}; +use iroh_base::key::PublicKey; use iroh_metrics::inc; use tokio::{sync::mpsc, task::JoinSet}; use tracing::{Instrument, Span}; @@ -12,13 +13,11 @@ use super::{ metrics::Metrics, types::Packet, }; -use crate::key::PublicKey; /// Number of times we try to send to a client connection before dropping the data; const RETRIES: usize = 3; /// Represents a connection to a client. -/// // TODO: expand to allow for _multiple connections_ associated with a single PublicKey. This // introduces some questions around which connection should be prioritized when forwarding packets // @@ -256,16 +255,14 @@ impl Clients { mod tests { use anyhow::Result; use bytes::Bytes; + use iroh_base::key::SecretKey; use tokio::io::DuplexStream; use tokio_util::codec::{Framed, FramedRead}; use super::*; use crate::{ - key::SecretKey, - relay::{ - codec::{recv_frame, DerpCodec, Frame, FrameType}, - server::streams::{MaybeTlsStream, RelayIo}, - }, + protos::relay::{recv_frame, DerpCodec, Frame, FrameType}, + server::streams::{MaybeTlsStream, RelayIo}, }; fn test_client_builder( diff --git a/iroh-net/src/relay/server/http_server.rs b/iroh-relay/src/server/http_server.rs similarity index 82% rename from iroh-net/src/relay/server/http_server.rs rename to iroh-relay/src/server/http_server.rs index 42756cf2bd8..d062a838f6c 100644 --- a/iroh-net/src/relay/server/http_server.rs +++ b/iroh-relay/src/server/http_server.rs @@ -19,13 +19,10 @@ use tracing::{debug, debug_span, error, info, info_span, warn, Instrument}; use tungstenite::handshake::derive_accept_key; use crate::{ - key::SecretKey, - relay::{ - http::{Protocol, LEGACY_RELAY_PATH, RELAY_PATH, SUPPORTED_WEBSOCKET_VERSION}, - server::{ - actor::{ClientConnHandler, ServerActorTask}, - streams::MaybeTlsStream, - }, + http::{Protocol, LEGACY_RELAY_PATH, RELAY_PATH, SUPPORTED_WEBSOCKET_VERSION}, + server::{ + actor::{ClientConnHandler, ServerActorTask}, + streams::MaybeTlsStream, }, }; @@ -148,18 +145,8 @@ pub struct TlsConfig { /// /// Defaults to handling relay requests on the "/relay" (and "/derp" for backwards compatibility) endpoint. /// Other HTTP endpoints can be added using [`ServerBuilder::request_handler`]. -/// -/// If no [`SecretKey`] is provided, it is assumed that you will provide a -/// [`ServerBuilder::relay_override`] function that handles requests to the relay -/// endpoint. Not providing a [`ServerBuilder::relay_override`] in this case will result in -/// an error on `spawn`. #[derive(derive_more::Debug)] pub struct ServerBuilder { - /// The secret key for this Server. - /// - /// When `None`, you must also provide a `relay_override` function that - /// will be run when someone hits the relay endpoint. - secret_key: Option, /// The ip + port combination for this server. addr: SocketAddr, /// Optional tls configuration/TlsAcceptor combination. @@ -171,42 +158,21 @@ pub struct ServerBuilder { /// Used when certain routes in your server should be made available at the same port as /// the relay server, and so must be handled along side requests to the relay endpoint. handlers: Handlers, - /// Use a custom relay response handler. - /// - /// Typically used when you want to disable any relay connections. - #[debug("{}", relay_override.as_ref().map_or("None", |_| "Some(Box, ResponseBuilder) -> Result + Send + Sync + 'static>)"))] - relay_override: Option, /// Headers to use for HTTP responses. headers: HeaderMap, - /// 404 not found response. - /// - /// When `None`, a default is provided. - #[debug("{}", not_found_fn.as_ref().map_or("None", |_| "Some(Box Result> + Send + Sync + 'static>)"))] - not_found_fn: Option, } impl ServerBuilder { /// Creates a new [ServerBuilder]. pub fn new(addr: SocketAddr) -> Self { Self { - secret_key: None, addr, tls_config: None, handlers: Default::default(), - relay_override: None, headers: HeaderMap::new(), - not_found_fn: None, } } - /// The [`SecretKey`] identity for this relay server. - /// - /// When set to `None`, the builder assumes you do not want to run a relay service. - pub fn secret_key(mut self, secret_key: Option) -> Self { - self.secret_key = secret_key; - self - } - /// Serves all requests content using TLS. pub fn tls_config(mut self, config: Option) -> Self { self.tls_config = config; @@ -224,21 +190,6 @@ impl ServerBuilder { self } - /// Sets a custom "404" handler. - #[allow(unused)] - pub fn not_found_handler(mut self, handler: HyperHandler) -> Self { - self.not_found_fn = Some(handler); - self - } - - /// Handles the relay endpoint in a custom way. - /// - /// This is required if no [`SecretKey`] was provided to the builder. - pub fn relay_override(mut self, handler: HyperHandler) -> Self { - self.relay_override = Some(handler); - self - } - /// Adds HTTP headers to responses. pub fn headers(mut self, headers: HeaderMap) -> Self { for (k, v) in headers.iter() { @@ -249,40 +200,10 @@ impl ServerBuilder { /// Builds and spawns an HTTP(S) Relay Server. pub async fn spawn(self) -> Result { - ensure!( - self.secret_key.is_some() || self.relay_override.is_some(), - "Must provide a `SecretKey` for the relay server OR pass in an override function for the 'relay' endpoint" - ); - let (relay_handler, relay_server) = if let Some(secret_key) = self.secret_key { - // spawns a server actor/task - let server = ServerActorTask::new(secret_key.clone()); - ( - RelayHandler::ConnHandler(server.client_conn_handler(self.headers.clone())), - Some(server), - ) - } else { - ( - RelayHandler::Override( - self.relay_override - .context("no relay handler override but also no secret key")?, - ), - None, - ) - }; - let h = self.headers.clone(); - let not_found_fn = match self.not_found_fn { - Some(f) => f, - None => Box::new(move |_req: Request, mut res: ResponseBuilder| { - for (k, v) in h.iter() { - res = res.header(k.clone(), v.clone()); - } - let body = body_full("Not Found"); - let r = res.status(StatusCode::NOT_FOUND).body(body)?; - HyperResult::Ok(r) - }), - }; + let relay_server = ServerActorTask::new(); + let relay_handler = relay_server.client_conn_handler(self.headers.clone()); - let service = RelayService::new(self.handlers, relay_handler, not_found_fn, self.headers); + let service = RelayService::new(self.handlers, relay_handler, self.headers); let server_state = ServerState { addr: self.addr, @@ -300,7 +221,7 @@ impl ServerBuilder { struct ServerState { addr: SocketAddr, tls_config: Option, - server: Option, + server: ServerActorTask, service: RelayService, } @@ -332,6 +253,13 @@ impl ServerState { _ = cancel.cancelled() => { break; } + Some(res) = set.join_next(), if !set.is_empty() => { + if let Err(err) = res { + if err.is_panic() { + panic!("task panicked: {:#?}", err); + } + } + } res = listener.accept() => match res { Ok((stream, peer_addr)) => { debug!("[{http_str}] relay: Connection opened from {peer_addr}"); @@ -360,11 +288,9 @@ impl ServerState { } } } - if let Some(server) = server { - // TODO: if the task this is running in is aborted this server is not shut - // down. - server.close().await; - } + // TODO: if the task this is running in is aborted this server is not shut + // down. + server.close().await; set.shutdown().await; debug!("[{http_str}] relay: server has been shutdown."); }.instrument(info_span!("relay-http-serve"))); @@ -501,19 +427,11 @@ impl Service> for RelayService { (req.method(), req.uri().path()), (&hyper::Method::GET, LEGACY_RELAY_PATH | RELAY_PATH) ) { - match &self.0.relay_handler { - RelayHandler::Override(f) => { - // see if we have some override response - let res = f(req, self.0.default_response()); - return Box::pin(async move { res }); - } - RelayHandler::ConnHandler(handler) => { - let h = handler.clone(); - // otherwise handle the relay connection as normal - return Box::pin(async move { h.call(req).await.map_err(Into::into) }); - } - } + let h = self.0.relay_handler.clone(); + // otherwise handle the relay connection as normal + return Box::pin(async move { h.call(req).await.map_err(Into::into) }); } + // check all other possible endpoints let uri = req.uri().clone(); if let Some(res) = self.0.handlers.get(&(req.method().clone(), uri.path())) { @@ -521,7 +439,7 @@ impl Service> for RelayService { return Box::pin(async move { f }); } // otherwise return 404 - let res = (self.0.not_found_fn)(req, self.0.default_response()); + let res = self.0.not_found_fn(req, self.0.default_response()); Box::pin(async move { res }) } } @@ -530,30 +448,11 @@ impl Service> for RelayService { #[derive(Clone, Debug)] struct RelayService(Arc); -#[derive(derive_more::Debug)] +#[derive(Debug)] struct Inner { - pub relay_handler: RelayHandler, - #[debug("Box Result> + Send + Sync + 'static>")] - pub not_found_fn: HyperHandler, - pub handlers: Handlers, - pub headers: HeaderMap, -} - -/// Action to take when a connection is made at the relay endpoint.` -#[derive(derive_more::Debug)] -enum RelayHandler { - /// Pass the connection to a [`ClientConnHandler`] to get added to the relay server. The default. - ConnHandler(ClientConnHandler), - /// Return some static response. Used when the http(s) should be running, but the relay portion - /// of the server is disabled. - // TODO: Can we remove this debug override? - Override( - #[debug( - "{}", - "Box, ResponseBuilder) -> Result + Send + Sync + 'static>" - )] - HyperHandler, - ), + relay_handler: ClientConnHandler, + handlers: Handlers, + headers: HeaderMap, } impl Inner { @@ -564,6 +463,19 @@ impl Inner { } response } + + fn not_found_fn( + &self, + _req: Request, + mut res: ResponseBuilder, + ) -> HyperResult> { + for (k, v) in self.headers.iter() { + res = res.header(k.clone(), v.clone()); + } + let body = body_full("Not Found"); + let r = res.status(StatusCode::NOT_FOUND).body(body)?; + HyperResult::Ok(r) + } } /// TLS Certificate Authority acceptor. @@ -577,16 +489,10 @@ pub enum TlsAcceptor { } impl RelayService { - fn new( - handlers: Handlers, - relay_handler: RelayHandler, - not_found_fn: HyperHandler, - headers: HeaderMap, - ) -> Self { + fn new(handlers: Handlers, relay_handler: ClientConnHandler, headers: HeaderMap) -> Self { Self(Arc::new(Inner { relay_handler, handlers, - not_found_fn, headers, })) } @@ -684,16 +590,14 @@ mod tests { use anyhow::Result; use bytes::Bytes; + use iroh_base::key::{PublicKey, SecretKey}; use reqwest::Url; use tokio::{sync::mpsc, task::JoinHandle}; use tracing::{info, info_span, Instrument}; use tracing_subscriber::{prelude::*, EnvFilter}; use super::*; - use crate::{ - key::{PublicKey, SecretKey}, - relay::client::{conn::ReceivedMessage, Client, ClientBuilder}, - }; + use crate::client::{conn::ReceivedMessage, Client, ClientBuilder}; pub(crate) fn make_tls_config() -> TlsConfig { let subject_alt_names = vec!["localhost".to_string()]; @@ -726,13 +630,11 @@ mod tests { async fn test_http_clients_and_server() -> Result<()> { let _guard = iroh_test::logging::setup(); - let server_key = SecretKey::generate(); let a_key = SecretKey::generate(); let b_key = SecretKey::generate(); // start server let server = ServerBuilder::new("127.0.0.1:0".parse().unwrap()) - .secret_key(Some(server_key)) .spawn() .await?; @@ -853,7 +755,6 @@ mod tests { .try_init() .ok(); - let server_key = SecretKey::generate(); let a_key = SecretKey::generate(); let b_key = SecretKey::generate(); @@ -862,7 +763,6 @@ mod tests { // start server let mut server = ServerBuilder::new("127.0.0.1:0".parse().unwrap()) - .secret_key(Some(server_key)) .tls_config(Some(tls_config)) .spawn() .await?; diff --git a/iroh-net/src/relay/server/metrics.rs b/iroh-relay/src/server/metrics.rs similarity index 100% rename from iroh-net/src/relay/server/metrics.rs rename to iroh-relay/src/server/metrics.rs diff --git a/iroh-net/src/relay/server/streams.rs b/iroh-relay/src/server/streams.rs similarity index 99% rename from iroh-net/src/relay/server/streams.rs rename to iroh-relay/src/server/streams.rs index 8979af9c1a3..190a658f181 100644 --- a/iroh-net/src/relay/server/streams.rs +++ b/iroh-relay/src/server/streams.rs @@ -12,7 +12,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tokio_tungstenite::WebSocketStream; use tokio_util::codec::Framed; -use crate::relay::codec::{DerpCodec, Frame}; +use crate::protos::relay::{DerpCodec, Frame}; #[derive(Debug)] pub(crate) enum RelayIo { diff --git a/iroh-relay/src/server/testing.rs b/iroh-relay/src/server/testing.rs new file mode 100644 index 00000000000..580d44addba --- /dev/null +++ b/iroh-relay/src/server/testing.rs @@ -0,0 +1,75 @@ +//! Internal utilities to support testing. +use std::net::Ipv4Addr; + +use anyhow::Result; +use tokio::sync::oneshot; + +use super::{CertConfig, RelayConfig, Server, ServerConfig, StunConfig, TlsConfig}; +use crate::{defaults::DEFAULT_STUN_PORT, RelayMap, RelayNode, RelayUrl}; + +/// A drop guard to clean up test infrastructure. +/// +/// After dropping the test infrastructure will asynchronously shutdown and release its +/// resources. +// Nightly sees the sender as dead code currently, but we only rely on Drop of the +// sender. +#[derive(Debug)] +#[allow(dead_code)] +pub struct CleanupDropGuard(pub(crate) oneshot::Sender<()>); + +/// Runs a relay server with STUN enabled suitable for tests. +/// +/// The returned `Url` is the url of the relay server in the returned [`RelayMap`]. +/// When dropped, the returned [`Server`] does will stop running. +pub async fn run_relay_server() -> Result<(RelayMap, RelayUrl, Server)> { + run_relay_server_with(Some(StunConfig { + bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), + })) + .await +} + +/// Runs a relay server. +/// +/// `stun` can be set to `None` to disable stun, or set to `Some` `StunConfig`, +/// to enable stun on a specific socket. +/// +/// The return value is similar to [`run_relay_server`]. +pub async fn run_relay_server_with( + stun: Option, +) -> Result<(RelayMap, RelayUrl, Server)> { + let cert = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .expect("valid"); + let rustls_cert = rustls::pki_types::CertificateDer::from(cert.serialize_der().unwrap()); + let private_key = + rustls::pki_types::PrivatePkcs8KeyDer::from(cert.get_key_pair().serialize_der()); + let private_key = rustls::pki_types::PrivateKeyDer::from(private_key); + + let config = ServerConfig { + relay: Some(RelayConfig { + http_bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), + tls: Some(TlsConfig { + cert: CertConfig::<(), ()>::Manual { + private_key, + certs: vec![rustls_cert], + }, + https_bind_addr: (Ipv4Addr::LOCALHOST, 0).into(), + }), + limits: Default::default(), + }), + stun, + #[cfg(feature = "metrics")] + metrics_addr: None, + }; + let server = Server::spawn(config).await.unwrap(); + let url: RelayUrl = format!("https://{}", server.https_addr().expect("configured")) + .parse() + .unwrap(); + let m = RelayMap::from_nodes([RelayNode { + url: url.clone(), + stun_only: false, + stun_port: server.stun_addr().map_or(DEFAULT_STUN_PORT, |s| s.port()), + }]) + .unwrap(); + Ok((m, url, server)) +} diff --git a/iroh-net/src/relay/server/types.rs b/iroh-relay/src/server/types.rs similarity index 87% rename from iroh-net/src/relay/server/types.rs rename to iroh-relay/src/server/types.rs index 29aac919a18..0ce67b1d1db 100644 --- a/iroh-net/src/relay/server/types.rs +++ b/iroh-relay/src/server/types.rs @@ -1,8 +1,9 @@ //! Types that are shared between [`super::actor`] and [`super::client_conn`]. use bytes::Bytes; +use iroh_base::key::PublicKey; -use crate::{key::PublicKey, relay::server::client_conn::ClientConnBuilder}; +use crate::server::client_conn::ClientConnBuilder; /// A request to write a dataframe to a Client #[derive(Debug, Clone)] diff --git a/iroh-router/Cargo.toml b/iroh-router/Cargo.toml new file mode 100644 index 00000000000..95d24ea1f2b --- /dev/null +++ b/iroh-router/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "iroh-router" +version = "0.28.0" +edition = "2021" +readme = "README.md" +description = "protocol router support for iroh" +license = "MIT OR Apache-2.0" +authors = ["dignifiedquire ", "n0 team"] +repository = "https://github.com/n0-computer/iroh" +keywords = ["quic", "networking", "holepunching", "p2p"] + + +[dependencies] +anyhow = "1.0.91" +futures-buffered = "0.2.9" +futures-lite = "2.3.0" +futures-util = "0.3.31" +iroh-net = { version = "0.28.1", path = "../iroh-net" } +tokio = "1.41.0" +tokio-util = "0.7.12" +tracing = "0.1.40" + +# Examples +clap = { version = "4", features = ["derive"], optional = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } + +[lints] +workspace = true + + +[features] +default = [] +examples = ["dep:clap", "dep:tracing-subscriber"] + +[[example]] +name = "custom-protocol" +required-features = ["examples"] diff --git a/iroh-router/README.md b/iroh-router/README.md new file mode 100644 index 00000000000..54639b5f9f7 --- /dev/null +++ b/iroh-router/README.md @@ -0,0 +1,20 @@ +# iroh-router + +This crate contains the definitions for custom protocols for `iroh`. + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-router/examples/custom-protocol.rs b/iroh-router/examples/custom-protocol.rs new file mode 100644 index 00000000000..b4bd6e8b236 --- /dev/null +++ b/iroh-router/examples/custom-protocol.rs @@ -0,0 +1,232 @@ +//! Example for adding a custom protocol. +//! +//! We are building a very simple custom protocol here. +//! +//! Our custom protocol allows querying the text stored on the other node. +//! +//! The example is contrived - we only use memory nodes, and our database is a hashmap in a mutex, +//! and our queries just match if the query string appears as-is. +//! +//! ## Usage +//! +//! In one terminal, run +//! +//! cargo run --example custom-protocol --features=examples -- listen "hello-world" "foo-bar" "hello-moon" +//! +//! This spawns an iroh endpoint with three blobs. It will print the node's node id. +//! +//! In another terminal, run +//! +//! cargo run --example custom-protocol --features=examples -- query hello +//! +//! Replace with the node id from above. This will connect to the listening node with our +//! custom protocol and query for the string `hello`. The listening node will return a number of how many +//! strings match the query. +//! +//! For this example, this will print: +//! +//! Found 2 matches +//! +//! That's it! Follow along in the code below, we added a bunch of comments to explain things. + +use std::{collections::BTreeSet, sync::Arc}; + +use anyhow::Result; +use clap::Parser; +use futures_lite::future::Boxed as BoxedFuture; +use iroh_net::{ + endpoint::{get_remote_node_id, Connecting}, + Endpoint, NodeId, +}; +use iroh_router::{ProtocolHandler, Router}; +use tokio::sync::Mutex; +use tracing_subscriber::{prelude::*, EnvFilter}; + +#[derive(Debug, Parser)] +pub struct Cli { + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Parser)] +pub enum Command { + /// Spawn a node in listening mode. + Listen { + /// Each text string will be imported as a blob and inserted into the search database. + text: Vec, + }, + /// Query a remote node for data and print the results. + Query { + /// The node id of the node we want to query. + node_id: NodeId, + /// The text we want to match. + query: String, + }, +} + +/// Each custom protocol is identified by its ALPN string. +/// +/// The ALPN, or application-layer protocol negotiation, is exchanged in the connection handshake, +/// and the connection is aborted unless both nodes pass the same bytestring. +const ALPN: &[u8] = b"iroh-example/text-search/0"; + +#[tokio::main] +async fn main() -> Result<()> { + setup_logging(); + let args = Cli::parse(); + + // Build an endpoint + let endpoint = Endpoint::builder().discovery_n0().bind().await?; + + // Build our custom protocol handler. The `builder` exposes access to various subsystems in the + // iroh node. In our case, we need a blobs client and the endpoint. + let proto = BlobSearch::new(endpoint.clone()); + + let builder = Router::builder(endpoint); + + // Add our protocol, identified by our ALPN, to the node, and spawn the node. + let router = builder.accept(ALPN.to_vec(), proto.clone()).spawn().await?; + + match args.command { + Command::Listen { text } => { + let node_id = router.endpoint().node_id(); + println!("our node id: {node_id}"); + + // Insert the text strings as blobs and index them. + for text in text.into_iter() { + proto.insert(text).await?; + } + + // Wait for Ctrl-C to be pressed. + tokio::signal::ctrl_c().await?; + } + Command::Query { node_id, query } => { + // Query the remote node. + // This will send the query over our custom protocol, read hashes on the reply stream, + // and download each hash over iroh-blobs. + let num_matches = proto.query_remote(node_id, &query).await?; + + // Print out our query results. + println!("Found {} matches", num_matches); + } + } + + router.shutdown().await?; + + Ok(()) +} + +#[derive(Debug, Clone)] +struct BlobSearch { + endpoint: Endpoint, + blobs: Arc>>, +} + +impl ProtocolHandler for BlobSearch { + /// The `accept` method is called for each incoming connection for our ALPN. + /// + /// The returned future runs on a newly spawned tokio task, so it can run as long as + /// the connection lasts. + fn accept(self: Arc, connecting: Connecting) -> BoxedFuture> { + // We have to return a boxed future from the handler. + Box::pin(async move { + // Wait for the connection to be fully established. + let connection = connecting.await?; + // We can get the remote's node id from the connection. + let node_id = get_remote_node_id(&connection)?; + println!("accepted connection from {node_id}"); + + // Our protocol is a simple request-response protocol, so we expect the + // connecting peer to open a single bi-directional stream. + let (mut send, mut recv) = connection.accept_bi().await?; + + // We read the query from the receive stream, while enforcing a max query length. + let query_bytes = recv.read_to_end(64).await?; + + // Now, we can perform the actual query on our local database. + let query = String::from_utf8(query_bytes)?; + let num_matches = self.query_local(&query).await; + + // We want to return a list of hashes. We do the simplest thing possible, and just send + // one hash after the other. Because the hashes have a fixed size of 32 bytes, this is + // very easy to parse on the other end. + send.write_all(&num_matches.to_le_bytes()).await?; + + // By calling `finish` on the send stream we signal that we will not send anything + // further, which makes the receive stream on the other end terminate. + send.finish()?; + + // Wait until the remote closes the connection, which it does once it + // received the response. + connection.closed().await; + + Ok(()) + }) + } +} + +impl BlobSearch { + /// Create a new protocol handler. + pub fn new(endpoint: Endpoint) -> Arc { + Arc::new(Self { + endpoint, + blobs: Default::default(), + }) + } + + /// Query a remote node, download all matching blobs and print the results. + pub async fn query_remote(&self, node_id: NodeId, query: &str) -> Result { + // Establish a connection to our node. + // We use the default node discovery in iroh, so we can connect by node id without + // providing further information. + let conn = self.endpoint.connect(node_id, ALPN).await?; + + // Open a bi-directional in our connection. + let (mut send, mut recv) = conn.open_bi().await?; + + // Send our query. + send.write_all(query.as_bytes()).await?; + + // Finish the send stream, signalling that no further data will be sent. + // This makes the `read_to_end` call on the accepting side terminate. + send.finish()?; + + // The response is a 64 bit integer + // We simply read it into a byte buffer. + let mut num_matches = [0u8; 8]; + + // Read 8 bytes from the stream. + recv.read_exact(&mut num_matches).await?; + + let num_matches = u64::from_le_bytes(num_matches); + + // Dropping the connection here will close it. + + Ok(num_matches) + } + + /// Query the local database. + /// + /// Returns how many matches were found. + pub async fn query_local(&self, query: &str) -> u64 { + let guard = self.blobs.lock().await; + let count: usize = guard.iter().filter(|text| text.contains(query)).count(); + count as u64 + } + + /// Insert a text string into the database. + pub async fn insert(&self, text: String) -> Result<()> { + let mut guard = self.blobs.lock().await; + guard.insert(text); + Ok(()) + } +} + +/// Set the RUST_LOG env var to one of {debug,info,warn} to see logging. +fn setup_logging() { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .with(EnvFilter::from_default_env()) + .try_init() + .ok(); +} diff --git a/iroh-router/src/lib.rs b/iroh-router/src/lib.rs new file mode 100644 index 00000000000..6bdd6cd5655 --- /dev/null +++ b/iroh-router/src/lib.rs @@ -0,0 +1,5 @@ +mod protocol; +mod router; + +pub use protocol::{ProtocolHandler, ProtocolMap}; +pub use router::{Router, RouterBuilder}; diff --git a/iroh-router/src/protocol.rs b/iroh-router/src/protocol.rs new file mode 100644 index 00000000000..6ced048992b --- /dev/null +++ b/iroh-router/src/protocol.rs @@ -0,0 +1,77 @@ +use std::{any::Any, collections::BTreeMap, sync::Arc}; + +use anyhow::Result; +use futures_buffered::join_all; +use futures_lite::future::Boxed as BoxedFuture; +use iroh_net::endpoint::Connecting; + +/// Handler for incoming connections. +/// +/// A router accepts connections for arbitrary ALPN protocols. +/// +/// With this trait, you can handle incoming connections for any protocol. +/// +/// Implement this trait on a struct that should handle incoming connections. +/// The protocol handler must then be registered on the node for an ALPN protocol with +/// [`crate::RouterBuilder::accept`]. +pub trait ProtocolHandler: Send + Sync + IntoArcAny + std::fmt::Debug + 'static { + /// Handle an incoming connection. + /// + /// This runs on a freshly spawned tokio task so this can be long-running. + fn accept(self: Arc, conn: Connecting) -> BoxedFuture>; + + /// Called when the node shuts down. + fn shutdown(self: Arc) -> BoxedFuture<()> { + Box::pin(async move {}) + } +} + +/// Helper trait to facilite casting from `Arc` to `Arc`. +/// +/// This trait has a blanket implementation so there is no need to implement this yourself. +pub trait IntoArcAny { + fn into_arc_any(self: Arc) -> Arc; +} + +impl IntoArcAny for T { + fn into_arc_any(self: Arc) -> Arc { + self + } +} + +/// A typed map of protocol handlers, mapping them from ALPNs. +#[derive(Debug, Clone, Default)] +pub struct ProtocolMap(BTreeMap, Arc>); + +impl ProtocolMap { + /// Returns the registered protocol handler for an ALPN as a concrete type. + pub fn get_typed(&self, alpn: &[u8]) -> Option> { + let protocol: Arc = self.0.get(alpn)?.clone(); + let protocol_any: Arc = protocol.into_arc_any(); + let protocol_ref = Arc::downcast(protocol_any).ok()?; + Some(protocol_ref) + } + + /// Returns the registered protocol handler for an ALPN as a [`Arc`]. + pub fn get(&self, alpn: &[u8]) -> Option> { + self.0.get(alpn).cloned() + } + + /// Inserts a protocol handler. + pub fn insert(&mut self, alpn: Vec, handler: Arc) { + self.0.insert(alpn, handler); + } + + /// Returns an iterator of all registered ALPN protocol identifiers. + pub fn alpns(&self) -> impl Iterator> { + self.0.keys() + } + + /// Shuts down all protocol handlers. + /// + /// Calls and awaits [`ProtocolHandler::shutdown`] for all registered handlers concurrently. + pub async fn shutdown(&self) { + let handlers = self.0.values().cloned().map(ProtocolHandler::shutdown); + join_all(handlers).await; + } +} diff --git a/iroh-router/src/router.rs b/iroh-router/src/router.rs new file mode 100644 index 00000000000..bf4d290623f --- /dev/null +++ b/iroh-router/src/router.rs @@ -0,0 +1,211 @@ +use std::sync::Arc; + +use anyhow::{anyhow, Result}; +use futures_util::{ + future::{MapErr, Shared}, + FutureExt, TryFutureExt, +}; +use iroh_net::Endpoint; +use tokio::task::{JoinError, JoinSet}; +use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; +use tracing::{debug, error, warn}; + +use crate::{ProtocolHandler, ProtocolMap}; + +#[derive(Clone, Debug)] +pub struct Router { + endpoint: Endpoint, + protocols: Arc, + // `Router` needs to be `Clone + Send`, and we need to `task.await` in its `shutdown()` impl. + // So we need + // - `Shared` so we can `task.await` from all `Node` clones + // - `MapErr` to map the `JoinError` to a `String`, because `JoinError` is `!Clone` + // - `AbortOnDropHandle` to make sure that the `task` is cancelled when all `Node`s are dropped + // (`Shared` acts like an `Arc` around its inner future). + task: Shared, JoinErrToStr>>, + cancel_token: CancellationToken, +} + +type JoinErrToStr = Box String + Send + Sync + 'static>; + +impl Router { + pub fn builder(endpoint: Endpoint) -> RouterBuilder { + RouterBuilder::new(endpoint) + } + + /// Returns a protocol handler for an ALPN. + /// + /// This downcasts to the concrete type and returns `None` if the handler registered for `alpn` + /// does not match the passed type. + pub fn get_protocol(&self, alpn: &[u8]) -> Option> { + self.protocols.get_typed(alpn) + } + + pub fn endpoint(&self) -> &Endpoint { + &self.endpoint + } + + pub async fn shutdown(self) -> Result<()> { + // Trigger shutdown of the main run task by activating the cancel token. + self.cancel_token.cancel(); + + // Wait for the main task to terminate. + self.task.await.map_err(|err| anyhow!(err))?; + + Ok(()) + } +} + +#[derive(Debug)] +pub struct RouterBuilder { + endpoint: Endpoint, + protocols: ProtocolMap, +} + +impl RouterBuilder { + pub fn new(endpoint: Endpoint) -> Self { + Self { + endpoint, + protocols: ProtocolMap::default(), + } + } + + pub fn accept(mut self, alpn: Vec, handler: Arc) -> Self { + self.protocols.insert(alpn, handler); + self + } + + /// Returns the [`Endpoint`] of the node. + pub fn endpoint(&self) -> &Endpoint { + &self.endpoint + } + + /// Returns a protocol handler for an ALPN. + /// + /// This downcasts to the concrete type and returns `None` if the handler registered for `alpn` + /// does not match the passed type. + pub fn get_protocol(&self, alpn: &[u8]) -> Option> { + self.protocols.get_typed(alpn) + } + + pub async fn spawn(self) -> Result { + // Update the endpoint with our alpns. + let alpns = self + .protocols + .alpns() + .map(|alpn| alpn.to_vec()) + .collect::>(); + + let protocols = Arc::new(self.protocols); + if let Err(err) = self.endpoint.set_alpns(alpns) { + shutdown(&self.endpoint, protocols.clone()).await; + return Err(err); + } + + let mut join_set = JoinSet::new(); + let endpoint = self.endpoint.clone(); + let protos = protocols.clone(); + let cancel = CancellationToken::new(); + let cancel_token = cancel.clone(); + + let run_loop_fut = async move { + let protocols = protos; + loop { + tokio::select! { + biased; + _ = cancel_token.cancelled() => { + break; + }, + // handle incoming p2p connections. + Some(incoming) = endpoint.accept() => { + let protocols = protocols.clone(); + join_set.spawn(async move { + handle_connection(incoming, protocols).await; + anyhow::Ok(()) + }); + }, + // handle task terminations and quit on panics. + res = join_set.join_next(), if !join_set.is_empty() => { + match res { + Some(Err(outer)) => { + if outer.is_panic() { + error!("Task panicked: {outer:?}"); + break; + } else if outer.is_cancelled() { + debug!("Task cancelled: {outer:?}"); + } else { + error!("Task failed: {outer:?}"); + break; + } + } + Some(Ok(Err(inner))) => { + debug!("Task errored: {inner:?}"); + } + _ => {} + } + }, + else => break, + } + } + + shutdown(&endpoint, protocols).await; + + // Abort remaining tasks. + tracing::info!("Shutting down remaining tasks"); + join_set.shutdown().await; + }; + let task = tokio::task::spawn(run_loop_fut); + let task = AbortOnDropHandle::new(task) + .map_err(Box::new(|e: JoinError| e.to_string()) as JoinErrToStr) + .shared(); + + Ok(Router { + endpoint: self.endpoint, + protocols, + task, + cancel_token: cancel, + }) + } +} + +/// Shutdown the different parts of the router concurrently. +async fn shutdown(endpoint: &Endpoint, protocols: Arc) { + let error_code = 1u16; + + // We ignore all errors during shutdown. + let _ = tokio::join!( + // Close the endpoint. + // Closing the Endpoint is the equivalent of calling Connection::close on all + // connections: Operations will immediately fail with ConnectionError::LocallyClosed. + // All streams are interrupted, this is not graceful. + endpoint + .clone() + .close(error_code.into(), b"provider terminating"), + // Shutdown protocol handlers. + protocols.shutdown(), + ); +} + +async fn handle_connection(incoming: iroh_net::endpoint::Incoming, protocols: Arc) { + let mut connecting = match incoming.accept() { + Ok(conn) => conn, + Err(err) => { + warn!("Ignoring connection: accepting failed: {err:#}"); + return; + } + }; + let alpn = match connecting.alpn().await { + Ok(alpn) => alpn, + Err(err) => { + warn!("Ignoring connection: invalid handshake: {err:#}"); + return; + } + }; + let Some(handler) = protocols.get(&alpn) else { + warn!("Ignoring connection: unsupported ALPN protocol"); + return; + }; + if let Err(err) = handler.accept(connecting).await { + warn!("Handling incoming connection ended with error: {err}"); + } +} diff --git a/iroh-test/Cargo.toml b/iroh-test/Cargo.toml index e35c1c2071e..7bbddc52180 100644 --- a/iroh-test/Cargo.toml +++ b/iroh-test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-test" -version = "0.27.0" +version = "0.28.0" edition = "2021" readme = "README.md" description = "Internal utilities to support testing of iroh." diff --git a/iroh-willow/Cargo.toml b/iroh-willow/Cargo.toml index eaede55d8d8..b4d57a52461 100644 --- a/iroh-willow/Cargo.toml +++ b/iroh-willow/Cargo.toml @@ -9,7 +9,7 @@ authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" # Sadly this also needs to be updated in .github/workflows/ci.yml -rust-version = "1.75" +rust-version = "1.76" [lints] workspace = true @@ -27,19 +27,24 @@ futures-lite = "2.3.0" futures-util = "0.3.30" genawaiter = "0.99.1" hex = "0.4.3" -iroh-base = { version = "0.27.0", path = "../iroh-base" } +iroh-base = { version = "0.28.0", path = "../iroh-base" } iroh-blake3 = "1.4.5" -iroh-blobs = { version = "0.27.0", path = "../iroh-blobs" } +iroh-blobs = { version = "0.28.0" } iroh-io = { version = "0.6.0", features = ["stats"] } -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics", optional = true } -iroh-net = { version = "0.27.0", path = "../iroh-net" } +iroh-metrics = { version = "0.28.0", path = "../iroh-metrics", optional = true } +iroh-net = { version = "0.28.0", path = "../iroh-net" } meadowcap = "0.1.0" +nested_enum_utils = "0.1.0" postcard = { version = "1", default-features = false, features = [ "alloc", "use-std", "experimental-derive", ] } +quic-rpc = "0.15.0" +quic-rpc-derive = "0.15.0" rand = "0.8.5" rand_core = "0.6.4" redb = { version = "2.0.0" } +ref-cast = "1.0.23" self_cell = "1.0.4" serde = { version = "1.0.164", features = ["derive"] } +serde-error = "0.1.3" sha2 = "0.10.8" strum = { version = "0.26", features = ["derive"] } syncify = "0.1.0" @@ -62,6 +67,7 @@ rand_chacha = "0.3.1" tokio = { version = "1", features = ["sync", "macros"] } proptest = "1.2.0" tempfile = "3.4" +testresult = "0.4.0" test-strategy = "0.3.1" tracing-subscriber = "0.3.18" diff --git a/iroh-willow/examples/bench.rs b/iroh-willow/examples/bench.rs index 6259c46c2d8..3f0bf7f51a8 100644 --- a/iroh-willow/examples/bench.rs +++ b/iroh-willow/examples/bench.rs @@ -99,11 +99,6 @@ mod util { use bytes::Bytes; use futures_concurrency::future::TryJoin; use iroh_net::{Endpoint, NodeId}; - use rand::SeedableRng; - use rand_chacha::ChaCha12Rng; - use rand_core::CryptoRngCore; - use tokio::task::JoinHandle; - use iroh_willow::{ engine::{AcceptOpts, Engine}, form::EntryForm, @@ -115,6 +110,10 @@ mod util { }, ALPN, }; + use rand::SeedableRng; + use rand_chacha::ChaCha12Rng; + use rand_core::CryptoRngCore; + use tokio::task::JoinHandle; pub fn create_rng(seed: &str) -> ChaCha12Rng { let seed = iroh_base::hash::Hash::new(seed); @@ -135,7 +134,7 @@ mod util { ) -> Result { let endpoint = Endpoint::builder() .secret_key(secret_key) - .relay_mode(iroh_net::relay::RelayMode::Disabled) + .relay_mode(iroh_net::RelayMode::Disabled) .alpns(vec![ALPN.to_vec()]) .bind() .await?; diff --git a/iroh-willow/src/engine.rs b/iroh-willow/src/engine.rs index 816f95546cc..4792cd8ca1b 100644 --- a/iroh-willow/src/engine.rs +++ b/iroh-willow/src/engine.rs @@ -25,9 +25,7 @@ mod actor; mod peer_manager; use self::peer_manager::PeerManager; - -pub use self::actor::ActorHandle; -pub use self::peer_manager::AcceptOpts; +pub use self::{actor::ActorHandle, peer_manager::AcceptOpts}; const PEER_MANAGER_INBOX_CAP: usize = 128; diff --git a/iroh-willow/src/engine/peer_manager.rs b/iroh-willow/src/engine/peer_manager.rs index 50cac9860a1..b24a8300faf 100644 --- a/iroh-willow/src/engine/peer_manager.rs +++ b/iroh-willow/src/engine/peer_manager.rs @@ -2,7 +2,6 @@ use std::{collections::HashMap, future::Future, sync::Arc, time::Duration}; use anyhow::{Context, Result}; use futures_buffered::join_all; - use futures_lite::{future::Boxed, StreamExt}; use futures_util::{FutureExt, TryFutureExt}; use iroh_net::{ @@ -14,10 +13,10 @@ use tokio::{ task::{AbortHandle, JoinSet}, }; use tokio_stream::{wrappers::ReceiverStream, StreamMap}; - use tokio_util::{either::Either, sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{debug, error_span, instrument, trace, warn, Instrument, Span}; +use super::actor::ActorHandle; use crate::{ interest::Interests, net::{ @@ -31,8 +30,6 @@ use crate::{ }, }; -use super::actor::ActorHandle; - /// Timeout at shutdown after which we abort connections that failed to terminate gracefully. const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); @@ -406,6 +403,7 @@ impl PeerManager { peer_info.conn_state = ConnState::None; match &peer_info.session_state { SessionState::None => { + println!("Error: {err:#}"); peer_info .abort_pending_intents(err.context("failed while establishing")) .await; diff --git a/iroh-willow/src/interest.rs b/iroh-willow/src/interest.rs index d6090433bd4..6c481ed0ba0 100644 --- a/iroh-willow/src/interest.rs +++ b/iroh-willow/src/interest.rs @@ -125,10 +125,10 @@ pub enum AreaOfInterestSelector { mod serde_area_of_interest_set { // TODO: Less clones and allocs. - use crate::proto::grouping::serde_encoding::SerdeAreaOfInterest; use serde::Deserializer; use super::*; + use crate::proto::grouping::serde_encoding::SerdeAreaOfInterest; pub fn serialize( items: &HashSet, serializer: S, diff --git a/iroh-willow/src/lib.rs b/iroh-willow/src/lib.rs index 52408cbec4d..194de648c3e 100644 --- a/iroh-willow/src/lib.rs +++ b/iroh-willow/src/lib.rs @@ -8,6 +8,7 @@ pub mod form; pub mod interest; pub(crate) mod net; pub mod proto; +pub mod rpc; pub mod session; pub mod store; pub mod util; diff --git a/iroh-willow/src/net.rs b/iroh-willow/src/net.rs index 11bf554a42f..87433f79889 100644 --- a/iroh-willow/src/net.rs +++ b/iroh-willow/src/net.rs @@ -420,6 +420,7 @@ mod tests { use rand_chacha::ChaCha12Rng; use tracing::{info, Instrument}; + use super::{establish, prepare_channels}; use crate::{ engine::ActorHandle, form::{AuthForm, EntryForm, PayloadForm, SubspaceForm, TimestampForm}, @@ -435,8 +436,6 @@ mod tests { session::{intents::Intent, Role, SessionHandle, SessionInit, SessionMode}, }; - use super::{establish, prepare_channels}; - const ALPN: &[u8] = b"iroh-willow/0"; fn create_rng(seed: &str) -> ChaCha12Rng { @@ -783,7 +782,7 @@ mod tests { ) -> Result<(Endpoint, NodeId, NodeAddr)> { let ep = Endpoint::builder() .secret_key(SecretKey::generate_with_rng(rng)) - .relay_mode(iroh_net::relay::RelayMode::Disabled) + .relay_mode(iroh_net::RelayMode::Disabled) .alpns(vec![ALPN.to_vec()]) .bind() .await?; diff --git a/iroh-willow/src/proto/data_model.rs b/iroh-willow/src/proto/data_model.rs index 1f6ceddec7d..98e6f44cd05 100644 --- a/iroh-willow/src/proto/data_model.rs +++ b/iroh-willow/src/proto/data_model.rs @@ -2,6 +2,7 @@ use iroh_base::hash::Hash; use ufotofu::sync::{consumer::IntoVec, producer::FromSlice}; +pub use willow_data_model::{InvalidPathError, UnauthorisedWriteError}; use willow_encoding::sync::{Decodable, Encodable}; use super::{ @@ -9,9 +10,6 @@ use super::{ meadowcap::{self}, }; -pub use willow_data_model::InvalidPathError; -pub use willow_data_model::UnauthorisedWriteError; - /// A type for identifying namespaces. pub type NamespaceId = keys::NamespaceId; @@ -206,14 +204,12 @@ pub type AuthorisedEntry = willow_data_model::AuthorisedEntry< AuthorisationToken, >; -use syncify::syncify; -use syncify::syncify_replace; +use syncify::{syncify, syncify_replace}; #[syncify(encoding_sync)] mod encoding { #[syncify_replace(use ufotofu::sync::{BulkConsumer, BulkProducer};)] use ufotofu::local_nb::{BulkConsumer, BulkProducer}; - #[syncify_replace(use willow_encoding::sync::{Decodable, Encodable};)] use willow_encoding::{Decodable, Encodable}; @@ -249,9 +245,8 @@ mod encoding { pub mod serde_encoding { use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; - use crate::util::codec2::{from_bytes, to_vec}; - use super::*; + use crate::util::codec2::{from_bytes, to_vec}; pub mod path { @@ -299,10 +294,10 @@ pub mod serde_encoding { pub struct SerdeEntry(#[serde(with = "entry")] pub Entry); pub mod authorised_entry { - use crate::proto::meadowcap::serde_encoding::SerdeMcCapability; use keys::UserSignature; use super::*; + use crate::proto::meadowcap::serde_encoding::SerdeMcCapability; pub fn serialize( entry: &AuthorisedEntry, serializer: S, diff --git a/iroh-willow/src/proto/grouping.rs b/iroh-willow/src/proto/grouping.rs index 140acffa313..ee73e812285 100644 --- a/iroh-willow/src/proto/grouping.rs +++ b/iroh-willow/src/proto/grouping.rs @@ -129,9 +129,8 @@ impl Point { pub mod serde_encoding { use serde::{de, Deserialize, Deserializer, Serialize}; - use crate::util::codec2::{from_bytes_relative, to_vec_relative}; - use super::*; + use crate::util::codec2::{from_bytes_relative, to_vec_relative}; pub mod area { use super::*; diff --git a/iroh-willow/src/proto/keys.rs b/iroh-willow/src/proto/keys.rs index a8d303edaf9..562c61816bd 100644 --- a/iroh-willow/src/proto/keys.rs +++ b/iroh-willow/src/proto/keys.rs @@ -12,9 +12,7 @@ use ed25519_dalek::{SignatureError, Signer, SigningKey, Verifier, VerifyingKey}; use iroh_base::base32; use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; -use willow_store::FixedSize; -use willow_store::IsLowerBound; -use willow_store::LowerBound; +use willow_store::{FixedSize, IsLowerBound, LowerBound}; use super::meadowcap::IsCommunal; @@ -593,9 +591,8 @@ impl FromStr for NamespaceId { } mod willow_impls { - use crate::util::increment_by_one; - use super::*; + use crate::util::increment_by_one; impl willow_data_model::SubspaceId for UserId { fn successor(&self) -> Option { @@ -675,14 +672,12 @@ mod willow_impls { } } -use syncify::syncify; -use syncify::syncify_replace; +use syncify::{syncify, syncify_replace}; #[syncify(encoding_sync)] mod encoding { #[syncify_replace(use ufotofu::sync::{BulkConsumer, BulkProducer};)] use ufotofu::local_nb::{BulkConsumer, BulkProducer}; - use willow_encoding::DecodeError; #[syncify_replace(use willow_encoding::sync::{Encodable, Decodable};)] use willow_encoding::{Decodable, Encodable}; diff --git a/iroh-willow/src/proto/meadowcap.rs b/iroh-willow/src/proto/meadowcap.rs index ef0e5f119c8..ce1bead5ae3 100644 --- a/iroh-willow/src/proto/meadowcap.rs +++ b/iroh-willow/src/proto/meadowcap.rs @@ -17,10 +17,10 @@ pub type NamespaceId = keys::NamespaceId; pub type UserSignature = keys::UserSignature; pub type NamespaceSignature = keys::NamespaceSignature; -use super::data_model::{Entry, MAX_COMPONENT_COUNT, MAX_COMPONENT_LENGTH, MAX_PATH_LENGTH}; - pub use meadowcap::{AccessMode, IsCommunal}; +use super::data_model::{Entry, MAX_COMPONENT_COUNT, MAX_COMPONENT_LENGTH, MAX_PATH_LENGTH}; + #[derive(Debug, derive_more::From, Serialize, Deserialize)] pub enum SecretKey { User(keys::UserSecretKey), @@ -137,13 +137,12 @@ pub fn is_wider_than(a: &McCapability, b: &McCapability) -> bool { pub mod serde_encoding { use serde::{de, Deserialize, Deserializer}; + use super::*; use crate::{ proto::grouping::Area, util::codec2::{from_bytes, from_bytes_relative, to_vec, to_vec_relative}, }; - use super::*; - pub mod read_authorisation { use super::*; pub fn serialize( diff --git a/iroh-willow/src/proto/pai.rs b/iroh-willow/src/proto/pai.rs index 7d7ad15a0c1..6fa524461e1 100644 --- a/iroh-willow/src/proto/pai.rs +++ b/iroh-willow/src/proto/pai.rs @@ -155,14 +155,12 @@ impl FragmentKit { } } -use syncify::syncify; -use syncify::syncify_replace; +use syncify::{syncify, syncify_replace}; #[syncify(encoding_sync)] mod encoding { #[syncify_replace(use ufotofu::sync::BulkConsumer;)] use ufotofu::local_nb::BulkConsumer; - #[syncify_replace(use willow_encoding::sync::Encodable;)] use willow_encoding::Encodable; diff --git a/iroh-willow/src/proto/wgps/messages.rs b/iroh-willow/src/proto/wgps/messages.rs index b189f8c416b..c3163711b05 100644 --- a/iroh-willow/src/proto/wgps/messages.rs +++ b/iroh-willow/src/proto/wgps/messages.rs @@ -2,10 +2,16 @@ use std::io::Write; use serde::{Deserialize, Serialize}; +use super::{ + channels::LogicalChannel, + fingerprint::Fingerprint, + handles::{ + AreaOfInterestHandle, CapabilityHandle, HandleType, IntersectionHandle, StaticTokenHandle, + }, +}; use crate::{ proto::{ - data_model::serde_encoding::SerdeEntry, - data_model::Entry, + data_model::{serde_encoding::SerdeEntry, Entry}, grouping::{ serde_encoding::{SerdeAreaOfInterest, SerdeRange3d}, Area, @@ -16,14 +22,6 @@ use crate::{ util::codec::{DecodeOutcome, Decoder, Encoder}, }; -use super::{ - channels::LogicalChannel, - fingerprint::Fingerprint, - handles::{ - AreaOfInterestHandle, CapabilityHandle, HandleType, IntersectionHandle, StaticTokenHandle, - }, -}; - pub type StaticToken = meadowcap::serde_encoding::SerdeMcCapability; // pub type ValidatedStaticToken = meadowcap::ValidatedCapability; pub type DynamicToken = meadowcap::UserSignature; diff --git a/iroh-willow/src/rpc.rs b/iroh-willow/src/rpc.rs new file mode 100644 index 00000000000..f4408bd9092 --- /dev/null +++ b/iroh-willow/src/rpc.rs @@ -0,0 +1,6 @@ +pub mod client; +pub mod handler; +pub mod proto; + +type RpcClient> = + quic_rpc::RpcClient; diff --git a/iroh/src/client/spaces.rs b/iroh-willow/src/rpc/client.rs similarity index 86% rename from iroh/src/client/spaces.rs rename to iroh-willow/src/rpc/client.rs index e585cfa7fe2..bf13ade264f 100644 --- a/iroh/src/client/spaces.rs +++ b/iroh-willow/src/rpc/client.rs @@ -11,7 +11,6 @@ use std::{ collections::HashMap, - path::PathBuf, pin::Pin, task::{ready, Context, Poll}, }; @@ -23,7 +22,13 @@ use futures_util::{Sink, SinkExt}; use iroh_base::key::NodeId; use iroh_blobs::Hash; use iroh_net::NodeAddr; -use iroh_willow::{ +use quic_rpc::transport::ConnectionErrors; +use ref_cast::RefCast; +use serde::{Deserialize, Serialize}; +use tokio_stream::{StreamMap, StreamNotifyClose}; + +use super::RpcClient; +use crate::{ form::{AuthForm, SubspaceForm, TimestampForm}, interest::{ AreaOfInterestSelector, CapSelector, CapabilityPack, DelegateTo, Interests, RestrictArea, @@ -34,33 +39,34 @@ use iroh_willow::{ keys::{NamespaceId, NamespaceKind, UserId}, meadowcap::{AccessMode, SecretKey}, }, + rpc::proto::*, session::{ intents::{serde_encoding::Event, Completion, IntentUpdate}, SessionInit, SessionMode, }, store::traits::{StoreEvent, SubscribeParams}, }; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; -use tokio::io::AsyncRead; -use tokio_stream::{StreamMap, StreamNotifyClose}; - -use crate::client::RpcClient; -use crate::rpc_protocol::spaces::*; /// Iroh Willow client. #[derive(Debug, Clone, RefCast)] #[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, +pub struct Client = quic_rpc::client::BoxedConnector> +where + C: ConnectionErrors, +{ + pub(super) rpc: RpcClient, } -impl Client { - fn net(&self) -> &super::net::Client { - super::net::Client::ref_cast(&self.rpc) +impl> Client +where + C: ConnectionErrors, +{ + pub fn new(rpc: RpcClient) -> Self { + Self { rpc } } + /// Create a new namespace in the Willow store. - pub async fn create(&self, kind: NamespaceKind, owner: UserId) -> Result { + pub async fn create(&self, kind: NamespaceKind, owner: UserId) -> Result> { let req = CreateNamespaceRequest { kind, owner }; let res = self.rpc.rpc(req).await??; Ok(Space::new(self.rpc.clone(), res.0)) @@ -103,7 +109,7 @@ impl Client { &self, ticket: SpaceTicket, mode: SessionMode, - ) -> Result<(Space, SyncHandleSet)> { + ) -> Result<(Space, SyncHandleSet)> { if ticket.caps.is_empty() { anyhow::bail!("Invalid ticket: Does not include any capabilities"); } @@ -119,7 +125,7 @@ impl Client { let mut intents = SyncHandleSet::default(); for addr in ticket.nodes { let node_id = addr.node_id; - self.net().add_node_addr(addr).await?; + self.add_node_addr(addr).await?; let intent = self.sync_with_peer(node_id, init.clone()).await?; intents.insert(node_id, intent)?; } @@ -160,32 +166,46 @@ impl Client { self.rpc.rpc(req).await??; Ok(()) } + + /// Fetches the [`NodeAddr`] for this node. + /// + /// See also [`Endpoint::node_addr`](iroh_net::Endpoint::node_addr). + pub async fn node_addr(&self) -> Result { + let addr = self.rpc.rpc(AddrRequest).await??; + Ok(addr) + } + + /// Adds a known node address to this node. + /// + /// See also [`Endpoint::add_node_addr`](iroh_net::Endpoint::add_node_addr). + pub async fn add_node_addr(&self, addr: NodeAddr) -> Result<()> { + self.rpc.rpc(AddAddrRequest { addr }).await??; + Ok(()) + } } /// A space to store entries in. #[derive(Debug, Clone)] -pub struct Space { - rpc: RpcClient, +pub struct Space = quic_rpc::client::BoxedConnector> +where + C: ConnectionErrors, +{ + rpc: RpcClient, namespace_id: NamespaceId, } -impl Space { - fn new(rpc: RpcClient, namespace_id: NamespaceId) -> Self { +impl> Space +where + C: ConnectionErrors, +{ + fn new(rpc: RpcClient, namespace_id: NamespaceId) -> Self { Self { rpc, namespace_id } } - fn blobs(&self) -> &super::blobs::Client { - super::blobs::Client::ref_cast(&self.rpc) - } - - fn spaces(&self) -> &Client { + fn spaces(&self) -> &Client { Client::ref_cast(&self.rpc) } - fn net(&self) -> &super::net::Client { - super::net::Client::ref_cast(&self.rpc) - } - /// Returns the identifier for this space. pub fn namespace_id(&self) -> NamespaceId { self.namespace_id @@ -216,46 +236,49 @@ impl Space { /// Inserts a new entry, with the payload imported from a byte string. pub async fn insert_bytes( &self, + blobs: &impl iroh_blobs::store::Store, entry: EntryForm, payload: impl Into, ) -> Result { - let batch = self.blobs().batch().await?; - let tag = batch.add_bytes(payload).await?; + let tag = blobs + .import_bytes(payload.into(), iroh_blobs::BlobFormat::Raw) + .await?; self.insert_hash(entry, *tag.hash()).await } - /// Inserts a new entry, with the payload imported from a byte reader. - pub async fn insert_reader( - &self, - entry: EntryForm, - payload: impl AsyncRead + Send + Unpin + 'static, - ) -> Result { - let batch = self.blobs().batch().await?; - let tag = batch.add_reader(payload).await?; - self.insert_hash(entry, *tag.hash()).await - } + // TODO(matheus23): figure out how to use blobs + // /// Inserts a new entry, with the payload imported from a byte reader. + // pub async fn insert_reader( + // &self, + // entry: EntryForm, + // payload: impl AsyncRead + Send + Unpin + 'static, + // ) -> Result { + // let batch = self.blobs().batch().await?; + // let tag = batch.add_reader(payload).await?; + // self.insert_hash(entry, *tag.hash()).await + // } - /// Inserts a new entry, with the payload imported from a byte stream. - pub async fn insert_stream( - &self, - entry: EntryForm, - payload: impl Stream> + Send + Unpin + 'static, - ) -> Result { - let batch = self.blobs().batch().await?; - let tag = batch.add_stream(payload).await?; - self.insert_hash(entry, *tag.hash()).await - } + // /// Inserts a new entry, with the payload imported from a byte stream. + // pub async fn insert_stream( + // &self, + // entry: EntryForm, + // payload: impl Stream> + Send + Unpin + 'static, + // ) -> Result { + // let batch = self.blobs().batch().await?; + // let tag = batch.add_stream(payload).await?; + // self.insert_hash(entry, *tag.hash()).await + // } - /// Inserts a new entry, with the payload imported from a file. - pub async fn insert_from_file( - &self, - entry: EntryForm, - file_path: PathBuf, - ) -> Result { - let batch = self.blobs().batch().await?; - let (tag, _len) = batch.add_file(file_path).await?; - self.insert_hash(entry, *tag.hash()).await - } + // /// Inserts a new entry, with the payload imported from a file. + // pub async fn insert_from_file( + // &self, + // entry: EntryForm, + // file_path: PathBuf, + // ) -> Result { + // let batch = self.blobs().batch().await?; + // let (tag, _len) = batch.add_file(file_path).await?; + // self.insert_hash(entry, *tag.hash()).await + // } /// Ingest an authorised entry. // TODO: Not sure if we should expose this on the client at all. @@ -290,7 +313,7 @@ impl Space { range, }; let stream = self.rpc.try_server_streaming(req).await?; - Ok(stream.map(|res| res.map(|r| r.0).map_err(Into::into))) + Ok(stream.map(|res| res.map(|r| r.0).map_err(anyhow::Error::from))) } /// Syncs with a peer and quit the session after a single reconciliation of the selected areas. @@ -351,7 +374,7 @@ impl Space { DelegateTo::new(receiver, restrict_area), ) .await?; - let node_addr = self.net().node_addr().await?; + let node_addr = self.spaces().node_addr().await?; Ok(SpaceTicket { caps, nodes: vec![node_addr], @@ -409,7 +432,6 @@ pub struct SpaceTicket { /// otherwise the session will be blocked from progressing. /// /// The `SyncHandle` can also submit new interests into the session. -/// // This version of SyncHandle differs from the one in iroh-willow intents module // by using the Event type instead of EventKind, which serializes the error to a string // to cross the RPC boundary. Maybe look into making the main iroh_willow Error type diff --git a/iroh/src/node/rpc/spaces.rs b/iroh-willow/src/rpc/handler.rs similarity index 71% rename from iroh/src/node/rpc/spaces.rs rename to iroh-willow/src/rpc/handler.rs index 3f8de9d8adc..3edd95a7e7c 100644 --- a/iroh/src/node/rpc/spaces.rs +++ b/iroh-willow/src/rpc/handler.rs @@ -1,36 +1,29 @@ use anyhow::Result; use futures_lite::Stream; -use futures_util::SinkExt; -use futures_util::StreamExt; -use iroh_base::rpc::{RpcError, RpcResult}; -use iroh_blobs::store::Store; -use iroh_willow::form::EntryOrForm; -use quic_rpc::server::{RpcChannel, RpcServerError}; +use futures_util::{SinkExt, StreamExt}; +use iroh_net::Endpoint; +use quic_rpc::server::{ChannelTypes, RpcChannel, RpcServerError}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; -use crate::node::IrohServerEndpoint; -use crate::rpc_protocol::spaces::*; -use crate::rpc_protocol::RpcService; - -use super::Handler; +use crate::{form::EntryOrForm, rpc::proto::*, Engine}; fn map_err(err: anyhow::Error) -> RpcError { - RpcError::from(err) + RpcError::new(&*err) } -impl Handler { - pub(crate) async fn handle_spaces_request( +impl Engine { + pub async fn handle_spaces_request>( self, + endpoint: Endpoint, msg: Request, - chan: RpcChannel, - ) -> Result<(), RpcServerError> { + chan: RpcChannel, + ) -> Result<(), RpcServerError> { use Request::*; match msg { IngestEntry(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .ingest_entry(req.authorised_entry) .await .map(|inserted| { @@ -45,10 +38,9 @@ impl Handler { .await } InsertEntry(msg) => { - chan.rpc(msg, self, |handler, req| async move { + chan.rpc(msg, self, |engine, req| async move { let entry = EntryOrForm::Form(req.entry.into()); - handler - .spaces()? + engine .insert_entry(entry, req.auth) .await .map(|(entry, inserted)| { @@ -63,9 +55,8 @@ impl Handler { .await } InsertSecret(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .insert_secret(req.secret) .await .map(|_| InsertSecretResponse) @@ -74,9 +65,8 @@ impl Handler { .await } GetEntries(msg) => { - chan.try_server_streaming(msg, self, |handler, req| async move { - let stream = handler - .spaces()? + chan.try_server_streaming(msg, self, |engine, req| async move { + let stream = engine .get_entries(req.namespace, req.range) .await .map_err(map_err)?; @@ -85,9 +75,8 @@ impl Handler { .await } GetEntry(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .get_entry(req.namespace, req.subspace, req.path) .await .map(|entry| GetEntryResponse(entry.map(Into::into))) @@ -96,9 +85,8 @@ impl Handler { .await } CreateNamespace(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .create_namespace(req.kind, req.owner) .await .map(CreateNamespaceResponse) @@ -107,9 +95,8 @@ impl Handler { .await } CreateUser(msg) => { - chan.rpc(msg, self, |handler, _| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, _| async move { + engine .create_user() .await .map(CreateUserResponse) @@ -118,9 +105,8 @@ impl Handler { .await } DelegateCaps(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .delegate_caps(req.from, req.access_mode, req.to) .await .map(DelegateCapsResponse) @@ -129,9 +115,8 @@ impl Handler { .await } ImportCaps(msg) => { - chan.rpc(msg, self, |handler, req| async move { - handler - .spaces()? + chan.rpc(msg, self, |engine, req| async move { + engine .import_caps(req.caps) .await .map(|_| ImportCapsResponse) @@ -140,14 +125,14 @@ impl Handler { .await } SyncWithPeer(msg) => { - chan.bidi_streaming(msg, self, |handler, req, update_stream| { + chan.bidi_streaming(msg, self, |engine, req, update_stream| { // TODO: refactor to use less tasks let (events_tx, events_rx) = tokio::sync::mpsc::channel(32); tokio::task::spawn(async move { if let Err(err) = - sync_with_peer(handler, req, events_tx.clone(), update_stream).await + sync_with_peer(&engine, req, events_tx.clone(), update_stream).await { - let _ = events_tx.send(Err(err.into())).await; + let _ = events_tx.send(Err(RpcError::new(&*err))).await; } }); ReceiverStream::new(events_rx) @@ -156,11 +141,10 @@ impl Handler { } SyncWithPeerUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), Subscribe(msg) => { - chan.try_server_streaming(msg, self, |handler, req| async move { + chan.try_server_streaming(msg, self, |engine, req| async move { let (tx, rx) = mpsc::channel(1024); if let Some(progress_id) = req.initial_progress_id { - handler - .spaces()? + engine .resume_subscription( progress_id, req.namespace, @@ -171,8 +155,7 @@ impl Handler { .await .map_err(map_err)?; } else { - handler - .spaces()? + engine .subscribe_area(req.namespace, req.area, req.params, tx) .await .map_err(map_err)?; @@ -181,18 +164,31 @@ impl Handler { }) .await } + Addr(msg) => { + chan.rpc(msg, endpoint, |endpoint, _req| async move { + let addr = endpoint.node_addr().await.map_err(map_err)?; + Ok(addr) + }) + .await + } + AddAddr(msg) => { + chan.rpc(msg, endpoint, |endpoint, req| async move { + endpoint.add_node_addr(req.addr).map_err(map_err)?; + Ok(()) + }) + .await + } } } } // TODO: Try to use the streams directly instead of spawning two tasks. -async fn sync_with_peer( - handler: Handler, +async fn sync_with_peer( + engine: &Engine, req: SyncWithPeerRequest, events_tx: mpsc::Sender>, mut update_stream: impl Stream + Unpin + Send + 'static, ) -> anyhow::Result<()> { - let engine = handler.spaces()?; let handle = engine .sync_with_peer(req.peer, req.init) .await diff --git a/iroh/src/rpc_protocol/spaces.rs b/iroh-willow/src/rpc/proto.rs similarity index 87% rename from iroh/src/rpc_protocol/spaces.rs rename to iroh-willow/src/rpc/proto.rs index 22aaffa5a29..b2caa05bcf0 100644 --- a/iroh/src/rpc_protocol/spaces.rs +++ b/iroh-willow/src/rpc/proto.rs @@ -1,7 +1,11 @@ -use iroh_base::rpc::{RpcError, RpcResult}; use iroh_blobs::Hash; -use iroh_net::NodeId; -use iroh_willow::{ +use iroh_net::{NodeAddr, NodeId}; +use nested_enum_utils::enum_conversions; +use quic_rpc::pattern::try_server_streaming::StreamCreated; +use quic_rpc_derive::rpc_requests; +use serde::{Deserialize, Serialize}; + +use crate::{ form::{AuthForm, SubspaceForm, TimestampForm}, interest::{CapSelector, CapabilityPack, DelegateTo}, proto::{ @@ -19,15 +23,22 @@ use iroh_willow::{ }, store::traits::{StoreEvent, SubscribeParams}, }; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; -use super::RpcService; +/// The RPC service type for the spaces protocol. +#[derive(Debug, Clone)] +pub struct RpcService; + +impl quic_rpc::Service for RpcService { + type Req = Request; + type Res = Response; +} + +pub type RpcError = serde_error::Error; +pub type RpcResult = std::result::Result; #[allow(missing_docs)] #[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] +#[enum_conversions] #[rpc_requests(RpcService)] pub enum Request { #[rpc(response = RpcResult)] @@ -53,11 +64,16 @@ pub enum Request { SyncWithPeerUpdate(SyncWithPeerUpdate), #[try_server_streaming(create_error = RpcError, item_error = RpcError, item = StoreEvent)] Subscribe(SubscribeRequest), + // requests for endpoint info + #[rpc(response = RpcResult)] + Addr(AddrRequest), + #[rpc(response = RpcResult<()>)] + AddAddr(AddAddrRequest), } #[allow(missing_docs)] #[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] +#[enum_conversions] pub enum Response { IngestEntry(RpcResult), InsertEntry(RpcResult), @@ -70,6 +86,10 @@ pub enum Response { ImportCaps(RpcResult), SyncWithPeer(RpcResult), Subscribe(RpcResult), + StreamCreated(RpcResult), + // responses for endpoint info + Addr(RpcResult), + AddAddr(RpcResult<()>), } #[derive(Debug, Serialize, Deserialize)] @@ -207,7 +227,7 @@ pub enum EntryOrForm { Form(FullEntryForm), } -impl From for iroh_willow::form::EntryOrForm { +impl From for crate::form::EntryOrForm { fn from(value: EntryOrForm) -> Self { match value { EntryOrForm::Entry(entry) => Self::Entry(entry), @@ -227,7 +247,7 @@ pub struct FullEntryForm { pub payload: PayloadForm, } -impl From for iroh_willow::form::EntryForm { +impl From for crate::form::EntryForm { fn from(value: FullEntryForm) -> Self { Self { namespace_id: value.namespace_id, @@ -248,7 +268,7 @@ pub enum PayloadForm { Unchecked(Hash, u64), } -impl From for iroh_willow::form::PayloadForm { +impl From for crate::form::PayloadForm { fn from(value: PayloadForm) -> Self { match value { PayloadForm::Checked(hash) => Self::Hash(hash), @@ -256,3 +276,11 @@ impl From for iroh_willow::form::PayloadForm { } } } + +#[derive(Serialize, Deserialize, Debug)] +pub struct AddrRequest; + +#[derive(Serialize, Deserialize, Debug)] +pub struct AddAddrRequest { + pub addr: NodeAddr, +} diff --git a/iroh-willow/src/session.rs b/iroh-willow/src/session.rs index d12eae4f9a3..a036a1e147e 100644 --- a/iroh-willow/src/session.rs +++ b/iroh-willow/src/session.rs @@ -32,10 +32,9 @@ mod resource; mod run; mod static_tokens; -pub(crate) use self::challenge::InitialTransmission; -pub(crate) use self::channels::Channels; -pub(crate) use self::error::Error; -pub(crate) use self::run::run_session; +pub(crate) use self::{ + challenge::InitialTransmission, channels::Channels, error::Error, run::run_session, +}; /// Id per session to identify store subscriptions. pub(crate) type SessionId = u64; diff --git a/iroh-willow/src/session/channels.rs b/iroh-willow/src/session/channels.rs index 1b283d2f214..46de9284073 100644 --- a/iroh-willow/src/session/channels.rs +++ b/iroh-willow/src/session/channels.rs @@ -7,6 +7,7 @@ use std::{ use futures_lite::Stream; use tracing::trace; +use super::Error; use crate::{ proto::wgps::{ Channel, DataMessage, IntersectionMessage, LogicalChannel, Message, ReconciliationMessage, @@ -15,8 +16,6 @@ use crate::{ util::channel::{Receiver, Sender, WriteError}, }; -use super::Error; - #[derive(Debug)] pub struct MessageReceiver { inner: Receiver, diff --git a/iroh-willow/src/session/data.rs b/iroh-willow/src/session/data.rs index b58b2dfece6..c1b6485258b 100644 --- a/iroh-willow/src/session/data.rs +++ b/iroh-willow/src/session/data.rs @@ -1,5 +1,9 @@ use futures_lite::StreamExt; +use super::{ + aoi_finder::AoiIntersection, + payload::{send_payload_chunked, CurrentPayload}, +}; use crate::{ proto::{ data_model::AuthorisedEntry, @@ -13,11 +17,6 @@ use crate::{ util::stream::CancelableReceiver, }; -use super::{ - aoi_finder::AoiIntersection, - payload::{send_payload_chunked, CurrentPayload}, -}; - #[derive(Debug)] pub enum Input { AoiIntersection(AoiIntersection), diff --git a/iroh-willow/src/session/intents.rs b/iroh-willow/src/session/intents.rs index 5095da03afe..1787aaa6fe3 100644 --- a/iroh-willow/src/session/intents.rs +++ b/iroh-willow/src/session/intents.rs @@ -673,9 +673,13 @@ fn flatten_interests(interests: &InterestMap) -> NamespaceInterests { pub mod serde_encoding { use serde::{Deserialize, Serialize}; - use crate::proto::grouping::serde_encoding::{SerdeArea, SerdeAreaOfInterest}; - use crate::proto::keys::NamespaceId; - use crate::session::intents::EventKind; + use crate::{ + proto::{ + grouping::serde_encoding::{SerdeArea, SerdeAreaOfInterest}, + keys::NamespaceId, + }, + session::intents::EventKind, + }; /// Serializable version of EventKind #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/iroh-willow/src/session/pai_finder.rs b/iroh-willow/src/session/pai_finder.rs index a06be6ac01d..b925ddfbc30 100644 --- a/iroh-willow/src/session/pai_finder.rs +++ b/iroh-willow/src/session/pai_finder.rs @@ -9,13 +9,11 @@ //! //! [earthstar]: https://github.com/earthstar-project/willow-js/blob/0db4b9ec7710fb992ab75a17bd8557040d9a1062/src/wgps/pai/pai_finder.ts //! [willow]: https://github.com/earthstar-project/earthstar/blob/16d6d4028c22fdbb72f7395013b29be7dcd9217a/src/schemes/schemes.ts#L662 -//! use std::collections::{HashMap, HashSet}; use anyhow::Result; use futures_lite::{Stream, StreamExt}; - use tracing::{debug, trace}; use crate::{ @@ -508,6 +506,7 @@ mod tests { use tokio_util::sync::PollSender; use tracing::{error_span, Instrument, Span}; + use super::{Input, Output, PaiFinder}; use crate::{ proto::{ data_model::{Path, PathExt}, @@ -522,8 +521,6 @@ mod tests { session::{pai_finder::PaiIntersection, Error}, }; - use super::{Input, Output, PaiFinder}; - #[tokio::test] async fn pai_smoke() { let _guard = iroh_test::logging::setup(); diff --git a/iroh-willow/src/session/payload.rs b/iroh-willow/src/session/payload.rs index be3596529ed..6f2fb91ad3a 100644 --- a/iroh-willow/src/session/payload.rs +++ b/iroh-willow/src/session/payload.rs @@ -12,14 +12,13 @@ use iroh_io::TokioStreamReader; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use super::Error; use crate::{ proto::{data_model::PayloadDigest, wgps::Message}, session::channels::ChannelSenders, util::pipe::chunked_pipe, }; -use super::Error; - const CHUNK_SIZE: usize = 1024 * 32; /// Send a payload in chunks. diff --git a/iroh-willow/src/session/resource.rs b/iroh-willow/src/session/resource.rs index eb145741da7..20d663b401b 100644 --- a/iroh-willow/src/session/resource.rs +++ b/iroh-willow/src/session/resource.rs @@ -3,9 +3,8 @@ use std::{ task::{Context, Poll, Waker}, }; -use crate::proto::wgps::{IsHandle, ResourceHandle}; - use super::Error; +use crate::proto::wgps::{IsHandle, ResourceHandle}; /// The bind scope for resources. /// diff --git a/iroh-willow/src/session/run.rs b/iroh-willow/src/session/run.rs index ad452285820..b76be5dc3ee 100644 --- a/iroh-willow/src/session/run.rs +++ b/iroh-willow/src/session/run.rs @@ -11,6 +11,12 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use tracing::{debug, error_span, trace, Instrument, Span}; +use super::{ + channels::ChannelReceivers, + data::{DataReceiver, DataSender}, + reconciler::Reconciler, + SessionMode, +}; use crate::{ net::ConnHandle, proto::wgps::{ControlIssueGuarantee, LogicalChannel, Message, SetupBindAreaOfInterest}, @@ -32,13 +38,6 @@ use crate::{ }, }; -use super::{ - channels::ChannelReceivers, - data::{DataReceiver, DataSender}, - reconciler::Reconciler, - SessionMode, -}; - const INITIAL_GUARANTEES: u64 = u64::MAX; pub(crate) async fn run_session( diff --git a/iroh-willow/src/store.rs b/iroh-willow/src/store.rs index b3828d03bd8..2f516e5c6fd 100644 --- a/iroh-willow/src/store.rs +++ b/iroh-willow/src/store.rs @@ -8,23 +8,22 @@ use anyhow::{anyhow, Context, Result}; use rand_core::CryptoRngCore; use traits::EntryStorage; +pub(crate) use self::traits::EntryOrigin; +use self::{ + auth::{Auth, AuthError}, + traits::Storage, +}; use crate::{ form::{AuthForm, EntryForm, EntryOrForm, SubspaceForm, TimestampForm}, interest::{CapSelector, UserSelector}, proto::{ - data_model::Entry, - data_model::{AuthorisedEntry, PayloadDigest}, + data_model::{AuthorisedEntry, Entry, PayloadDigest}, keys::{NamespaceId, NamespaceKind, NamespaceSecretKey, UserId}, }, store::traits::SecretStorage, util::time::system_time_now, }; -use self::auth::{Auth, AuthError}; -use self::traits::Storage; - -pub(crate) use self::traits::EntryOrigin; - pub(crate) mod auth; pub mod memory; pub mod persistent; diff --git a/iroh-willow/src/store/memory.rs b/iroh-willow/src/store/memory.rs index c609a9d1de9..afeb20f1841 100644 --- a/iroh-willow/src/store/memory.rs +++ b/iroh-willow/src/store/memory.rs @@ -5,32 +5,33 @@ //! It does not have good performance, it does a lot of iterating. But it is concise and can //! hopefully easily kept correct. -use std::cell::RefCell; -use std::collections::{HashMap, VecDeque}; -use std::pin::Pin; -use std::rc::{Rc, Weak}; -use std::task::{ready, Context, Poll, Waker}; +use std::{ + cell::RefCell, + collections::{HashMap, VecDeque}, + pin::Pin, + rc::{Rc, Weak}, + task::{ready, Context, Poll, Waker}, +}; use anyhow::Result; use futures_util::Stream; use tracing::debug; -use crate::proto::data_model::PathExt; -use crate::proto::grouping::Area; +use super::{ + traits::{StoreEvent, SubscribeParams}, + EntryOrigin, +}; use crate::{ interest::{CapSelector, CapabilityPack}, proto::{ - data_model::{AuthorisedEntry, Path, SubspaceId, WriteCapability}, - grouping::Range3d, + data_model::{AuthorisedEntry, Path, PathExt, SubspaceId, WriteCapability}, + grouping::{Area, Range3d}, keys::{NamespaceId, NamespaceSecretKey, UserId, UserSecretKey}, meadowcap::{self, is_wider_than, ReadAuthorisation}, }, store::traits, }; -use super::traits::{StoreEvent, SubscribeParams}; -use super::EntryOrigin; - #[derive(Debug, Clone, Default)] pub struct Store { secrets: Rc>, diff --git a/iroh-willow/src/store/persistent.rs b/iroh-willow/src/store/persistent.rs index f7f44ef773b..f5d6c1a22bf 100644 --- a/iroh-willow/src/store/persistent.rs +++ b/iroh-willow/src/store/persistent.rs @@ -1,7 +1,3 @@ -use anyhow::Result; -use ed25519_dalek::ed25519; -use futures_util::Stream; -use redb::{Database, ReadableTable}; use std::{ cell::{Ref, RefCell, RefMut}, collections::HashMap, @@ -12,9 +8,19 @@ use std::{ task::{ready, Context, Poll}, time::Duration, }; + +use anyhow::Result; +use ed25519_dalek::ed25519; +use futures_util::Stream; +use redb::{Database, ReadableTable}; use willow_data_model::SubspaceId as _; use willow_store::{QueryRange, QueryRange3d}; +use super::{ + memory, + traits::{self, SplitAction, StoreEvent, SubscribeParams}, + willow_store_glue::{to_query, IrohWillowParams}, +}; use crate::{ interest::{CapSelector, CapabilityPack}, proto::{ @@ -32,12 +38,6 @@ use crate::{ }, }; -use super::{ - memory, - traits::{self, SplitAction, StoreEvent, SubscribeParams}, - willow_store_glue::{to_query, IrohWillowParams}, -}; - mod tables; const MAX_COMMIT_DELAY: Duration = Duration::from_millis(500); diff --git a/iroh-willow/tests/basic.rs b/iroh-willow/tests/basic.rs index 9bf92cba890..4531c84bba6 100644 --- a/iroh-willow/tests/basic.rs +++ b/iroh-willow/tests/basic.rs @@ -4,7 +4,6 @@ use anyhow::Result; use bytes::Bytes; use futures_concurrency::future::TryJoin; use futures_lite::StreamExt; - use iroh_blobs::store::{Map, MapEntry}; use iroh_io::AsyncSliceReaderExt; use iroh_net::key::SecretKey; @@ -375,11 +374,6 @@ mod util { use bytes::Bytes; use futures_concurrency::future::TryJoin; use iroh_net::{Endpoint, NodeId}; - use rand::SeedableRng; - use rand_chacha::ChaCha12Rng; - use rand_core::CryptoRngCore; - use tokio::task::JoinHandle; - use iroh_willow::{ engine::{AcceptOpts, Engine}, form::EntryForm, @@ -391,6 +385,10 @@ mod util { }, ALPN, }; + use rand::SeedableRng; + use rand_chacha::ChaCha12Rng; + use rand_core::CryptoRngCore; + use tokio::task::JoinHandle; pub fn create_rng(seed: &str) -> ChaCha12Rng { let seed = iroh_base::hash::Hash::new(seed); @@ -412,7 +410,7 @@ mod util { ) -> Result { let endpoint = Endpoint::builder() .secret_key(secret_key) - .relay_mode(iroh_net::relay::RelayMode::Disabled) + .relay_mode(iroh_net::RelayMode::Disabled) .alpns(vec![ALPN.to_vec()]) .bind() .await?; diff --git a/iroh/tests/spaces.rs b/iroh-willow/tests/spaces.rs similarity index 59% rename from iroh/tests/spaces.rs rename to iroh-willow/tests/spaces.rs index cc411603630..ff01d8ce6af 100644 --- a/iroh/tests/spaces.rs +++ b/iroh-willow/tests/spaces.rs @@ -2,12 +2,10 @@ use std::{collections::BTreeMap, time::Duration}; use anyhow::ensure; use futures_lite::StreamExt; -use iroh::client::{ - spaces::{EntryForm, Space}, - Iroh, -}; -use iroh_net::{key::SecretKey, NodeAddr}; +use iroh_io::AsyncSliceReaderExt; +use iroh_net::{key::SecretKey, Endpoint, NodeAddr}; use iroh_willow::{ + engine::AcceptOpts, interest::{AreaOfInterestSelector, CapSelector, DelegateTo, RestrictArea}, proto::{ data_model::{Path, PathExt}, @@ -15,34 +13,167 @@ use iroh_willow::{ keys::{NamespaceKind, UserId}, meadowcap::AccessMode, }, + rpc::{ + client::{Client, EntryForm, Space}, + proto::RpcService, + }, session::{intents::Completion, SessionMode}, store::traits::{EntryOrigin, StoreEvent}, + Engine, }; use proptest::{collection::vec, prelude::Strategy, sample::select}; +use quic_rpc::RpcServer; use test_strategy::proptest; use testresult::TestResult; +use tokio::task::JoinSet; +use tokio_util::sync::{CancellationToken, DropGuard}; use tracing::{error, info}; /// Spawn an iroh node in a separate thread and tokio runtime, and return /// the address and client. -async fn spawn_node(persist_test_mode: bool) -> (NodeAddr, Iroh) { +async fn spawn_node( + persist_test_mode: bool, +) -> (NodeAddr, Client, iroh_blobs::store::mem::Store, DropGuard) { let (sender, receiver) = tokio::sync::oneshot::channel(); std::thread::spawn(move || { let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; runtime.block_on(async move { + let blobs_store = iroh_blobs::store::mem::Store::default(); let secret_key = SecretKey::generate(); - let node = iroh::node::Builder::default() - .enable_spaces_persist_test_mode(persist_test_mode) + let endpoint = Endpoint::builder() .secret_key(secret_key) - .relay_mode(iroh_net::relay::RelayMode::Disabled) - .node_discovery(iroh::node::DiscoveryConfig::None) - .spawn() + .alpns(vec![iroh_willow::ALPN.to_vec()]) + .relay_mode(iroh_net::RelayMode::Disabled) + .bind() .await?; - let addr = node.net().node_addr().await?; - sender.send((addr, node.client().clone())).unwrap(); - node.cancel_token().cancelled().await; + + let store = blobs_store.clone(); + let engine = if persist_test_mode { + Engine::spawn( + endpoint.clone(), + move || { + iroh_willow::store::persistent::Store::new_memory(store) + .expect("couldn't initialize store") + }, + AcceptOpts::default(), + ) + } else { + Engine::spawn( + endpoint.clone(), + move || iroh_willow::store::memory::Store::new(store), + AcceptOpts::default(), + ) + }; + let (rpc, controller) = quic_rpc::transport::flume::channel::< + iroh_willow::rpc::proto::Request, + iroh_willow::rpc::proto::Response, + >(32); + let rpc = quic_rpc::transport::boxed::BoxedListener::new(rpc); + let controller = quic_rpc::transport::boxed::BoxedConnector::new(controller); + let client = + iroh_willow::rpc::client::Client::new(quic_rpc::RpcClient::new(controller.clone())); + + let rpc = RpcServer::::new(rpc); + + // wait for direct addresses + // endpoint.direct_addresses().next().await; + let addr = endpoint.node_addr().await?; + + let cancel_token = CancellationToken::new(); + sender + .send((addr, client, blobs_store, cancel_token.clone().drop_guard())) + .unwrap(); + + let mut tasks = JoinSet::new(); + + loop { + tokio::select! { + biased; + _ = cancel_token.cancelled() => { + break; + }, + request = rpc.accept() => { + match request { + Ok(accepting) => { + tasks.spawn({ + let engine = engine.clone(); + let endpoint = endpoint.clone(); + async move { + let (msg, chan) = accepting.read_first().await?; + engine.handle_spaces_request(endpoint, msg, chan).await?; + anyhow::Ok(()) + } + }); + } + Err(e) => { + tracing::warn!("RPC request error: {e:?}"); + } + } + }, + Some(incoming) = endpoint.accept() => { + tasks.spawn({ + let engine = engine.clone(); + async move { + let mut connecting = match incoming.accept() { + Ok(conn) => conn, + Err(err) => { + tracing::warn!("Ignoring iroh-net connection: accept failed: {err:#}"); + return Ok(()); + } + }; + let alpn = match connecting.alpn().await { + Ok(alpn) => alpn, + Err(err) => { + tracing::warn!("Ignoring connection: Invalid handshake: {err:#}"); + return Ok(()); + } + }; + if alpn != iroh_willow::ALPN { + tracing::warn!("Ignoring connection: ALPN is not willow ({})", hex::encode(alpn)); + return Ok(()); + } + let conn = match connecting.await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!("Ignoring connection: Failed to connect: {err:#}"); + return Ok(()); + } + }; + + if let Err(err) = engine.handle_connection(conn).await { + tracing::warn!("Handling incoming connection ended with error: {err}"); + } + + anyhow::Ok(()) + } + }); + }, + res = tasks.join_next(), if !tasks.is_empty() => { + match res { + Some(Err(outer)) => { + if outer.is_panic() { + tracing::error!("Task panicked: {outer:?}"); + break; + } else if outer.is_cancelled() { + tracing::debug!("Task cancelled: {outer:?}"); + } else { + tracing::error!("Task failed: {outer:?}"); + break; + } + } + Some(Ok(Err(inner))) => { + tracing::debug!("Task errored: {inner:?}"); + } + _ => {} + } + }, + } + } + + engine.shutdown().await?; + anyhow::Ok(()) })?; anyhow::Ok(()) @@ -93,14 +224,14 @@ fn prop_sync_simulation_matches_model( .block_on(async { let mut simulated_entries: BTreeMap<(Peer, String), String> = BTreeMap::new(); - let (addr_x, iroh_x) = spawn_node(x_is_persist).await; - let (addr_y, iroh_y) = spawn_node(y_is_persist).await; + let (addr_x, iroh_x, blobs_x, _guard_x) = spawn_node(x_is_persist).await; + let (addr_y, iroh_y, blobs_y, _guard_y) = spawn_node(y_is_persist).await; let node_id_x = addr_x.node_id; let node_id_y = addr_y.node_id; - iroh_x.net().add_node_addr(addr_y.clone()).await?; - iroh_y.net().add_node_addr(addr_x.clone()).await?; - let user_x = iroh_x.spaces().create_user().await?; - let user_y = iroh_y.spaces().create_user().await?; + iroh_x.add_node_addr(addr_y.clone()).await?; + iroh_y.add_node_addr(addr_x.clone()).await?; + let user_x = iroh_x.create_user().await?; + let user_y = iroh_y.create_user().await?; info!( "X is node {} user {}", node_id_x.fmt_short(), @@ -111,7 +242,7 @@ fn prop_sync_simulation_matches_model( node_id_y.fmt_short(), user_y.fmt_short() ); - let space_x = iroh_x.spaces().create(NamespaceKind::Owned, user_x).await?; + let space_x = iroh_x.create(NamespaceKind::Owned, user_x).await?; let ticket = space_x .share(user_y, AccessMode::Write, RestrictArea::None) @@ -119,7 +250,6 @@ fn prop_sync_simulation_matches_model( // give betty access let (space_y, syncs) = iroh_y - .spaces() .import_and_sync(ticket, SessionMode::ReconcileOnce) .await?; @@ -132,9 +262,9 @@ fn prop_sync_simulation_matches_model( let count = rounds.len(); for (i, (peer, round)) in rounds.into_iter().enumerate() { let i = i + 1; - let (space, user) = match peer { - Peer::X => (&space_x, user_x), - Peer::Y => (&space_y, user_y), + let (space, blobs, user) = match peer { + Peer::X => (&space_x, &blobs_x, user_x), + Peer::Y => (&space_y, &blobs_y, user_y), }; info!(active=?peer, "[{i}/{count}] round start"); @@ -142,6 +272,7 @@ fn prop_sync_simulation_matches_model( info!(?key, ?value, "[{i}/{count}] write"); space .insert_bytes( + blobs, EntryForm::new(user, Path::from_bytes(&[key.as_bytes()])?), value.clone().into_bytes(), ) @@ -174,8 +305,8 @@ fn prop_sync_simulation_matches_model( info!("[{i}/{count}] sync complete"); - let map_x = space_to_map(&space_x, &iroh_x, user_x, user_y).await?; - let map_y = space_to_map(&space_y, &iroh_y, user_x, user_y).await?; + let map_x = space_to_map(&space_x, &blobs_x, user_x, user_y).await?; + let map_y = space_to_map(&space_y, &blobs_y, user_x, user_y).await?; ensure!( map_x == map_y, "states out of sync:\n{map_x:#?}\n !=\n{map_y:#?}" @@ -198,7 +329,7 @@ fn prop_sync_simulation_matches_model( info!("completed {count} rounds successfully"); - tokio::try_join!(iroh_x.shutdown(false), iroh_y.shutdown(false))?; + // tokio::try_join!(iroh_x.shutdown(false), iroh_y.shutdown(false))?; Ok(()) }); @@ -210,7 +341,7 @@ fn prop_sync_simulation_matches_model( async fn space_to_map( space: &Space, - node: &Iroh, + blobs: &impl iroh_blobs::store::Store, user_x: UserId, user_y: UserId, ) -> anyhow::Result> { @@ -229,7 +360,13 @@ async fn space_to_map( .ok_or_else(|| anyhow::anyhow!("path component missing"))?; let key = String::from_utf8(key_component.to_vec())?; - let value = node.blobs().read_to_bytes(entry.payload_digest().0).await?; + use iroh_blobs::store::*; + let entry = blobs + .get(&entry.payload_digest().0) + .await? + .ok_or_else(|| anyhow::anyhow!("blob missing"))?; + let mut reader = entry.data_reader().await?; + let value = reader.read_to_end().await?; let user = auth.capability.receiver(); let peer = role_lookup @@ -268,28 +405,27 @@ impl std::error::Error for AnyhowStdErr { #[tokio::test] async fn spaces_smoke() -> TestResult { iroh_test::logging::setup_multithreaded(); - let (alfie_addr, alfie) = spawn_node(false).await; - let (betty_addr, betty) = spawn_node(false).await; + let (alfie_addr, alfie, alfie_blobs, _g1) = spawn_node(false).await; + let (betty_addr, betty, betty_blobs, _g2) = spawn_node(false).await; info!("alfie is {}", alfie_addr.node_id.fmt_short()); info!("betty is {}", betty_addr.node_id.fmt_short()); - let betty_user = betty.spaces().create_user().await?; - let alfie_user = alfie.spaces().create_user().await?; - let alfie_space = alfie - .spaces() - .create(NamespaceKind::Owned, alfie_user) - .await?; + let betty_user = betty.create_user().await?; + let alfie_user = alfie.create_user().await?; + let alfie_space = alfie.create(NamespaceKind::Owned, alfie_user).await?; let namespace = alfie_space.namespace_id(); alfie_space .insert_bytes( + &alfie_blobs, EntryForm::new(alfie_user, Path::from_bytes(&[b"foo", b"bar"])?), "hello betty", ) .await?; alfie_space .insert_bytes( + &alfie_blobs, EntryForm::new(alfie_user, Path::from_bytes(&[b"foo", b"boo"])?), "this is alfie", ) @@ -301,7 +437,6 @@ async fn spaces_smoke() -> TestResult { println!("ticket {ticket:?}"); let (betty_space, betty_sync_intent) = betty - .spaces() .import_and_sync(ticket, SessionMode::ReconcileOnce) .await?; @@ -319,6 +454,7 @@ async fn spaces_smoke() -> TestResult { let res = betty_space .insert_bytes( + &betty_blobs, EntryForm::new(betty_user, Path::from_bytes(&[b"hello"])?), "this is betty", ) @@ -328,24 +464,24 @@ async fn spaces_smoke() -> TestResult { let area = Area::new_subspace(betty_user); let caps = alfie - .spaces() .delegate_caps( CapSelector::any(namespace), AccessMode::Write, DelegateTo::new(betty_user, RestrictArea::Restrict(area)), ) .await?; - betty.spaces().import_caps(caps).await?; + betty.import_caps(caps).await?; let res = betty_space .insert_bytes( + &betty_blobs, EntryForm::new(betty_user, Path::from_bytes(&[b"hello"])?), "this is betty", ) .await; assert!(res.is_ok()); - alfie.net().add_node_addr(betty_addr.clone()).await?; + alfie.add_node_addr(betty_addr.clone()).await?; let mut alfie_sync_intent = alfie_space .sync_once(betty_addr.node_id, Default::default()) .await?; @@ -364,17 +500,14 @@ async fn spaces_smoke() -> TestResult { #[tokio::test] async fn spaces_subscription() -> TestResult { iroh_test::logging::setup_multithreaded(); - let (alfie_addr, alfie) = spawn_node(false).await; - let (betty_addr, betty) = spawn_node(false).await; + let (alfie_addr, alfie, alfie_blobs, _g1) = spawn_node(false).await; + let (betty_addr, betty, betty_blobs, _g2) = spawn_node(false).await; info!("alfie is {}", alfie_addr.node_id.fmt_short()); info!("betty is {}", betty_addr.node_id.fmt_short()); - let betty_user = betty.spaces().create_user().await?; - let alfie_user = alfie.spaces().create_user().await?; - let alfie_space = alfie - .spaces() - .create(NamespaceKind::Owned, alfie_user) - .await?; + let betty_user = betty.create_user().await?; + let alfie_user = alfie.create_user().await?; + let alfie_space = alfie.create(NamespaceKind::Owned, alfie_user).await?; let _namespace = alfie_space.namespace_id(); @@ -387,7 +520,6 @@ async fn spaces_subscription() -> TestResult { .await?; let (betty_space, betty_sync_intent) = betty - .spaces() .import_and_sync(ticket, SessionMode::Continuous) .await?; @@ -403,6 +535,7 @@ async fn spaces_subscription() -> TestResult { betty_space .insert_bytes( + &betty_blobs, EntryForm::new(betty_user, Path::from_bytes(&[b"foo"])?), "hi", ) @@ -421,6 +554,7 @@ async fn spaces_subscription() -> TestResult { alfie_space .insert_bytes( + &alfie_blobs, EntryForm::new(alfie_user, Path::from_bytes(&[b"bar"])?), "hi!!", ) @@ -449,16 +583,13 @@ async fn spaces_subscription() -> TestResult { async fn test_restricted_area() -> testresult::TestResult { iroh_test::logging::setup_multithreaded(); const TIMEOUT: Duration = Duration::from_secs(20); - let (alfie_addr, alfie) = spawn_node(false).await; - let (betty_addr, betty) = spawn_node(false).await; + let (alfie_addr, alfie, _, _g1) = spawn_node(false).await; + let (betty_addr, betty, _, _g2) = spawn_node(false).await; info!("alfie is {}", alfie_addr.node_id.fmt_short()); info!("betty is {}", betty_addr.node_id.fmt_short()); - let alfie_user = alfie.spaces().create_user().await?; - let betty_user = betty.spaces().create_user().await?; - let alfie_space = alfie - .spaces() - .create(NamespaceKind::Owned, alfie_user) - .await?; + let alfie_user = alfie.create_user().await?; + let betty_user = betty.create_user().await?; + let alfie_space = alfie.create(NamespaceKind::Owned, alfie_user).await?; let space_ticket = alfie_space .share( betty_user, @@ -467,7 +598,6 @@ async fn test_restricted_area() -> testresult::TestResult { ) .await?; let (betty_space, syncs) = betty - .spaces() .import_and_sync(space_ticket, SessionMode::ReconcileOnce) .await?; let completion = tokio::time::timeout(TIMEOUT, syncs.complete_all()).await?; diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 93d0bb5c0b8..4b7c9186a1c 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh" -version = "0.27.0" +version = "0.28.1" edition = "2021" readme = "README.md" description = "A toolkit for building distributed applications" @@ -16,6 +16,7 @@ rust-version = "1.76" workspace = true [dependencies] +cc = "=1.1.31" # enforce cc version, because of https://github.com/rust-lang/cc-rs/issues/1278 anyhow = { version = "1" } async-channel = "2.3.1" bao-tree = { version = "0.13", features = ["tokio_fsm"], default-features = false } @@ -26,22 +27,22 @@ futures-lite = "2.3" futures-util = "0.3" genawaiter = { version = "0.99", default-features = false, features = ["futures03"] } hex = { version = "0.4.3" } -iroh-blobs = { version = "0.27.0", path = "../iroh-blobs", features = ["downloader"] } -iroh-base = { version = "0.27.0", path = "../iroh-base", features = ["key"] } +iroh-blobs = { version = "0.28.0", features = ["downloader"] } +iroh-base = { version = "0.28.0", features = ["key"] } iroh-io = { version = "0.6.0", features = ["stats"] } -iroh-metrics = { version = "0.27.0", path = "../iroh-metrics", optional = true } -iroh-net = { version = "0.27.0", path = "../iroh-net", features = ["discovery-local-network"] } -iroh-willow = { version = "0.27.0", path = "../iroh-willow" } +iroh-metrics = { version = "0.28.0", optional = true } +iroh-net = { version = "0.28.1", features = ["discovery-local-network"] } +iroh-router = { version = "0.28.0" } nested_enum_utils = "0.1.0" num_cpus = { version = "1.15.0" } portable-atomic = "1" -iroh-docs = { version = "0.27.0", path = "../iroh-docs" } -iroh-gossip = { version = "0.27.0", path = "../iroh-gossip" } +iroh-docs = { version = "0.28.0", features = ["rpc"] } +iroh-gossip = "0.28.1" parking_lot = "0.12.1" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -quic-rpc = { version = "0.12", default-features = false, features = ["flume-transport", "quinn-transport"] } -quic-rpc-derive = { version = "0.12" } -quinn = { package = "iroh-quinn", version = "0.11" } +quic-rpc = { version = "0.15", default-features = false, features = ["flume-transport", "quinn-transport"] } +quic-rpc-derive = { version = "0.15" } +quinn = { package = "iroh-quinn", version = "0.12" } rand = "0.8" serde = { version = "1", features = ["derive"] } strum = { version = "0.25", features = ["derive"] } @@ -51,7 +52,7 @@ tokio = { version = "1", features = ["io-util", "rt"] } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["codec", "io-util", "io", "time"] } tracing = "0.1" -walkdir = "2" +iroh-relay = { version = "0.28", path = "../iroh-relay" } # Examples clap = { version = "4", features = ["derive"], optional = true } @@ -61,6 +62,7 @@ console = { version = "0.15.5", optional = true } # Documentation tests url = { version = "2.5.0", features = ["serde"] } +serde-error = "0.1.3" [features] default = ["metrics", "fs-store"] @@ -76,7 +78,7 @@ test-utils = ["iroh-net/test-utils"] anyhow = { version = "1" } genawaiter = { version = "0.99", features = ["futures03"] } iroh = { path = ".", features = ["test-utils"] } -iroh-test = { path = "../iroh-test" } +iroh-test = "0.28.0" proptest = "1.2.0" rand_chacha = "0.3.1" regex = { version = "1.7.1", features = ["std"] } diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index 3cc0840b2f7..4d2338012aa 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -16,6 +16,7 @@ async fn main() -> anyhow::Result<()> { // Could also use `node` directly, as it derefs to the client. let client = node.client(); + let blobs = client.blobs(); let doc = client.docs().create().await?; let author = client.authors().default().await?; @@ -24,8 +25,7 @@ async fn main() -> anyhow::Result<()> { let mut stream = doc.get_many(Query::all()).await?; while let Some(entry) = stream.try_next().await? { println!("entry {}", fmt_entry(&entry)); - // You can pass either `&doc` or the `client`. - let content = entry.content_bytes(&doc).await?; + let content = blobs.read_to_bytes(entry.content_hash()).await?; println!(" content {}", std::str::from_utf8(&content)?) } diff --git a/iroh/examples/collection-provide.rs b/iroh/examples/collection-provide.rs index 429288a2f2e..06d9f6dcfee 100644 --- a/iroh/examples/collection-provide.rs +++ b/iroh/examples/collection-provide.rs @@ -7,7 +7,7 @@ //! run this example from the project root: //! $ cargo run --example collection-provide use iroh::blobs::{format::collection::Collection, util::SetTagOption, BlobFormat}; -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -44,14 +44,9 @@ async fn main() -> anyhow::Result<()> { // create a ticket // tickets wrap all details needed to get a collection - let ticket = node - .blobs() - .share( - hash, - BlobFormat::HashSeq, - AddrInfoOptions::RelayAndAddresses, - ) - .await?; + let mut addr = node.net().node_addr().await?; + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, BlobFormat::HashSeq)?; // print some info about the node println!("serving hash: {}", ticket.hash()); diff --git a/iroh/examples/custom-protocol.rs b/iroh/examples/custom-protocol.rs index f5c224e1483..b6bdd56d387 100644 --- a/iroh/examples/custom-protocol.rs +++ b/iroh/examples/custom-protocol.rs @@ -50,7 +50,7 @@ use iroh::{ endpoint::{get_remote_node_id, Connecting}, Endpoint, NodeId, }, - node::ProtocolHandler, + router::ProtocolHandler, }; use tracing_subscriber::{prelude::*, EnvFilter}; @@ -118,7 +118,7 @@ async fn main() -> Result<()> { // Print out our query results. for hash in hashes { - read_and_print(node.blobs(), hash).await?; + read_and_print(&node.blobs(), hash).await?; } } } diff --git a/iroh/examples/hello-world-provide.rs b/iroh/examples/hello-world-provide.rs index 15c21801dea..a74bc3bae35 100644 --- a/iroh/examples/hello-world-provide.rs +++ b/iroh/examples/hello-world-provide.rs @@ -3,7 +3,7 @@ //! This is using an in memory database and a random node id. //! run this example from the project root: //! $ cargo run --example hello-world-provide -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -27,10 +27,9 @@ async fn main() -> anyhow::Result<()> { let res = node.blobs().add_bytes("Hello, world!").await?; // create a ticket - let ticket = node - .blobs() - .share(res.hash, res.format, AddrInfoOptions::RelayAndAddresses) - .await?; + let mut addr = node.net().node_addr().await?; + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, res.hash, res.format)?; // print some info about the node println!("serving hash: {}", ticket.hash()); diff --git a/iroh/examples/local-swarm-discovery.rs b/iroh/examples/local-swarm-discovery.rs index 9dd1076c9f2..641e62ec22e 100644 --- a/iroh/examples/local-swarm-discovery.rs +++ b/iroh/examples/local-swarm-discovery.rs @@ -71,7 +71,7 @@ async fn main() -> anyhow::Result<()> { .secret_key(key) .node_discovery(cfg) .bind_random_port() - .relay_mode(iroh_net::relay::RelayMode::Disabled) + .relay_mode(iroh_net::RelayMode::Disabled) .spawn() .await?; diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 9980efde718..e8ffc051a1b 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -14,25 +14,18 @@ pub use crate::rpc_protocol::RpcService; mod quic; -pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; -pub use self::{docs::Doc, net::NodeStatus}; +pub use iroh_blobs::rpc::client::{blobs, tags}; +pub use iroh_docs::rpc::client::{authors, docs, docs::Doc}; +pub use iroh_gossip::rpc::client as gossip; -pub mod authors; -pub mod blobs; -pub mod docs; -pub mod gossip; +pub use self::net::NodeStatus; +pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; pub mod net; -pub mod spaces; -pub mod tags; - -/// Iroh rpc connection - boxed so that we can have a concrete type. -pub(crate) type RpcConnection = quic_rpc::transport::boxed::Connection; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. /// Iroh rpc client - boxed so that we can have a concrete type. -pub type RpcClient = - quic_rpc::RpcClient>; +pub type RpcClient = quic_rpc::RpcClient; /// An iroh client. /// @@ -62,33 +55,28 @@ impl Iroh { } /// Returns the blobs client. - pub fn blobs(&self) -> &blobs::Client { - blobs::Client::ref_cast(&self.rpc) + pub fn blobs(&self) -> blobs::Client { + blobs::Client::new(self.rpc.clone().map().boxed()) } /// Returns the docs client. - pub fn docs(&self) -> &docs::Client { - docs::Client::ref_cast(&self.rpc) + pub fn docs(&self) -> iroh_docs::rpc::client::docs::Client { + iroh_docs::rpc::client::docs::Client::new(self.rpc.clone().map().boxed()) } - /// Returns the spaces client. - pub fn spaces(&self) -> &spaces::Client { - spaces::Client::ref_cast(&self.rpc) - } - - /// Returns the authors client. - pub fn authors(&self) -> &authors::Client { - authors::Client::ref_cast(&self.rpc) + /// Returns the docs client. + pub fn authors(&self) -> iroh_docs::rpc::client::authors::Client { + iroh_docs::rpc::client::authors::Client::new(self.rpc.clone().map().boxed()) } /// Returns the tags client. - pub fn tags(&self) -> &tags::Client { - tags::Client::ref_cast(&self.rpc) + pub fn tags(&self) -> tags::Client { + tags::Client::new(self.rpc.clone().map().boxed()) } /// Returns the gossip client. - pub fn gossip(&self) -> &gossip::Client { - gossip::Client::ref_cast(&self.rpc) + pub fn gossip(&self) -> gossip::Client { + gossip::Client::new(self.rpc.clone().map().boxed()) } /// Returns the net client. diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs deleted file mode 100644 index ac606a748c1..00000000000 --- a/iroh/src/client/authors.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! API for author management. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::authors()`](crate::client::Iroh::authors). - -use anyhow::Result; -use futures_lite::{stream::StreamExt, Stream}; -use iroh_docs::{Author, AuthorId}; -use ref_cast::RefCast; - -use super::{flatten, RpcClient}; -use crate::rpc_protocol::authors::{ - CreateRequest, DeleteRequest, ExportRequest, GetDefaultRequest, ImportRequest, ListRequest, - SetDefaultRequest, -}; - -/// Iroh authors client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl Client { - /// Creates a new document author. - /// - /// You likely want to save the returned [`AuthorId`] somewhere so that you can use this author - /// again. - /// - /// If you need only a single author, use [`Self::default`]. - pub async fn create(&self) -> Result { - let res = self.rpc.rpc(CreateRequest).await??; - Ok(res.author_id) - } - - /// Returns the default document author of this node. - /// - /// On persistent nodes, the author is created on first start and its public key is saved - /// in the data directory. - /// - /// The default author can be set with [`Self::set_default`]. - pub async fn default(&self) -> Result { - let res = self.rpc.rpc(GetDefaultRequest).await??; - Ok(res.author_id) - } - - /// Sets the node-wide default author. - /// - /// If the author does not exist, an error is returned. - /// - /// On a persistent node, the author id will be saved to a file in the data directory and - /// reloaded after a restart. - pub async fn set_default(&self, author_id: AuthorId) -> Result<()> { - self.rpc.rpc(SetDefaultRequest { author_id }).await??; - Ok(()) - } - - /// Lists document authors for which we have a secret key. - /// - /// It's only possible to create writes from authors that we have the secret key of. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest {}).await?; - Ok(flatten(stream).map(|res| res.map(|res| res.author_id))) - } - - /// Exports the given author. - /// - /// Warning: The [`Author`] struct contains sensitive data. - pub async fn export(&self, author: AuthorId) -> Result> { - let res = self.rpc.rpc(ExportRequest { author }).await??; - Ok(res.author) - } - - /// Imports the given author. - /// - /// Warning: The [`Author`] struct contains sensitive data. - pub async fn import(&self, author: Author) -> Result<()> { - self.rpc.rpc(ImportRequest { author }).await??; - Ok(()) - } - - /// Deletes the given author by id. - /// - /// Warning: This permanently removes this author. - /// - /// Returns an error if attempting to delete the default author. - pub async fn delete(&self, author: AuthorId) -> Result<()> { - self.rpc.rpc(DeleteRequest { author }).await??; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::node::Node; - - #[tokio::test] - async fn test_authors() -> Result<()> { - let node = Node::memory().enable_docs().spawn().await?; - - // default author always exists - let authors: Vec<_> = node.authors().list().await?.try_collect().await?; - assert_eq!(authors.len(), 1); - let default_author = node.authors().default().await?; - assert_eq!(authors, vec![default_author]); - - let author_id = node.authors().create().await?; - - let authors: Vec<_> = node.authors().list().await?.try_collect().await?; - assert_eq!(authors.len(), 2); - - let author = node - .authors() - .export(author_id) - .await? - .expect("should have author"); - node.authors().delete(author_id).await?; - let authors: Vec<_> = node.authors().list().await?.try_collect().await?; - assert_eq!(authors.len(), 1); - - node.authors().import(author).await?; - - let authors: Vec<_> = node.authors().list().await?.try_collect().await?; - assert_eq!(authors.len(), 2); - - assert!(node.authors().default().await? != author_id); - node.authors().set_default(author_id).await?; - assert_eq!(node.authors().default().await?, author_id); - - Ok(()) - } -} diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs deleted file mode 100644 index 3dde9e39a87..00000000000 --- a/iroh/src/client/blobs.rs +++ /dev/null @@ -1,1688 +0,0 @@ -//! API for blobs management. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::blobs()`](crate::client::Iroh::blobs). -//! -//! ## Interacting with the local blob store -//! -//! ### Importing data -//! -//! There are several ways to import data into the local blob store: -//! -//! - [`add_bytes`](Client::add_bytes) -//! imports in memory data. -//! - [`add_stream`](Client::add_stream) -//! imports data from a stream of bytes. -//! - [`add_reader`](Client::add_reader) -//! imports data from an [async reader](tokio::io::AsyncRead). -//! - [`add_from_path`](Client::add_from_path) -//! imports data from a file. -//! -//! The last method imports data from a file on the local filesystem. -//! This is the most efficient way to import large amounts of data. -//! -//! ### Exporting data -//! -//! There are several ways to export data from the local blob store: -//! -//! - [`read_to_bytes`](Client::read_to_bytes) reads data into memory. -//! - [`read`](Client::read) creates a [reader](Reader) to read data from. -//! - [`export`](Client::export) eports data to a file on the local filesystem. -//! -//! ## Interacting with remote nodes -//! -//! - [`download`](Client::download) downloads data from a remote node. -//! - [`share`](Client::share) allows creating a ticket to share data with a -//! remote node. -//! -//! ## Interacting with the blob store itself -//! -//! These are more advanced operations that are usually not needed in normal -//! operation. -//! -//! - [`consistency_check`](Client::consistency_check) checks the internal -//! consistency of the local blob store. -//! - [`validate`](Client::validate) validates the locally stored data against -//! their BLAKE3 hashes. -//! - [`delete_blob`](Client::delete_blob) deletes a blob from the local store. -//! -//! ### Batch operations -//! -//! For complex update operations, there is a [`batch`](Client::batch) API that -//! allows you to add multiple blobs in a single logical batch. -//! -//! Operations in a batch return [temporary tags](crate::blobs::TempTag) that -//! protect the added data from garbage collection as long as the batch is -//! alive. -//! -//! To store the data permanently, a temp tag needs to be upgraded to a -//! permanent tag using [`persist`](crate::client::blobs::Batch::persist) or -//! [`persist_to`](crate::client::blobs::Batch::persist_to). -use std::{ - future::Future, - io, - path::PathBuf, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use futures_util::SinkExt; -use genawaiter::sync::{Co, Gen}; -use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; -use iroh_blobs::{ - export::ExportProgress as BytesExportProgress, - format::collection::{Collection, SimpleStore}, - get::db::DownloadProgress as BytesDownloadProgress, - store::{BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ValidateProgress}, - util::SetTagOption, - BlobFormat, Hash, Tag, -}; -use iroh_net::NodeAddr; -use portable_atomic::{AtomicU64, Ordering}; -use quic_rpc::client::BoxStreamSync; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; -use tokio_util::io::{ReaderStream, StreamReader}; -use tracing::warn; -mod batch; -pub use batch::{AddDirOpts, AddFileOpts, AddReaderOpts, Batch}; - -use super::{flatten, tags, Iroh, RpcClient}; -use crate::rpc_protocol::{ - blobs::{ - AddPathRequest, AddStreamRequest, AddStreamUpdate, BatchCreateRequest, BatchCreateResponse, - BlobStatusRequest, ConsistencyCheckRequest, CreateCollectionRequest, - CreateCollectionResponse, DeleteRequest, DownloadRequest, ExportRequest, - ListIncompleteRequest, ListRequest, ReadAtRequest, ReadAtResponse, ValidateRequest, - }, - node::StatusRequest, -}; - -/// Iroh blobs client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl<'a> From<&'a Iroh> for &'a RpcClient { - fn from(client: &'a Iroh) -> &'a RpcClient { - &client.blobs().rpc - } -} - -impl Client { - /// Check if a blob is completely stored on the node. - /// - /// Note that this will return false for blobs that are partially stored on - /// the node. - pub async fn status(&self, hash: Hash) -> Result { - let status = self.rpc.rpc(BlobStatusRequest { hash }).await??; - Ok(status.0) - } - - /// Check if a blob is completely stored on the node. - /// - /// This is just a convenience wrapper around `status` that returns a boolean. - pub async fn has(&self, hash: Hash) -> Result { - match self.status(hash).await { - Ok(BlobStatus::Complete { .. }) => Ok(true), - Ok(_) => Ok(false), - Err(err) => Err(err), - } - } - - /// Create a new batch for adding data. - /// - /// A batch is a context in which temp tags are created and data is added to the node. Temp tags - /// are automatically deleted when the batch is dropped, leading to the data being garbage collected - /// unless a permanent tag is created for it. - pub async fn batch(&self) -> Result { - let (updates, mut stream) = self.rpc.bidi(BatchCreateRequest).await?; - let BatchCreateResponse::Id(batch) = stream.next().await.context("expected scope id")??; - let rpc = self.rpc.clone(); - Ok(Batch::new(batch, rpc, updates, 1024)) - } - - /// Stream the contents of a a single blob. - /// - /// Returns a [`Reader`], which can report the size of the blob before reading it. - pub async fn read(&self, hash: Hash) -> Result { - Reader::from_rpc_read(&self.rpc, hash).await - } - - /// Read offset + len from a single blob. - /// - /// If `len` is `None` it will read the full blob. - pub async fn read_at(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result { - Reader::from_rpc_read_at(&self.rpc, hash, offset, len).await - } - - /// Read all bytes of single blob. - /// - /// This allocates a buffer for the full blob. Use only if you know that the blob you're - /// reading is small. If not sure, use [`Self::read`] and check the size with - /// [`Reader::size`] before calling [`Reader::read_to_bytes`]. - pub async fn read_to_bytes(&self, hash: Hash) -> Result { - Reader::from_rpc_read(&self.rpc, hash) - .await? - .read_to_bytes() - .await - } - - /// Read all bytes of single blob at `offset` for length `len`. - /// - /// This allocates a buffer for the full length. - pub async fn read_at_to_bytes(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result { - Reader::from_rpc_read_at(&self.rpc, hash, offset, len) - .await? - .read_to_bytes() - .await - } - - /// Import a blob from a filesystem path. - /// - /// `path` should be an absolute path valid for the file system on which - /// the node runs. - /// If `in_place` is true, Iroh will assume that the data will not change and will share it in - /// place without copying to the Iroh data directory. - pub async fn add_from_path( - &self, - path: PathBuf, - in_place: bool, - tag: SetTagOption, - wrap: WrapOption, - ) -> Result { - let stream = self - .rpc - .server_streaming(AddPathRequest { - path, - in_place, - tag, - wrap, - }) - .await?; - Ok(AddProgress::new(stream)) - } - - /// Create a collection from already existing blobs. - /// - /// For automatically clearing the tags for the passed in blobs you can set - /// `tags_to_delete` to those tags, and they will be deleted once the collection is created. - pub async fn create_collection( - &self, - collection: Collection, - tag: SetTagOption, - tags_to_delete: Vec, - ) -> anyhow::Result<(Hash, Tag)> { - let CreateCollectionResponse { hash, tag } = self - .rpc - .rpc(CreateCollectionRequest { - collection, - tag, - tags_to_delete, - }) - .await??; - Ok((hash, tag)) - } - - /// Write a blob by passing an async reader. - pub async fn add_reader( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - tag: SetTagOption, - ) -> anyhow::Result { - const CAP: usize = 1024 * 64; // send 64KB per request by default - let input = ReaderStream::with_capacity(reader, CAP); - self.add_stream(input, tag).await - } - - /// Write a blob by passing a stream of bytes. - pub async fn add_stream( - &self, - input: impl Stream> + Send + Unpin + 'static, - tag: SetTagOption, - ) -> anyhow::Result { - let (mut sink, progress) = self.rpc.bidi(AddStreamRequest { tag }).await?; - let mut input = input.map(|chunk| match chunk { - Ok(chunk) => Ok(AddStreamUpdate::Chunk(chunk)), - Err(err) => { - warn!("Abort send, reason: failed to read from source stream: {err:?}"); - Ok(AddStreamUpdate::Abort) - } - }); - tokio::spawn(async move { - // TODO: Is it important to catch this error? It should also result in an error on the - // response stream. If we deem it important, we could one-shot send it into the - // BlobAddProgress and return from there. Not sure. - if let Err(err) = sink.send_all(&mut input).await { - warn!("Failed to send input stream to remote: {err:?}"); - } - }); - - Ok(AddProgress::new(progress)) - } - - /// Write a blob by passing bytes. - pub async fn add_bytes(&self, bytes: impl Into) -> anyhow::Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream(input, SetTagOption::Auto).await?.await - } - - /// Write a blob by passing bytes, setting an explicit tag name. - pub async fn add_bytes_named( - &self, - bytes: impl Into, - name: impl Into, - ) -> anyhow::Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream(input, SetTagOption::Named(name.into())) - .await? - .await - } - - /// Validate hashes on the running node. - /// - /// If `repair` is true, repair the store by removing invalid data. - pub async fn validate( - &self, - repair: bool, - ) -> Result>> { - let stream = self - .rpc - .server_streaming(ValidateRequest { repair }) - .await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Validate hashes on the running node. - /// - /// If `repair` is true, repair the store by removing invalid data. - pub async fn consistency_check( - &self, - repair: bool, - ) -> Result>> { - let stream = self - .rpc - .server_streaming(ConsistencyCheckRequest { repair }) - .await?; - Ok(stream.map(|r| r.map_err(anyhow::Error::from))) - } - - /// Download a blob from another node and add it to the local database. - pub async fn download(&self, hash: Hash, node: NodeAddr) -> Result { - self.download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await - } - - /// Download a hash sequence from another node and add it to the local database. - pub async fn download_hash_seq(&self, hash: Hash, node: NodeAddr) -> Result { - self.download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await - } - - /// Download a blob, with additional options. - pub async fn download_with_opts( - &self, - hash: Hash, - opts: DownloadOptions, - ) -> Result { - let DownloadOptions { - format, - nodes, - tag, - mode, - } = opts; - let stream = self - .rpc - .server_streaming(DownloadRequest { - hash, - format, - nodes, - tag, - mode, - }) - .await?; - Ok(DownloadProgress::new( - stream.map(|res| res.map_err(anyhow::Error::from)), - )) - } - - /// Export a blob from the internal blob store to a path on the node's filesystem. - /// - /// `destination` should be an writeable, absolute path on the local node's filesystem. - /// - /// If `format` is set to [`ExportFormat::Collection`], and the `hash` refers to a collection, - /// all children of the collection will be exported. See [`ExportFormat`] for details. - /// - /// The `mode` argument defines if the blob should be copied to the target location or moved out of - /// the internal store into the target location. See [`ExportMode`] for details. - pub async fn export( - &self, - hash: Hash, - destination: PathBuf, - format: ExportFormat, - mode: ExportMode, - ) -> Result { - let req = ExportRequest { - hash, - path: destination, - format, - mode, - }; - let stream = self.rpc.server_streaming(req).await?; - Ok(ExportProgress::new( - stream.map(|r| r.map_err(anyhow::Error::from)), - )) - } - - /// List all complete blobs. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest).await?; - Ok(flatten(stream)) - } - - /// List all incomplete (partial) blobs. - pub async fn list_incomplete(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListIncompleteRequest).await?; - Ok(flatten(stream)) - } - - /// Read the content of a collection. - pub async fn get_collection(&self, hash: Hash) -> Result { - Collection::load(hash, self).await - } - - /// List all collections. - pub fn list_collections(&self) -> Result>> { - let this = self.clone(); - Ok(Gen::new(|co| async move { - if let Err(cause) = this.list_collections_impl(&co).await { - co.yield_(Err(cause)).await; - } - })) - } - - async fn list_collections_impl(&self, co: &Co>) -> Result<()> { - let tags = self.tags_client(); - let mut tags = tags.list_hash_seq().await?; - while let Some(tag) = tags.next().await { - let tag = tag?; - if let Ok(collection) = self.get_collection(tag.hash).await { - let info = CollectionInfo { - tag: tag.name, - hash: tag.hash, - total_blobs_count: Some(collection.len() as u64 + 1), - total_blobs_size: Some(0), - }; - co.yield_(Ok(info)).await; - } - } - Ok(()) - } - - /// Delete a blob. - /// - /// **Warning**: this operation deletes the blob from the local store even - /// if it is tagged. You should usually not do this manually, but rely on the - /// node to remove data that is not tagged. - pub async fn delete_blob(&self, hash: Hash) -> Result<()> { - self.rpc.rpc(DeleteRequest { hash }).await??; - Ok(()) - } - - /// Share a blob. - pub async fn share( - &self, - hash: Hash, - blob_format: BlobFormat, - addr_options: AddrInfoOptions, - ) -> Result { - let mut addr = self.rpc.rpc(StatusRequest).await??.addr; - addr.apply_options(addr_options); - let ticket = BlobTicket::new(addr, hash, blob_format).expect("correct ticket"); - - Ok(ticket) - } - - fn tags_client(&self) -> tags::Client { - tags::Client { - rpc: self.rpc.clone(), - } - } -} - -impl SimpleStore for Client { - async fn load(&self, hash: Hash) -> anyhow::Result { - self.read_to_bytes(hash).await - } -} - -/// Defines the way to read bytes. -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy)] -pub enum ReadAtLen { - /// Reads all available bytes. - #[default] - All, - /// Reads exactly this many bytes, erroring out on larger or smaller. - Exact(u64), - /// Reads at most this many bytes. - AtMost(u64), -} - -impl ReadAtLen { - pub(crate) fn as_result_len(&self, size_remaining: u64) -> u64 { - match self { - ReadAtLen::All => size_remaining, - ReadAtLen::Exact(len) => *len, - ReadAtLen::AtMost(len) => std::cmp::min(*len, size_remaining), - } - } -} - -/// Whether to wrap the added data in a collection. -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub enum WrapOption { - /// Do not wrap the file or directory. - #[default] - NoWrap, - /// Wrap the file or directory in a collection. - Wrap { - /// Override the filename in the wrapping collection. - name: Option, - }, -} - -/// Status information about a blob. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum BlobStatus { - /// The blob is not stored at all. - NotFound, - /// The blob is only stored partially. - Partial { - /// The size of the currently stored partial blob. - size: BaoBlobSize, - }, - /// The blob is stored completely. - Complete { - /// The size of the blob. - size: u64, - }, -} - -/// Outcome of a blob add operation. -#[derive(Debug, Clone)] -pub struct AddOutcome { - /// The hash of the blob - pub hash: Hash, - /// The format the blob - pub format: BlobFormat, - /// The size of the blob - pub size: u64, - /// The tag of the blob - pub tag: Tag, -} - -/// Information about a stored collection. -#[derive(Debug, Serialize, Deserialize)] -pub struct CollectionInfo { - /// Tag of the collection - pub tag: Tag, - - /// Hash of the collection - pub hash: Hash, - /// Number of children in the collection - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_count: Option, - /// Total size of the raw data referred to by all links - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_size: Option, -} - -/// Information about a complete blob. -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobInfo { - /// Location of the blob - pub path: String, - /// The hash of the blob - pub hash: Hash, - /// The size of the blob - pub size: u64, -} - -/// Information about an incomplete blob. -#[derive(Debug, Serialize, Deserialize)] -pub struct IncompleteBlobInfo { - /// The size we got - pub size: u64, - /// The size we expect - pub expected_size: u64, - /// The hash of the blob - pub hash: Hash, -} - -/// Progress stream for blob add operations. -#[derive(derive_more::Debug)] -pub struct AddProgress { - #[debug(skip)] - stream: Pin< - Box> + Send + Unpin + 'static>, - >, - current_total_size: Arc, -} - -impl AddProgress { - fn new( - stream: (impl Stream< - Item = Result, impl Into>, - > + Send - + Unpin - + 'static), - ) -> Self { - let current_total_size = Arc::new(AtomicU64::new(0)); - let total_size = current_total_size.clone(); - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - if let iroh_blobs::provider::AddProgress::Found { size, .. } = &item { - total_size.fetch_add(*size, Ordering::Relaxed); - } - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_total_size, - } - } - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`AddOutcome`] which contains a tag, format, hash and a size. - /// When importing a single blob, this is the hash and size of that blob. - /// When importing a collection, the hash is the hash of the collection and the size - /// is the total size of all imported blobs (but excluding the size of the collection blob - /// itself). - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for AddProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for AddProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - iroh_blobs::provider::AddProgress::AllDone { hash, format, tag } => { - let outcome = AddOutcome { - hash, - format, - tag, - size: self.current_total_size.load(Ordering::Relaxed), - }; - return Poll::Ready(Ok(outcome)); - } - iroh_blobs::provider::AddProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Outcome of a blob download operation. -#[derive(Debug, Clone)] -pub struct DownloadOutcome { - /// The size of the data we already had locally - pub local_size: u64, - /// The size of the data we downloaded from the network - pub downloaded_size: u64, - /// Statistics about the download - pub stats: iroh_blobs::get::Stats, -} - -/// Progress stream for blob download operations. -#[derive(derive_more::Debug)] -pub struct DownloadProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, - current_local_size: Arc, - current_network_size: Arc, -} - -impl DownloadProgress { - /// Create a [`DownloadProgress`] that can help you easily poll the [`BytesDownloadProgress`] stream from your download until it is finished or errors. - pub fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let current_local_size = Arc::new(AtomicU64::new(0)); - let current_network_size = Arc::new(AtomicU64::new(0)); - - let local_size = current_local_size.clone(); - let network_size = current_network_size.clone(); - - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - match &item { - BytesDownloadProgress::FoundLocal { size, .. } => { - local_size.fetch_add(size.value(), Ordering::Relaxed); - } - BytesDownloadProgress::Found { size, .. } => { - network_size.fetch_add(*size, Ordering::Relaxed); - } - _ => {} - } - - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_local_size, - current_network_size, - } - } - - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`DownloadOutcome`] which contains the size of the content we downloaded and the size of the content we already had locally. - /// When importing a single blob, this is the size of that blob. - /// When importing a collection, this is the total size of all imported blobs (but excluding the size of the collection blob itself). - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for DownloadProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for DownloadProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - BytesDownloadProgress::AllDone(stats) => { - let outcome = DownloadOutcome { - local_size: self.current_local_size.load(Ordering::Relaxed), - downloaded_size: self.current_network_size.load(Ordering::Relaxed), - stats, - }; - return Poll::Ready(Ok(outcome)); - } - BytesDownloadProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Outcome of a blob export operation. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ExportOutcome { - /// The total size of the exported data. - total_size: u64, -} - -/// Progress stream for blob export operations. -#[derive(derive_more::Debug)] -pub struct ExportProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, - current_total_size: Arc, -} - -impl ExportProgress { - /// Create a [`ExportProgress`] that can help you easily poll the [`BytesExportProgress`] stream from your - /// download until it is finished or errors. - pub fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let current_total_size = Arc::new(AtomicU64::new(0)); - let total_size = current_total_size.clone(); - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - if let BytesExportProgress::Found { size, .. } = &item { - let size = size.value(); - total_size.fetch_add(size, Ordering::Relaxed); - } - - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_total_size, - } - } - - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`ExportOutcome`] which contains the size of the content we exported. - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for ExportProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for ExportProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - BytesExportProgress::AllDone => { - let outcome = ExportOutcome { - total_size: self.current_total_size.load(Ordering::Relaxed), - }; - return Poll::Ready(Ok(outcome)); - } - BytesExportProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Data reader for a single blob. -/// -/// Implements [`AsyncRead`]. -#[derive(derive_more::Debug)] -pub struct Reader { - size: u64, - response_size: u64, - is_complete: bool, - #[debug("StreamReader")] - stream: tokio_util::io::StreamReader>, Bytes>, -} - -impl Reader { - fn new( - size: u64, - response_size: u64, - is_complete: bool, - stream: BoxStreamSync<'static, io::Result>, - ) -> Self { - Self { - size, - response_size, - is_complete, - stream: StreamReader::new(stream), - } - } - - pub(crate) async fn from_rpc_read(rpc: &RpcClient, hash: Hash) -> anyhow::Result { - Self::from_rpc_read_at(rpc, hash, 0, ReadAtLen::All).await - } - - async fn from_rpc_read_at( - rpc: &RpcClient, - hash: Hash, - offset: u64, - len: ReadAtLen, - ) -> anyhow::Result { - let stream = rpc - .server_streaming(ReadAtRequest { hash, offset, len }) - .await?; - let mut stream = flatten(stream); - - let (size, is_complete) = match stream.next().await { - Some(Ok(ReadAtResponse::Entry { size, is_complete })) => (size, is_complete), - Some(Err(err)) => return Err(err), - Some(Ok(_)) => return Err(anyhow!("Expected header frame, but got data frame")), - None => return Err(anyhow!("Expected header frame, but RPC stream was dropped")), - }; - - let stream = stream.map(|item| match item { - Ok(ReadAtResponse::Data { chunk }) => Ok(chunk), - Ok(_) => Err(io::Error::new(io::ErrorKind::Other, "Expected data frame")), - Err(err) => Err(io::Error::new(io::ErrorKind::Other, format!("{err}"))), - }); - let len = len.as_result_len(size.value() - offset); - Ok(Self::new(size.value(), len, is_complete, Box::pin(stream))) - } - - /// Total size of this blob. - pub fn size(&self) -> u64 { - self.size - } - - /// Whether this blob has been downloaded completely. - /// - /// Returns false for partial blobs for which some chunks are missing. - pub fn is_complete(&self) -> bool { - self.is_complete - } - - /// Read all bytes of the blob. - pub async fn read_to_bytes(&mut self) -> anyhow::Result { - let mut buf = Vec::with_capacity(self.response_size as usize); - self.read_to_end(&mut buf).await?; - Ok(buf.into()) - } -} - -impl AsyncRead for Reader { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.stream).poll_read(cx, buf) - } -} - -impl Stream for Reader { - type Item = io::Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).get_pin_mut().poll_next(cx) - } - - fn size_hint(&self) -> (usize, Option) { - self.stream.get_ref().size_hint() - } -} - -/// Options to configure a download request. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DownloadOptions { - /// The format of the data to download. - pub format: BlobFormat, - /// Source nodes to download from. - /// - /// If set to more than a single node, they will all be tried. If `mode` is set to - /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds. - /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel, - /// if the concurrency limits permit. - pub nodes: Vec, - /// Optional tag to tag the data with. - pub tag: SetTagOption, - /// Whether to directly start the download or add it to the download queue. - pub mode: DownloadMode, -} - -/// Set the mode for whether to directly start the download or add it to the download queue. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum DownloadMode { - /// Start the download right away. - /// - /// No concurrency limits or queuing will be applied. It is up to the user to manage download - /// concurrency. - Direct, - /// Queue the download. - /// - /// The download queue will be processed in-order, while respecting the downloader concurrency limits. - Queued, -} - -#[cfg(test)] -mod tests { - use iroh_blobs::hashseq::HashSeq; - use iroh_net::NodeId; - use rand::RngCore; - use testresult::TestResult; - use tokio::{io::AsyncWriteExt, sync::mpsc}; - - use super::*; - - #[tokio::test] - async fn test_blob_create_collection() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let mut paths = Vec::new(); - for i in 0..5 { - let path = in_root.join(format!("test-{i}")); - let size = 100; - let mut buf = vec![0u8; size]; - rand::thread_rng().fill_bytes(&mut buf); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - paths.push(path); - } - - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - // import files - for path in &paths { - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - collection.push( - path.file_name().unwrap().to_str().unwrap().to_string(), - import_outcome.hash, - ); - tags.push(import_outcome.tag); - } - - let (hash, tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - let collections: Vec<_> = client.blobs().list_collections()?.try_collect().await?; - - assert_eq!(collections.len(), 1); - { - let CollectionInfo { - tag, - hash, - total_blobs_count, - .. - } = &collections[0]; - assert_eq!(tag, tag); - assert_eq!(hash, hash); - // 5 blobs + 1 meta - assert_eq!(total_blobs_count, &Some(5 + 1)); - } - - // check that "temp" tags have been deleted - let tags: Vec<_> = client.tags().list().await?.try_collect().await?; - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].hash, hash); - assert_eq!(tags[0].name, tag); - assert_eq!(tags[0].format, BlobFormat::HashSeq); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_read_at() -> Result<()> { - // let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let path = in_root.join("test-blob"); - let size = 1024 * 128; - let buf: Vec = (0..size).map(|i| i as u8).collect(); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - - let client = node.client(); - - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - let hash = import_outcome.hash; - - // Read everything - let res = client.blobs().read_to_bytes(hash).await?; - assert_eq!(&res, &buf[..]); - - // Read at smaller than blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(100)) - .await?; - assert_eq!(res.len(), 100); - assert_eq!(&res[..], &buf[0..100]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(120)) - .await?; - assert_eq!(res.len(), 120); - assert_eq!(&res[..], &buf[20..140]); - - // Read at equal to blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(1024 * 64)) - .await?; - assert_eq!(res.len(), 1024 * 64); - assert_eq!(&res[..], &buf[0..1024 * 64]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(1024 * 64)) - .await?; - assert_eq!(res.len(), 1024 * 64); - assert_eq!(&res[..], &buf[20..(20 + 1024 * 64)]); - - // Read at larger than blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(10 + 1024 * 64)) - .await?; - assert_eq!(res.len(), 10 + 1024 * 64); - assert_eq!(&res[..], &buf[0..(10 + 1024 * 64)]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(10 + 1024 * 64)) - .await?; - assert_eq!(res.len(), 10 + 1024 * 64); - assert_eq!(&res[..], &buf[20..(20 + 10 + 1024 * 64)]); - - // full length - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::All) - .await?; - assert_eq!(res.len(), 1024 * 128 - 20); - assert_eq!(&res[..], &buf[20..]); - - // size should be total - let reader = client - .blobs() - .read_at(hash, 0, ReadAtLen::Exact(20)) - .await?; - assert_eq!(reader.size(), 1024 * 128); - assert_eq!(reader.response_size, 20); - - // last chunk - exact - let res = client - .blobs() - .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::Exact(1024)) - .await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // last chunk - open - let res = client - .blobs() - .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::All) - .await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // last chunk - larger - let mut res = client - .blobs() - .read_at(hash, 1024 * 127, ReadAtLen::AtMost(2048)) - .await?; - assert_eq!(res.size, 1024 * 128); - assert_eq!(res.response_size, 1024); - let res = res.read_to_bytes().await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // out of bounds - too long - let res = client - .blobs() - .read_at(hash, 0, ReadAtLen::Exact(1024 * 128 + 1)) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of bound")); - - // out of bounds - offset larger than blob - let res = client - .blobs() - .read_at(hash, 1024 * 128 + 1, ReadAtLen::All) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of range")); - - // out of bounds - offset + length too large - let res = client - .blobs() - .read_at(hash, 1024 * 127, ReadAtLen::Exact(1025)) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of bound")); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_get_collection() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let mut paths = Vec::new(); - for i in 0..5 { - let path = in_root.join(format!("test-{i}")); - let size = 100; - let mut buf = vec![0u8; size]; - rand::thread_rng().fill_bytes(&mut buf); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - paths.push(path); - } - - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - // import files - for path in &paths { - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - collection.push( - path.file_name().unwrap().to_str().unwrap().to_string(), - import_outcome.hash, - ); - tags.push(import_outcome.tag); - } - - let (hash, _tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - let collection = client.blobs().get_collection(hash).await?; - - // 5 blobs - assert_eq!(collection.len(), 5); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_share() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let path = in_root.join("test-blob"); - let size = 1024 * 128; - let buf: Vec = (0..size).map(|i| i as u8).collect(); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - - let client = node.client(); - - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - let ticket = client - .blobs() - .share(import_outcome.hash, BlobFormat::Raw, Default::default()) - .await?; - assert_eq!(ticket.hash(), import_outcome.hash); - - let status = client.blobs().status(import_outcome.hash).await?; - assert_eq!(status, BlobStatus::Complete { size }); - - Ok(()) - } - - #[derive(Debug, Clone)] - struct BlobEvents { - sender: mpsc::Sender, - } - - impl BlobEvents { - fn new(cap: usize) -> (Self, mpsc::Receiver) { - let (s, r) = mpsc::channel(cap); - (Self { sender: s }, r) - } - } - - impl iroh_blobs::provider::CustomEventSender for BlobEvents { - fn send(&self, event: iroh_blobs::provider::Event) -> futures_lite::future::Boxed<()> { - let sender = self.sender.clone(); - Box::pin(async move { - sender.send(event).await.ok(); - }) - } - - fn try_send(&self, event: iroh_blobs::provider::Event) { - self.sender.try_send(event).ok(); - } - } - - #[tokio::test] - async fn test_blob_provide_events() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let (node1_events, mut node1_events_r) = BlobEvents::new(16); - let node1 = crate::node::Node::memory() - .blobs_events(node1_events) - .spawn() - .await?; - - let (node2_events, mut node2_events_r) = BlobEvents::new(16); - let node2 = crate::node::Node::memory() - .blobs_events(node2_events) - .spawn() - .await?; - - let import_outcome = node1.blobs().add_bytes(&b"hello world"[..]).await?; - - // Download in node2 - let node1_addr = node1.net().node_addr().await?; - let res = node2 - .blobs() - .download(import_outcome.hash, node1_addr) - .await? - .await?; - dbg!(&res); - assert_eq!(res.local_size, 0); - assert_eq!(res.downloaded_size, 11); - - node1.shutdown().await?; - node2.shutdown().await?; - - let mut ev1 = Vec::new(); - while let Some(ev) = node1_events_r.recv().await { - ev1.push(ev); - } - // assert_eq!(ev1.len(), 3); - assert!(matches!( - ev1[0], - iroh_blobs::provider::Event::ClientConnected { .. } - )); - assert!(matches!( - ev1[1], - iroh_blobs::provider::Event::GetRequestReceived { .. } - )); - assert!(matches!( - ev1[2], - iroh_blobs::provider::Event::TransferProgress { .. } - )); - assert!(matches!( - ev1[3], - iroh_blobs::provider::Event::TransferCompleted { .. } - )); - dbg!(&ev1); - - let mut ev2 = Vec::new(); - while let Some(ev) = node2_events_r.recv().await { - ev2.push(ev); - } - - // Node 2 did not provide anything - assert!(ev2.is_empty()); - Ok(()) - } - /// Download a existing blob from oneself - #[tokio::test] - async fn test_blob_get_self_existing() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - let node_id = node.node_id(); - let client = node.client(); - - let AddOutcome { hash, size, .. } = client.blobs().add_bytes("foo").await?; - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - Ok(()) - } - - /// Download a missing blob from oneself - #[tokio::test] - async fn test_blob_get_self_missing() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - let node_id = node.node_id(); - let client = node.client(); - - let hash = Hash::from_bytes([0u8; 32]); - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await; - assert!(res.is_err()); - assert_eq!( - res.err().unwrap().to_string().as_str(), - "No nodes to download from provided" - ); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await; - assert!(res.is_err()); - assert_eq!( - res.err().unwrap().to_string().as_str(), - "No provider nodes found" - ); - - Ok(()) - } - - /// Download a existing collection. Check that things succeed and no download is performed. - #[tokio::test] - async fn test_blob_get_existing_collection() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - // We use a nonexisting node id because we just want to check that this succeeds without - // hitting the network. - let node_id = NodeId::from_bytes(&[0u8; 32])?; - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - let mut size = 0; - for value in ["iroh", "is", "cool"] { - let import_outcome = client.blobs().add_bytes(value).await.context("add bytes")?; - collection.push(value.to_string(), import_outcome.hash); - tags.push(import_outcome.tag); - size += import_outcome.size; - } - - let (hash, _tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - // load the hashseq and collection header manually to calculate our expected size - let hashseq_bytes = client.blobs().read_to_bytes(hash).await?; - size += hashseq_bytes.len() as u64; - let hashseq = HashSeq::try_from(hashseq_bytes)?; - let collection_header_bytes = client - .blobs() - .read_to_bytes(hashseq.into_iter().next().expect("header to exist")) - .await?; - size += collection_header_bytes.len() as u64; - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await - .context("direct (download)")?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await - .context("queued")?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - Ok(()) - } - - #[tokio::test] - #[cfg_attr(target_os = "windows", ignore = "flaky")] - async fn test_blob_delete_mem() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - let res = node.blobs().add_bytes(&b"hello world"[..]).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert_eq!(hashes.len(), 1); - assert_eq!(hashes[0].hash, res.hash); - - // delete - node.blobs().delete_blob(res.hash).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert!(hashes.is_empty()); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_delete_fs() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let dir = tempfile::tempdir()?; - let node = crate::node::Node::persistent(dir.path()) - .await? - .spawn() - .await?; - - let res = node.blobs().add_bytes(&b"hello world"[..]).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert_eq!(hashes.len(), 1); - assert_eq!(hashes[0].hash, res.hash); - - // delete - node.blobs().delete_blob(res.hash).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert!(hashes.is_empty()); - - Ok(()) - } -} diff --git a/iroh/src/client/blobs/batch.rs b/iroh/src/client/blobs/batch.rs deleted file mode 100644 index 4c72fc3a241..00000000000 --- a/iroh/src/client/blobs/batch.rs +++ /dev/null @@ -1,462 +0,0 @@ -use std::{ - io, - path::PathBuf, - sync::{Arc, Mutex}, -}; - -use anyhow::{anyhow, Context, Result}; -use bytes::Bytes; -use futures_buffered::BufferedStreamExt; -use futures_lite::StreamExt; -use futures_util::{sink::Buffer, FutureExt, SinkExt, Stream}; -use iroh_blobs::{ - format::collection::Collection, - provider::BatchAddPathProgress, - store::ImportMode, - util::{SetTagOption, TagDrop}, - BlobFormat, HashAndFormat, Tag, TempTag, -}; -use quic_rpc::client::UpdateSink; -use tokio::io::AsyncRead; -use tokio_util::io::ReaderStream; -use tracing::{debug, warn}; - -use super::WrapOption; -use crate::{ - client::{RpcClient, RpcConnection, RpcService}, - rpc_protocol::{ - blobs::{ - BatchAddPathRequest, BatchAddStreamRequest, BatchAddStreamResponse, - BatchAddStreamUpdate, BatchCreateTempTagRequest, BatchId, BatchUpdate, - }, - tags::{self, SyncMode}, - }, -}; - -/// A scope in which blobs can be added. -#[derive(derive_more::Debug)] -struct BatchInner { - /// The id of the scope. - batch: BatchId, - /// The rpc client. - rpc: RpcClient, - /// The stream to send drop - #[debug(skip)] - updates: Mutex, BatchUpdate>>, -} - -/// A batch for write operations. -/// -/// This serves mostly as a scope for temporary tags. -/// -/// It is not a transaction, so things in a batch are not atomic. Also, there is -/// no isolation between batches. -#[derive(derive_more::Debug)] -pub struct Batch(Arc); - -impl TagDrop for BatchInner { - fn on_drop(&self, content: &HashAndFormat) { - let mut updates = self.updates.lock().unwrap(); - // make a spirited attempt to notify the server that we are dropping the content - // - // this will occasionally fail, but that's acceptable. The temp tags for the batch - // will be cleaned up as soon as the entire batch is dropped. - // - // E.g. a typical scenario is that you create a large array of temp tags, and then - // store them in a hash sequence and then drop the array. You will get many drops - // at the same time, and might get a send failure here. - // - // But that just means that the server will clean up the temp tags when the batch is - // dropped. - updates.feed(BatchUpdate::Drop(*content)).now_or_never(); - updates.flush().now_or_never(); - } -} - -/// Options for adding a file as a blob -#[derive(Debug, Clone, Copy, Default)] -pub struct AddFileOpts { - /// The import mode - pub import_mode: ImportMode, - /// The format of the blob - pub format: BlobFormat, -} - -/// Options for adding a directory as a collection -#[derive(Debug, Clone)] -pub struct AddDirOpts { - /// The import mode - pub import_mode: ImportMode, - /// Whether to preserve the directory name - pub wrap: WrapOption, - /// Io parallelism - pub io_parallelism: usize, -} - -impl Default for AddDirOpts { - fn default() -> Self { - Self { - import_mode: ImportMode::TryReference, - wrap: WrapOption::NoWrap, - io_parallelism: 4, - } - } -} - -/// Options for adding a directory as a collection -#[derive(Debug, Clone)] -pub struct AddReaderOpts { - /// The format of the blob - pub format: BlobFormat, - /// Size of the chunks to send - pub chunk_size: usize, -} - -impl Default for AddReaderOpts { - fn default() -> Self { - Self { - format: BlobFormat::Raw, - chunk_size: 1024 * 64, - } - } -} - -impl Batch { - pub(super) fn new( - batch: BatchId, - rpc: RpcClient, - updates: UpdateSink, - buffer_size: usize, - ) -> Self { - let updates = updates.buffer(buffer_size); - Self(Arc::new(BatchInner { - batch, - rpc, - updates: updates.into(), - })) - } - - /// Write a blob by passing bytes. - pub async fn add_bytes(&self, bytes: impl Into) -> Result { - self.add_bytes_with_opts(bytes, Default::default()).await - } - - /// Import a blob from a filesystem path, using the default options. - /// - /// For more control, use [`Self::add_file_with_opts`]. - pub async fn add_file(&self, path: PathBuf) -> Result<(TempTag, u64)> { - self.add_file_with_opts(path, AddFileOpts::default()).await - } - - /// Add a directory as a hashseq in iroh collection format - pub async fn add_dir(&self, root: PathBuf) -> Result { - self.add_dir_with_opts(root, Default::default()).await - } - - /// Write a blob by passing an async reader. - /// - /// This will consume the stream in 64KB chunks, and use a format of [BlobFormat::Raw]. - /// - /// For more options, see [`Self::add_reader_with_opts`]. - pub async fn add_reader( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - ) -> anyhow::Result { - self.add_reader_with_opts(reader, Default::default()).await - } - - /// Write a blob by passing a stream of bytes. - pub async fn add_stream( - &self, - input: impl Stream> + Send + Unpin + 'static, - ) -> Result { - self.add_stream_with_opts(input, Default::default()).await - } - - /// Creates a temp tag to protect some content (blob or hashseq) from being deleted. - /// - /// This is a lower-level API. The other functions in [`Batch`] already create [`TempTag`]s automatically. - /// - /// [`TempTag`]s allow you to protect some data from deletion while a download is ongoing, - /// even if you don't want to protect it permanently. - pub async fn temp_tag(&self, content: HashAndFormat) -> Result { - // Notify the server that we want one temp tag for the given content - self.0 - .rpc - .rpc(BatchCreateTempTagRequest { - batch: self.0.batch, - content, - }) - .await??; - // Only after success of the above call, we can create the corresponding local temp tag - Ok(self.local_temp_tag(content, None)) - } - - /// Write a blob by passing an async reader. - /// - /// This consumes the stream in chunks using `opts.chunk_size`. A good default is 64KB. - pub async fn add_reader_with_opts( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - opts: AddReaderOpts, - ) -> anyhow::Result { - let AddReaderOpts { format, chunk_size } = opts; - let input = ReaderStream::with_capacity(reader, chunk_size); - self.add_stream_with_opts(input, format).await - } - - /// Write a blob by passing bytes. - pub async fn add_bytes_with_opts( - &self, - bytes: impl Into, - format: BlobFormat, - ) -> Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream_with_opts(input, format).await - } - - /// Import a blob from a filesystem path. - /// - /// `path` should be an absolute path valid for the file system on which - /// the node runs, which refers to a file. - /// - /// If you use [`ImportMode::TryReference`], Iroh will assume that the data will not - /// change and will share it in place without copying to the Iroh data directory - /// if appropriate. However, for tiny files, Iroh will copy the data. - /// - /// If you use [`ImportMode::Copy`], Iroh will always copy the data. - /// - /// Will return a temp tag for the added blob, as well as the size of the file. - pub async fn add_file_with_opts( - &self, - path: PathBuf, - opts: AddFileOpts, - ) -> Result<(TempTag, u64)> { - let AddFileOpts { - import_mode, - format, - } = opts; - anyhow::ensure!( - path.is_absolute(), - "Path must be absolute, but got: {:?}", - path - ); - anyhow::ensure!(path.is_file(), "Path does not refer to a file: {:?}", path); - let mut stream = self - .0 - .rpc - .server_streaming(BatchAddPathRequest { - path, - import_mode, - format, - batch: self.0.batch, - }) - .await?; - let mut res_hash = None; - let mut res_size = None; - while let Some(item) = stream.next().await { - match item?.0 { - BatchAddPathProgress::Abort(cause) => { - Err(cause)?; - } - BatchAddPathProgress::Done { hash } => { - res_hash = Some(hash); - } - BatchAddPathProgress::Found { size } => { - res_size = Some(size); - } - _ => {} - } - } - let hash = res_hash.context("Missing hash")?; - let size = res_size.context("Missing size")?; - Ok(( - self.local_temp_tag(HashAndFormat { hash, format }, Some(size)), - size, - )) - } - - /// Add a directory as a hashseq in iroh collection format - /// - /// This can also be used to add a single file as a collection, if - /// wrap is set to [WrapOption::Wrap]. - /// - /// However, if you want to add a single file as a raw blob, use add_file instead. - pub async fn add_dir_with_opts(&self, root: PathBuf, opts: AddDirOpts) -> Result { - let AddDirOpts { - import_mode, - wrap, - io_parallelism, - } = opts; - anyhow::ensure!(root.is_absolute(), "Path must be absolute"); - - // let (send, recv) = flume::bounded(32); - // let import_progress = FlumeProgressSender::new(send); - - // import all files below root recursively - let data_sources = crate::util::fs::scan_path(root, wrap)?; - let opts = AddFileOpts { - import_mode, - format: BlobFormat::Raw, - }; - let result: Vec<_> = futures_lite::stream::iter(data_sources) - .map(|source| { - // let import_progress = import_progress.clone(); - async move { - let name = source.name().to_string(); - let (tag, size) = self - .add_file_with_opts(source.path().to_owned(), opts) - .await?; - let hash = *tag.hash(); - anyhow::Ok((name, hash, size, tag)) - } - }) - .buffered_ordered(io_parallelism) - .try_collect() - .await?; - - // create a collection - let (collection, child_tags): (Collection, Vec<_>) = result - .into_iter() - .map(|(name, hash, _, tag)| ((name, hash), tag)) - .unzip(); - - let tag = self.add_collection(collection).await?; - drop(child_tags); - Ok(tag) - } - - /// Write a blob by passing a stream of bytes. - /// - /// For convenient interop with common sources of data, this function takes a stream of `io::Result`. - /// If you have raw bytes, you need to wrap them in `io::Result::Ok`. - pub async fn add_stream_with_opts( - &self, - mut input: impl Stream> + Send + Unpin + 'static, - format: BlobFormat, - ) -> Result { - let (mut sink, mut stream) = self - .0 - .rpc - .bidi(BatchAddStreamRequest { - batch: self.0.batch, - format, - }) - .await?; - let mut size = 0u64; - while let Some(item) = input.next().await { - match item { - Ok(chunk) => { - size += chunk.len() as u64; - sink.send(BatchAddStreamUpdate::Chunk(chunk)) - .await - .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?; - } - Err(err) => { - warn!("Abort send, reason: failed to read from source stream: {err:?}"); - sink.send(BatchAddStreamUpdate::Abort) - .await - .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?; - break; - } - } - } - // this is needed for the remote to notice that the stream is closed - drop(sink); - let mut res = None; - while let Some(item) = stream.next().await { - match item? { - BatchAddStreamResponse::Abort(cause) => { - Err(cause)?; - } - BatchAddStreamResponse::Result { hash } => { - res = Some(hash); - } - _ => {} - } - } - let hash = res.context("Missing answer")?; - Ok(self.local_temp_tag(HashAndFormat { hash, format }, Some(size))) - } - - /// Add a collection. - /// - /// This is a convenience function that converts the collection into two blobs - /// (the metadata and the hash sequence) and adds them, returning a temp tag for - /// the hash sequence. - /// - /// Note that this does not guarantee that the data that the collection refers to - /// actually exists. It will just create 2 blobs, the metadata and the hash sequence - /// itself. - pub async fn add_collection(&self, collection: Collection) -> Result { - self.add_blob_seq(collection.to_blobs()).await - } - - /// Add a sequence of blobs, where the last is a hash sequence. - /// - /// It is a common pattern in iroh to have a hash sequence with one or more - /// blobs of metadata, and the remaining blobs being the actual data. E.g. - /// a collection is a hash sequence where the first child is the metadata. - pub async fn add_blob_seq(&self, iter: impl Iterator) -> Result { - let mut blobs = iter.peekable(); - // put the tags somewhere - let mut tags = vec![]; - loop { - let blob = blobs.next().context("Failed to get next blob")?; - if blobs.peek().is_none() { - return self.add_bytes_with_opts(blob, BlobFormat::HashSeq).await; - } else { - tags.push(self.add_bytes(blob).await?); - } - } - } - - /// Upgrades a temp tag to a persistent tag. - pub async fn persist(&self, tt: TempTag) -> Result { - let tag = self - .0 - .rpc - .rpc(tags::CreateRequest { - value: tt.hash_and_format(), - batch: Some(self.0.batch), - sync: SyncMode::Full, - }) - .await??; - Ok(tag) - } - - /// Upgrades a temp tag to a persistent tag with a specific name. - pub async fn persist_to(&self, tt: TempTag, tag: Tag) -> Result<()> { - self.0 - .rpc - .rpc(tags::SetRequest { - name: tag, - value: Some(tt.hash_and_format()), - batch: Some(self.0.batch), - sync: SyncMode::Full, - }) - .await??; - Ok(()) - } - - /// Upgrades a temp tag to a persistent tag with either a specific name or - /// an automatically generated name. - pub async fn persist_with_opts(&self, tt: TempTag, opts: SetTagOption) -> Result { - match opts { - SetTagOption::Auto => self.persist(tt).await, - SetTagOption::Named(tag) => { - self.persist_to(tt, tag.clone()).await?; - Ok(tag) - } - } - } - - /// Creates a temp tag for the given hash and format, without notifying the server. - /// - /// Caution: only do this for data for which you know the server side has created a temp tag. - fn local_temp_tag(&self, inner: HashAndFormat, _size: Option) -> TempTag { - let on_drop: Arc = self.0.clone(); - let on_drop = Some(Arc::downgrade(&on_drop)); - TempTag::new(inner, on_drop) - } -} diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs deleted file mode 100644 index 7935b5b3b9f..00000000000 --- a/iroh/src/client/docs.rs +++ /dev/null @@ -1,879 +0,0 @@ -//! API for document management. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::docs()`](crate::client::Iroh::docs). - -use std::{ - path::{Path, PathBuf}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use derive_more::{Display, FromStr}; -use futures_lite::{Stream, StreamExt}; -use iroh_base::{key::PublicKey, node_addr::AddrInfoOptions, rpc::RpcError}; -use iroh_blobs::{export::ExportProgress, store::ExportMode, Hash}; -#[doc(inline)] -pub use iroh_docs::engine::{Origin, SyncEvent, SyncReason}; -use iroh_docs::{ - actor::OpenState, - store::{DownloadPolicy, Query}, - AuthorId, Capability, CapabilityKind, ContentStatus, DocTicket, NamespaceId, PeerIdBytes, - RecordIdentifier, -}; -use iroh_net::NodeAddr; -use portable_atomic::{AtomicBool, Ordering}; -use quic_rpc::message::RpcMsg; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; - -use super::{blobs, flatten, RpcClient}; -use crate::rpc_protocol::{ - docs::{ - CloseRequest, CreateRequest, DelRequest, DelResponse, DocListRequest, DocSubscribeRequest, - DropRequest, ExportFileRequest, GetDownloadPolicyRequest, GetExactRequest, GetManyRequest, - GetSyncPeersRequest, ImportFileRequest, ImportRequest, LeaveRequest, OpenRequest, - SetDownloadPolicyRequest, SetHashRequest, SetRequest, ShareRequest, StartSyncRequest, - StatusRequest, - }, - RpcService, -}; - -/// Iroh docs client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl Client { - /// Creates a new document. - pub async fn create(&self) -> Result { - let res = self.rpc.rpc(CreateRequest {}).await??; - let doc = Doc::new(self.rpc.clone(), res.id); - Ok(doc) - } - - /// Deletes a document from the local node. - /// - /// This is a destructive operation. Both the document secret key and all entries in the - /// document will be permanently deleted from the node's storage. Content blobs will be deleted - /// through garbage collection unless they are referenced from another document or tag. - pub async fn drop_doc(&self, doc_id: NamespaceId) -> Result<()> { - self.rpc.rpc(DropRequest { doc_id }).await??; - Ok(()) - } - - /// Imports a document from a namespace capability. - /// - /// This does not start sync automatically. Use [`Doc::start_sync`] to start sync. - pub async fn import_namespace(&self, capability: Capability) -> Result { - let res = self.rpc.rpc(ImportRequest { capability }).await??; - let doc = Doc::new(self.rpc.clone(), res.doc_id); - Ok(doc) - } - - /// Imports a document from a ticket and joins all peers in the ticket. - pub async fn import(&self, ticket: DocTicket) -> Result { - let DocTicket { capability, nodes } = ticket; - let doc = self.import_namespace(capability).await?; - doc.start_sync(nodes).await?; - Ok(doc) - } - - /// Imports a document from a ticket, creates a subscription stream and joins all peers in the ticket. - /// - /// Returns the [`Doc`] and a [`Stream`] of [`LiveEvent`]s. - /// - /// The subscription stream is created before the sync is started, so the first call to this - /// method after starting the node is guaranteed to not miss any sync events. - pub async fn import_and_subscribe( - &self, - ticket: DocTicket, - ) -> Result<(Doc, impl Stream>)> { - let DocTicket { capability, nodes } = ticket; - let res = self.rpc.rpc(ImportRequest { capability }).await??; - let doc = Doc::new(self.rpc.clone(), res.doc_id); - let events = doc.subscribe().await?; - doc.start_sync(nodes).await?; - Ok((doc, events)) - } - - /// Lists all documents. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(DocListRequest {}).await?; - Ok(flatten(stream).map(|res| res.map(|res| (res.id, res.capability)))) - } - - /// Returns a [`Doc`] client for a single document. - /// - /// Returns None if the document cannot be found. - pub async fn open(&self, id: NamespaceId) -> Result> { - self.rpc.rpc(OpenRequest { doc_id: id }).await??; - let doc = Doc::new(self.rpc.clone(), id); - Ok(Some(doc)) - } -} - -/// Document handle -#[derive(Debug, Clone)] -pub struct Doc(Arc); - -impl PartialEq for Doc { - fn eq(&self, other: &Self) -> bool { - self.0.id == other.0.id - } -} - -impl Eq for Doc {} - -#[derive(Debug)] -struct DocInner { - id: NamespaceId, - rpc: RpcClient, - closed: AtomicBool, - rt: tokio::runtime::Handle, -} - -impl Drop for DocInner { - fn drop(&mut self) { - let doc_id = self.id; - let rpc = self.rpc.clone(); - if !self.closed.swap(true, Ordering::Relaxed) { - self.rt.spawn(async move { - rpc.rpc(CloseRequest { doc_id }).await.ok(); - }); - } - } -} - -impl Doc { - fn new(rpc: RpcClient, id: NamespaceId) -> Self { - Self(Arc::new(DocInner { - rpc, - id, - closed: AtomicBool::new(false), - rt: tokio::runtime::Handle::current(), - })) - } - - async fn rpc(&self, msg: M) -> Result - where - M: RpcMsg, - { - let res = self.0.rpc.rpc(msg).await?; - Ok(res) - } - - /// Returns the document id of this doc. - pub fn id(&self) -> NamespaceId { - self.0.id - } - - /// Closes the document. - pub async fn close(&self) -> Result<()> { - if !self.0.closed.swap(true, Ordering::Relaxed) { - self.rpc(CloseRequest { doc_id: self.id() }).await??; - } - Ok(()) - } - - fn ensure_open(&self) -> Result<()> { - if self.0.closed.load(Ordering::Relaxed) { - Err(anyhow!("document is closed")) - } else { - Ok(()) - } - } - - /// Sets the content of a key to a byte array. - pub async fn set_bytes( - &self, - author_id: AuthorId, - key: impl Into, - value: impl Into, - ) -> Result { - self.ensure_open()?; - let res = self - .rpc(SetRequest { - doc_id: self.id(), - author_id, - key: key.into(), - value: value.into(), - }) - .await??; - Ok(res.entry.content_hash()) - } - - /// Sets an entries on the doc via its key, hash, and size. - pub async fn set_hash( - &self, - author_id: AuthorId, - key: impl Into, - hash: Hash, - size: u64, - ) -> Result<()> { - self.ensure_open()?; - self.rpc(SetHashRequest { - doc_id: self.id(), - author_id, - key: key.into(), - hash, - size, - }) - .await??; - Ok(()) - } - - /// Adds an entry from an absolute file path - pub async fn import_file( - &self, - author: AuthorId, - key: Bytes, - path: impl AsRef, - in_place: bool, - ) -> Result { - self.ensure_open()?; - let stream = self - .0 - .rpc - .server_streaming(ImportFileRequest { - doc_id: self.id(), - author_id: author, - path: path.as_ref().into(), - key, - in_place, - }) - .await?; - Ok(ImportFileProgress::new(stream)) - } - - /// Exports an entry as a file to a given absolute path. - pub async fn export_file( - &self, - entry: Entry, - path: impl AsRef, - mode: ExportMode, - ) -> Result { - self.ensure_open()?; - let stream = self - .0 - .rpc - .server_streaming(ExportFileRequest { - entry: entry.0, - path: path.as_ref().into(), - mode, - }) - .await?; - Ok(ExportFileProgress::new(stream)) - } - - /// Deletes entries that match the given `author` and key `prefix`. - /// - /// This inserts an empty entry with the key set to `prefix`, effectively clearing all other - /// entries whose key starts with or is equal to the given `prefix`. - /// - /// Returns the number of entries deleted. - pub async fn del(&self, author_id: AuthorId, prefix: impl Into) -> Result { - self.ensure_open()?; - let res = self - .rpc(DelRequest { - doc_id: self.id(), - author_id, - prefix: prefix.into(), - }) - .await??; - let DelResponse { removed } = res; - Ok(removed) - } - - /// Returns an entry for a key and author. - /// - /// Optionally also returns the entry unless it is empty (i.e. a deletion marker). - pub async fn get_exact( - &self, - author: AuthorId, - key: impl AsRef<[u8]>, - include_empty: bool, - ) -> Result> { - self.ensure_open()?; - let res = self - .rpc(GetExactRequest { - author, - key: key.as_ref().to_vec().into(), - doc_id: self.id(), - include_empty, - }) - .await??; - Ok(res.entry.map(|entry| entry.into())) - } - - /// Returns all entries matching the query. - pub async fn get_many( - &self, - query: impl Into, - ) -> Result>> { - self.ensure_open()?; - let stream = self - .0 - .rpc - .server_streaming(GetManyRequest { - doc_id: self.id(), - query: query.into(), - }) - .await?; - Ok(flatten(stream).map(|res| res.map(|res| res.entry.into()))) - } - - /// Returns a single entry. - pub async fn get_one(&self, query: impl Into) -> Result> { - self.get_many(query).await?.next().await.transpose() - } - - /// Shares this document with peers over a ticket. - pub async fn share( - &self, - mode: ShareMode, - addr_options: AddrInfoOptions, - ) -> anyhow::Result { - self.ensure_open()?; - let res = self - .rpc(ShareRequest { - doc_id: self.id(), - mode, - addr_options, - }) - .await??; - Ok(res.0) - } - - /// Starts to sync this document with a list of peers. - pub async fn start_sync(&self, peers: Vec) -> Result<()> { - self.ensure_open()?; - let _res = self - .rpc(StartSyncRequest { - doc_id: self.id(), - peers, - }) - .await??; - Ok(()) - } - - /// Stops the live sync for this document. - pub async fn leave(&self) -> Result<()> { - self.ensure_open()?; - let _res = self.rpc(LeaveRequest { doc_id: self.id() }).await??; - Ok(()) - } - - /// Subscribes to events for this document. - pub async fn subscribe(&self) -> anyhow::Result>> { - self.ensure_open()?; - let stream = self - .0 - .rpc - .try_server_streaming(DocSubscribeRequest { doc_id: self.id() }) - .await?; - Ok(stream.map(|res| match res { - Ok(res) => Ok(res.event.into()), - Err(err) => Err(err.into()), - })) - } - - /// Returns status info for this document - pub async fn status(&self) -> anyhow::Result { - self.ensure_open()?; - let res = self.rpc(StatusRequest { doc_id: self.id() }).await??; - Ok(res.status) - } - - /// Sets the download policy for this document - pub async fn set_download_policy(&self, policy: DownloadPolicy) -> Result<()> { - self.rpc(SetDownloadPolicyRequest { - doc_id: self.id(), - policy, - }) - .await??; - Ok(()) - } - - /// Returns the download policy for this document - pub async fn get_download_policy(&self) -> Result { - let res = self - .rpc(GetDownloadPolicyRequest { doc_id: self.id() }) - .await??; - Ok(res.policy) - } - - /// Returns sync peers for this document - pub async fn get_sync_peers(&self) -> Result>> { - let res = self - .rpc(GetSyncPeersRequest { doc_id: self.id() }) - .await??; - Ok(res.peers) - } -} - -impl<'a> From<&'a Doc> for &'a RpcClient { - fn from(doc: &'a Doc) -> &'a RpcClient { - &doc.0.rpc - } -} - -/// A single entry in a [`Doc`]. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] -pub struct Entry(iroh_docs::Entry); - -impl From for Entry { - fn from(value: iroh_docs::Entry) -> Self { - Self(value) - } -} - -impl From for Entry { - fn from(value: iroh_docs::SignedEntry) -> Self { - Self(value.into()) - } -} - -impl Entry { - /// Returns the [`RecordIdentifier`] for this entry. - pub fn id(&self) -> &RecordIdentifier { - self.0.id() - } - - /// Returns the [`AuthorId`] of this entry. - pub fn author(&self) -> AuthorId { - self.0.author() - } - - /// Returns the [`struct@Hash`] of the content data of this record. - pub fn content_hash(&self) -> Hash { - self.0.content_hash() - } - - /// Returns the length of the data addressed by this record's content hash. - pub fn content_len(&self) -> u64 { - self.0.content_len() - } - - /// Returns the key of this entry. - pub fn key(&self) -> &[u8] { - self.0.key() - } - - /// Returns the timestamp of this entry. - pub fn timestamp(&self) -> u64 { - self.0.timestamp() - } - - /// Reads the content of an [`Entry`] as a streaming [`blobs::Reader`]. - /// - /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. - pub async fn content_reader(&self, client: impl Into<&RpcClient>) -> Result { - blobs::Reader::from_rpc_read(client.into(), self.content_hash()).await - } - - /// Reads all content of an [`Entry`] into a buffer. - /// - /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. - pub async fn content_bytes(&self, client: impl Into<&RpcClient>) -> Result { - blobs::Reader::from_rpc_read(client.into(), self.content_hash()) - .await? - .read_to_bytes() - .await - } -} - -/// Progress messages for an doc import operation -/// -/// An import operation involves computing the outboard of a file, and then -/// either copying or moving the file into the database, then setting the author, hash, size, and tag of that -/// file as an entry in the doc. -#[derive(Debug, Serialize, Deserialize)] -pub enum ImportProgress { - /// An item was found with name `name`, from now on referred to via `id`. - Found { - /// A new unique id for this entry. - id: u64, - /// The name of the entry. - name: String, - /// The size of the entry in bytes. - size: u64, - }, - /// We got progress ingesting item `id`. - Progress { - /// The unique id of the entry. - id: u64, - /// The offset of the progress, in bytes. - offset: u64, - }, - /// We are done adding `id` to the data store and the hash is `hash`. - IngestDone { - /// The unique id of the entry. - id: u64, - /// The hash of the entry. - hash: Hash, - }, - /// We are done setting the entry to the doc. - AllDone { - /// The key of the entry - key: Bytes, - }, - /// We got an error and need to abort. - /// - /// This will be the last message in the stream. - Abort(RpcError), -} - -/// Intended capability for document share tickets -#[derive(Serialize, Deserialize, Debug, Clone, Display, FromStr)] -pub enum ShareMode { - /// Read-only access - Read, - /// Write access - Write, -} - -/// Events informing about actions of the live sync progress. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] -pub enum LiveEvent { - /// A local insertion. - InsertLocal { - /// The inserted entry. - entry: Entry, - }, - /// Received a remote insert. - InsertRemote { - /// The peer that sent us the entry. - from: PublicKey, - /// The inserted entry. - entry: Entry, - /// If the content is available at the local node - content_status: ContentStatus, - }, - /// The content of an entry was downloaded and is now available at the local node - ContentReady { - /// The content hash of the newly available entry content - hash: Hash, - }, - /// We have a new neighbor in the swarm. - NeighborUp(PublicKey), - /// We lost a neighbor in the swarm. - NeighborDown(PublicKey), - /// A set-reconciliation sync finished. - SyncFinished(SyncEvent), - /// All pending content is now ready. - /// - /// This event signals that all queued content downloads from the last sync run have either - /// completed or failed. - /// - /// It will only be emitted after a [`Self::SyncFinished`] event, never before. - /// - /// Receiving this event does not guarantee that all content in the document is available. If - /// blobs failed to download, this event will still be emitted after all operations completed. - PendingContentReady, -} - -impl From for LiveEvent { - fn from(event: crate::docs::engine::LiveEvent) -> LiveEvent { - match event { - crate::docs::engine::LiveEvent::InsertLocal { entry } => Self::InsertLocal { - entry: entry.into(), - }, - crate::docs::engine::LiveEvent::InsertRemote { - from, - entry, - content_status, - } => Self::InsertRemote { - from, - content_status, - entry: entry.into(), - }, - crate::docs::engine::LiveEvent::ContentReady { hash } => Self::ContentReady { hash }, - crate::docs::engine::LiveEvent::NeighborUp(node) => Self::NeighborUp(node), - crate::docs::engine::LiveEvent::NeighborDown(node) => Self::NeighborDown(node), - crate::docs::engine::LiveEvent::SyncFinished(details) => Self::SyncFinished(details), - crate::docs::engine::LiveEvent::PendingContentReady => Self::PendingContentReady, - } - } -} - -/// Progress stream for [`Doc::import_file`]. -#[derive(derive_more::Debug)] -#[must_use = "streams do nothing unless polled"] -pub struct ImportFileProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, -} - -impl ImportFileProgress { - fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let stream = stream.map(|item| match item { - Ok(item) => Ok(item.into()), - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - } - } - - /// Finishes writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`ImportFileOutcome`] which contains a tag, key, and hash and the size of the - /// content. - pub async fn finish(mut self) -> Result { - let mut entry_size = 0; - let mut entry_hash = None; - while let Some(msg) = self.next().await { - match msg? { - ImportProgress::Found { size, .. } => { - entry_size = size; - } - ImportProgress::AllDone { key } => { - let hash = entry_hash - .context("expected DocImportProgress::IngestDone event to occur")?; - let outcome = ImportFileOutcome { - hash, - key, - size: entry_size, - }; - return Ok(outcome); - } - ImportProgress::Abort(err) => return Err(err.into()), - ImportProgress::Progress { .. } => {} - ImportProgress::IngestDone { hash, .. } => { - entry_hash = Some(hash); - } - } - } - Err(anyhow!("Response stream ended prematurely")) - } -} - -/// Outcome of a [`Doc::import_file`] operation -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ImportFileOutcome { - /// The hash of the entry's content - pub hash: Hash, - /// The size of the entry - pub size: u64, - /// The key of the entry - pub key: Bytes, -} - -impl Stream for ImportFileProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -/// Progress stream for [`Doc::export_file`]. -#[derive(derive_more::Debug)] -pub struct ExportFileProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, -} -impl ExportFileProgress { - fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let stream = stream.map(|item| match item { - Ok(item) => Ok(item.into()), - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - } - } - - /// Iterates through the export progress stream, returning when the stream has completed. - /// - /// Returns a [`ExportFileOutcome`] which contains a file path the data was written to and the size of the content. - pub async fn finish(mut self) -> Result { - let mut total_size = 0; - let mut path = None; - while let Some(msg) = self.next().await { - match msg? { - ExportProgress::Found { size, outpath, .. } => { - total_size = size.value(); - path = Some(outpath); - } - ExportProgress::AllDone => { - let path = path.context("expected ExportProgress::Found event to occur")?; - let outcome = ExportFileOutcome { - size: total_size, - path, - }; - return Ok(outcome); - } - ExportProgress::Done { .. } => {} - ExportProgress::Abort(err) => return Err(anyhow!(err)), - ExportProgress::Progress { .. } => {} - } - } - Err(anyhow!("Response stream ended prematurely")) - } -} - -/// Outcome of a [`Doc::export_file`] operation -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ExportFileOutcome { - /// The size of the entry - size: u64, - /// The path to which the entry was saved - path: PathBuf, -} - -impl Stream for ExportFileProgress { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -#[cfg(test)] -mod tests { - use rand::RngCore; - use tokio::io::AsyncWriteExt; - - use super::*; - - // TODO(Frando): This fails consistently with timeout, the task never joins. - // Something in the addition of willow engine to the node makes this - // test timeout - debugging outstanding. - #[tokio::test] - async fn test_drop_doc_client_sync() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().enable_docs().spawn().await?; - - let client = node.client(); - let doc = client.docs().create().await?; - - let res = std::thread::spawn(move || { - println!("now drop doc"); - drop(doc); - println!("now drop node"); - drop(node); - println!("done"); - }); - - println!("wait task"); - tokio::task::spawn_blocking(move || res.join().map_err(|e| anyhow::anyhow!("{:?}", e))) - .await??; - println!("task done"); - - Ok(()) - } - - /// Test that closing a doc does not close other instances. - #[tokio::test] - async fn test_doc_close() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().enable_docs().spawn().await?; - let author = node.authors().default().await?; - // open doc two times - let doc1 = node.docs().create().await?; - let doc2 = node.docs().open(doc1.id()).await?.expect("doc to exist"); - // close doc1 instance - doc1.close().await?; - // operations on doc1 now fail. - assert!(doc1.set_bytes(author, "foo", "bar").await.is_err()); - // dropping doc1 will close the doc if not already closed - // wait a bit because the close-on-drop spawns a task for which we cannot track completion. - drop(doc1); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // operations on doc2 still succeed - doc2.set_bytes(author, "foo", "bar").await?; - Ok(()) - } - - #[tokio::test] - async fn test_doc_import_export() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().enable_docs().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - let out_root = temp_dir.path().join("out"); - - let path = in_root.join("test"); - - let size = 100; - let mut buf = vec![0u8; size]; - rand::thread_rng().fill_bytes(&mut buf); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - - // create doc & author - let client = node.client(); - let doc = client.docs().create().await.context("doc create")?; - let author = client.authors().create().await.context("author create")?; - - // import file - let import_outcome = doc - .import_file( - author, - crate::util::fs::path_to_key(path.clone(), None, Some(in_root))?, - path, - true, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - // export file - let entry = doc - .get_one(Query::author(author).key_exact(import_outcome.key)) - .await - .context("get one")? - .unwrap(); - let key = entry.key().to_vec(); - let export_outcome = doc - .export_file( - entry, - crate::util::fs::key_to_path(key, None, Some(out_root))?, - ExportMode::Copy, - ) - .await - .context("export file")? - .finish() - .await - .context("export finish")?; - - let got_bytes = tokio::fs::read(export_outcome.path) - .await - .context("tokio read")?; - assert_eq!(buf, got_bytes); - - Ok(()) - } -} diff --git a/iroh/src/client/gossip.rs b/iroh/src/client/gossip.rs deleted file mode 100644 index edb2bc0bd0f..00000000000 --- a/iroh/src/client/gossip.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Gossip client. -//! -//! The gossip client allows you to subscribe to gossip topics and send updates to them. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::gossip()`](crate::client::Iroh::gossip). -//! -//! The gossip API is extremely simple. You use [`subscribe`](Client::subscribe) -//! to subscribe to a topic. This returns a sink to send updates to the topic -//! and a stream of responses. -//! -//! [`Client::subscribe_with_opts`] allows you to specify advanced options -//! such as the buffer size. -use std::collections::BTreeSet; - -use anyhow::Result; -use futures_lite::{Stream, StreamExt}; -use futures_util::{Sink, SinkExt}; -use iroh_gossip::proto::TopicId; -use iroh_net::NodeId; -use ref_cast::RefCast; - -use super::RpcClient; -pub use crate::rpc_protocol::gossip::{SubscribeRequest, SubscribeResponse, SubscribeUpdate}; - -/// Iroh gossip client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -/// Options for subscribing to a gossip topic. -#[derive(Debug, Clone)] -pub struct SubscribeOpts { - /// Bootstrap nodes to connect to. - pub bootstrap: BTreeSet, - /// Subscription capacity. - pub subscription_capacity: usize, -} - -impl Default for SubscribeOpts { - fn default() -> Self { - Self { - bootstrap: BTreeSet::new(), - subscription_capacity: 256, - } - } -} - -impl Client { - /// Subscribes to a gossip topic. - /// - /// Returns a sink to send updates to the topic and a stream of responses. - /// - /// Updates are either [Broadcast](iroh_gossip::net::Command::Broadcast) - /// or [BroadcastNeighbors](iroh_gossip::net::Command::BroadcastNeighbors). - /// - /// Broadcasts are gossiped to the entire swarm, while BroadcastNeighbors are sent to - /// just the immediate neighbors of the node. - /// - /// Responses are either [Gossip](iroh_gossip::net::Event::Gossip) or - /// [Lagged](iroh_gossip::net::Event::Lagged). - /// - /// Gossip events contain the actual message content, as well as information about the - /// immediate neighbors of the node. - /// - /// A Lagged event indicates that the gossip stream has not been consumed quickly enough. - /// You can adjust the buffer size with the [`SubscribeOpts::subscription_capacity`] option. - pub async fn subscribe_with_opts( - &self, - topic: TopicId, - opts: SubscribeOpts, - ) -> Result<( - impl Sink, - impl Stream>, - )> { - let (sink, stream) = self - .rpc - .bidi(SubscribeRequest { - topic, - bootstrap: opts.bootstrap, - subscription_capacity: opts.subscription_capacity, - }) - .await?; - let stream = stream.map(|item| anyhow::Ok(item??)); - let sink = sink.sink_map_err(|_| anyhow::anyhow!("send error")); - Ok((sink, stream)) - } - - /// Subscribes to a gossip topic with default options. - pub async fn subscribe( - &self, - topic: impl Into, - bootstrap: impl IntoIterator>, - ) -> Result<( - impl Sink, - impl Stream>, - )> { - let bootstrap = bootstrap.into_iter().map(Into::into).collect(); - self.subscribe_with_opts( - topic.into(), - SubscribeOpts { - bootstrap, - ..Default::default() - }, - ) - .await - } -} diff --git a/iroh/src/client/net.rs b/iroh/src/client/net.rs index d4a11b5f014..274db7cd5f5 100644 --- a/iroh/src/client/net.rs +++ b/iroh/src/client/net.rs @@ -13,7 +13,8 @@ use std::net::SocketAddr; use anyhow::Result; use futures_lite::{Stream, StreamExt}; -use iroh_net::{endpoint::RemoteInfo, relay::RelayUrl, NodeAddr, NodeId}; +use iroh_base::node_addr::RelayUrl; +use iroh_net::{endpoint::RemoteInfo, NodeAddr, NodeId}; use ref_cast::RefCast; use serde::{Deserialize, Serialize}; @@ -39,7 +40,11 @@ use crate::rpc_protocol::net::{ /// # Examples /// ``` /// use std::str::FromStr; -/// use iroh_base::{key::NodeId, node_addr::{RelayUrl, NodeAddr}}; +/// +/// use iroh_base::{ +/// key::NodeId, +/// node_addr::{NodeAddr, RelayUrl}, +/// }; /// use url::Url; /// /// # async fn run() -> anyhow::Result<()> { @@ -50,12 +55,13 @@ use crate::rpc_protocol::net::{ /// // Provide your node an address for another node /// let relay_url = RelayUrl::from(Url::parse("https://example.com").unwrap()); /// let addr = NodeAddr::from_parts( -/// // the node_id -/// NodeId::from_str("ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6").unwrap(), -/// // the home relay -/// Some(relay_url), -/// // the direct addresses -/// ["120.0.0.1:0".parse().unwrap()], +/// // the node_id +/// NodeId::from_str("ae58ff8833241ac82d6ff7611046ed67b5072d142c588d0063e942d9a75502b6") +/// .unwrap(), +/// // the home relay +/// Some(relay_url), +/// // the direct addresses +/// ["120.0.0.1:0".parse().unwrap()], /// ); /// net_client.add_node_addr(addr).await?; /// // Shut down the node. Passing `true` will force the shutdown, passing in diff --git a/iroh/src/client/quic.rs b/iroh/src/client/quic.rs index a9af6917d96..f7634cd57df 100644 --- a/iroh/src/client/quic.rs +++ b/iroh/src/client/quic.rs @@ -8,13 +8,10 @@ use std::{ }; use anyhow::{bail, Context}; -use quic_rpc::transport::{boxed::Connection as BoxedConnection, quinn::QuinnConnection}; +use quic_rpc::transport::{boxed::BoxedConnector, quinn::QuinnConnector}; use super::{Iroh, RpcClient}; -use crate::{ - node::RpcStatus, - rpc_protocol::{node::StatusRequest, RpcService}, -}; +use crate::{node::RpcStatus, rpc_protocol::node::StatusRequest}; /// ALPN used by irohs RPC mechanism. // TODO: Change to "/iroh-rpc/1" @@ -46,8 +43,8 @@ pub(crate) async fn connect_raw(addr: SocketAddr) -> anyhow::Result { let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; let server_name = "localhost".to_string(); - let connection = QuinnConnection::::new(endpoint, addr, server_name); - let connection = BoxedConnection::new(connection); + let connection = QuinnConnector::new(endpoint, addr, server_name); + let connection = BoxedConnector::new(connection); let client = RpcClient::new(connection); // Do a status request to check if the server is running. let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(StatusRequest)) diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs deleted file mode 100644 index 1b0accf03aa..00000000000 --- a/iroh/src/client/tags.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! API for tag management. -//! -//! The purpose of tags is to mark information as important to prevent it -//! from being garbage-collected (if the garbage collector is turned on). -//! Currently this is used for blobs. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::tags()`](crate::client::Iroh::tags). -//! -//! [`Client::list`] can be used to list all tags. -//! [`Client::list_hash_seq`] can be used to list all tags with a hash_seq format. -//! -//! [`Client::delete`] can be used to delete a tag. -use anyhow::Result; -use futures_lite::{Stream, StreamExt}; -use iroh_blobs::{BlobFormat, Hash, Tag}; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; - -use super::RpcClient; -use crate::rpc_protocol::tags::{DeleteRequest, ListRequest}; - -/// Iroh tags client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl Client { - /// Lists all tags. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest::all()).await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Lists all tags with a hash_seq format. - pub async fn list_hash_seq(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest::hash_seq()).await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Deletes a tag. - pub async fn delete(&self, name: Tag) -> Result<()> { - self.rpc.rpc(DeleteRequest { name }).await??; - Ok(()) - } -} - -/// Information about a tag. -#[derive(Debug, Serialize, Deserialize)] -pub struct TagInfo { - /// Name of the tag - pub name: Tag, - /// Format of the data - pub format: BlobFormat, - /// Hash of the data - pub hash: Hash, -} diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index 921b3604aec..6906f7b4f5c 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -51,11 +51,8 @@ //! tags to tell iroh what data is important //! - [gossip](crate::client::gossip): //! exchange data with other nodes via a gossip protocol -//! -//! - [authors](crate::client::authors): -//! interact with document authors //! - [docs](crate::client::docs): -//! interact with documents +//! interact with documents and document authors //! //! The subsystem clients can be obtained cheaply from the main iroh client. //! They are also cheaply cloneable and can be shared across threads. @@ -87,7 +84,6 @@ //! //! - `metrics`: Enable metrics collection. Enabled by default. //! - `fs-store`: Enables the disk based storage backend for `iroh-blobs`. Enabled by default. -//! #![cfg_attr(iroh_docsrs, feature(doc_cfg))] #![deny(missing_docs, rustdoc::broken_intra_doc_links)] @@ -103,7 +99,7 @@ pub use iroh_gossip as gossip; #[doc(inline)] pub use iroh_net as net; #[doc(inline)] -pub use iroh_willow as spaces; +pub use iroh_router as router; pub mod client; pub mod node; diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 1f1b233b3f1..7012865b61d 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -50,32 +50,29 @@ use futures_lite::StreamExt; use futures_util::future::{MapErr, Shared}; use iroh_base::key::PublicKey; use iroh_blobs::{ - protocol::Closed, + net_protocol::Blobs as BlobsProtocol, store::Store as BaoStore, util::local_pool::{LocalPool, LocalPoolHandle}, }; -use iroh_docs::net::DOCS_ALPN; +use iroh_docs::{engine::Engine, net::DOCS_ALPN}; use iroh_net::{ endpoint::{DirectAddrsStream, RemoteInfo}, AddrInfo, Endpoint, NodeAddr, }; -use protocol::BlobsProtocol; -use quic_rpc::{transport::ServerEndpoint as _, RpcServer}; +use iroh_router::{ProtocolHandler, Router}; +use quic_rpc::{transport::Listener as _, RpcServer}; use tokio::task::{JoinError, JoinSet}; use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{debug, error, info, info_span, trace, warn, Instrument}; -use crate::node::{docs::DocsEngine, nodes_storage::store_node_addrs, protocol::ProtocolMap}; +use crate::node::nodes_storage::store_node_addrs; mod builder; -mod docs; mod nodes_storage; -mod protocol; mod rpc; mod rpc_status; -pub use protocol::ProtocolHandler; - +pub(crate) use self::rpc::RpcResult; pub use self::{ builder::{ Builder, DiscoveryConfig, DocsStorage, GcPolicy, ProtocolBuilder, StorageConfig, @@ -90,7 +87,7 @@ const SAVE_NODES_INTERVAL: Duration = Duration::from_secs(30); /// The quic-rpc server endpoint for the iroh node. /// /// We use a boxed endpoint here to allow having a concrete type for the server endpoint. -pub type IrohServerEndpoint = quic_rpc::transport::boxed::ServerEndpoint< +pub type IrohServerEndpoint = quic_rpc::transport::boxed::BoxedListener< crate::rpc_protocol::Request, crate::rpc_protocol::Response, >; @@ -115,7 +112,7 @@ pub struct Node { // - `AbortOnDropHandle` to make sure that the `task` is cancelled when all `Node`s are dropped // (`Shared` acts like an `Arc` around its inner future). task: Shared, JoinErrToStr>>, - protocols: Arc, + router: Router, } pub(crate) type JoinErrToStr = Box String + Send + Sync + 'static>; @@ -128,7 +125,6 @@ struct NodeInner { cancel_token: CancellationToken, client: crate::client::Iroh, local_pool_handle: LocalPoolHandle, - willow: Option, } /// In memory node. @@ -208,7 +204,7 @@ impl Node { } /// Get the relay server we are connected to. - pub fn home_relay(&self) -> Option { + pub fn home_relay(&self) -> Option { self.inner.endpoint.home_relay() } @@ -245,7 +241,7 @@ impl Node { /// This downcasts to the concrete type and returns `None` if the handler registered for `alpn` /// does not match the passed type. pub fn get_protocol(&self, alpn: &[u8]) -> Option> { - self.protocols.get_typed(alpn) + self.router.get_protocol(alpn) } } @@ -273,7 +269,7 @@ impl NodeInner { self: Arc, external_rpc: IrohServerEndpoint, internal_rpc: IrohServerEndpoint, - protocols: Arc, + router: Router, gc_policy: GcPolicy, gc_done_callback: Option>, nodes_data_path: Option, @@ -295,11 +291,11 @@ impl NodeInner { // Spawn a task for the garbage collection. if let GcPolicy::Interval(gc_period) = gc_policy { - let protocols = protocols.clone(); + let router = router.clone(); let handle = local_pool.spawn(move || async move { - let docs_engine = protocols.get_typed::(DOCS_ALPN); - let blobs = protocols - .get_typed::>(iroh_blobs::protocol::ALPN) + let docs_engine = router.get_protocol::>(DOCS_ALPN); + let blobs = router + .get_protocol::>(iroh_blobs::protocol::ALPN) .expect("missing blobs"); blobs @@ -399,7 +395,7 @@ impl NodeInner { request = external_rpc.accept() => { match request { Ok(accepting) => { - rpc::Handler::spawn_rpc_request(self.clone(), &mut join_set, accepting, protocols.clone()); + rpc::Handler::spawn_rpc_request(self.clone(), &mut join_set, accepting, router.clone()); } Err(e) => { info!("rpc request error: {:?}", e); @@ -410,21 +406,13 @@ impl NodeInner { request = internal_rpc.accept() => { match request { Ok(accepting) => { - rpc::Handler::spawn_rpc_request(self.clone(), &mut join_set, accepting, protocols.clone()); + rpc::Handler::spawn_rpc_request(self.clone(), &mut join_set, accepting, router.clone()); } Err(e) => { info!("internal rpc request error: {:?}", e); } } }, - // handle incoming p2p connections. - Some(incoming) = self.endpoint.accept() => { - let protocols = protocols.clone(); - join_set.spawn(async move { - handle_connection(incoming, protocols).await; - Ok(()) - }); - }, // handle task terminations and quit on panics. res = join_set.join_next(), if !join_set.is_empty() => { match res { @@ -449,7 +437,9 @@ impl NodeInner { } } - self.shutdown(protocols).await; + if let Err(err) = router.shutdown().await { + tracing::warn!("Error when shutting down router: {:?}", err); + }; // Abort remaining tasks. join_set.shutdown().await; @@ -459,48 +449,6 @@ impl NodeInner { tracing::info!("Shutting down local pool"); local_pool.shutdown().await; } - - /// Shutdown the different parts of the node concurrently. - async fn shutdown(&self, protocols: Arc) { - let error_code = Closed::ProviderTerminating; - - // We ignore all errors during shutdown. - let _ = tokio::join!( - // Close the endpoint. - // Closing the Endpoint is the equivalent of calling Connection::close on all - // connections: Operations will immediately fail with ConnectionError::LocallyClosed. - // All streams are interrupted, this is not graceful. - self.endpoint - .clone() - .close(error_code.into(), error_code.reason()), - // Shutdown protocol handlers. - protocols.shutdown(), - ); - } -} - -async fn handle_connection(incoming: iroh_net::endpoint::Incoming, protocols: Arc) { - let mut connecting = match incoming.accept() { - Ok(conn) => conn, - Err(err) => { - warn!("Ignoring connection: accepting failed: {err:#}"); - return; - } - }; - let alpn = match connecting.alpn().await { - Ok(alpn) => alpn, - Err(err) => { - warn!("Ignoring connection: invalid handshake: {err:#}"); - return; - } - }; - let Some(handler) = protocols.get(&alpn) else { - warn!("Ignoring connection: unsupported ALPN protocol"); - return; - }; - if let Err(err) = handler.accept(connecting).await { - warn!("Handling incoming connection ended with error: {err}"); - } } fn node_addresses_for_storage(ep: &Endpoint) -> Vec { @@ -549,9 +497,9 @@ fn node_address_for_storage(info: RemoteInfo) -> Option { mod tests { use anyhow::{bail, Context}; use bytes::Bytes; - use iroh_base::node_addr::AddrInfoOptions; + use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use iroh_blobs::{provider::AddProgress, util::SetTagOption, BlobFormat}; - use iroh_net::{key::SecretKey, relay::RelayMode, test_utils::DnsPkarrServer, NodeAddr}; + use iroh_net::{key::SecretKey, test_utils::DnsPkarrServer, NodeAddr, RelayMode}; use super::*; use crate::client::blobs::{AddOutcome, WrapOption}; @@ -570,11 +518,9 @@ mod tests { .hash; let _drop_guard = node.cancel_token().drop_guard(); - let ticket = node - .blobs() - .share(hash, BlobFormat::Raw, AddrInfoOptions::RelayAndAddresses) - .await - .unwrap(); + let mut addr = node.net().node_addr().await.unwrap(); + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, BlobFormat::Raw).unwrap(); println!("addrs: {:?}", ticket.node_addr().info); assert!(!ticket.node_addr().info.direct_addresses.is_empty()); } @@ -744,142 +690,4 @@ mod tests { ); Ok(()) } - - #[tokio::test] - async fn test_default_author_memory() -> Result<()> { - let iroh = Node::memory().enable_docs().spawn().await?; - let author = iroh.authors().default().await?; - assert!(iroh.authors().export(author).await?.is_some()); - assert!(iroh.authors().delete(author).await.is_err()); - Ok(()) - } - - #[cfg(feature = "fs-store")] - #[tokio::test] - async fn test_default_author_persist() -> Result<()> { - use crate::util::path::IrohPaths; - - let _guard = iroh_test::logging::setup(); - - let iroh_root_dir = tempfile::TempDir::new().unwrap(); - let iroh_root = iroh_root_dir.path(); - - // check that the default author exists and cannot be deleted. - let default_author = { - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await - .unwrap(); - let author = iroh.authors().default().await.unwrap(); - assert!(iroh.authors().export(author).await.unwrap().is_some()); - assert!(iroh.authors().delete(author).await.is_err()); - iroh.shutdown().await.unwrap(); - author - }; - - // check that the default author is persisted across restarts. - { - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await - .unwrap(); - let author = iroh.authors().default().await.unwrap(); - assert_eq!(author, default_author); - assert!(iroh.authors().export(author).await.unwrap().is_some()); - assert!(iroh.authors().delete(author).await.is_err()); - iroh.shutdown().await.unwrap(); - }; - - // check that a new default author is created if the default author file is deleted - // manually. - let default_author = { - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) - .await - .unwrap(); - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await - .unwrap(); - let author = iroh.authors().default().await.unwrap(); - assert!(author != default_author); - assert!(iroh.authors().export(author).await.unwrap().is_some()); - assert!(iroh.authors().delete(author).await.is_err()); - iroh.shutdown().await.unwrap(); - author - }; - - // check that the node fails to start if the default author is missing from the docs store. - { - let mut docs_store = iroh_docs::store::fs::Store::persistent( - IrohPaths::DocsDatabase.with_root(iroh_root), - ) - .unwrap(); - docs_store.delete_author(default_author).unwrap(); - docs_store.flush().unwrap(); - drop(docs_store); - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await; - assert!(iroh.is_err()); - - // somehow the blob store is not shutdown correctly (yet?) on macos. - // so we give it some time until we find a proper fix. - #[cfg(target_os = "macos")] - tokio::time::sleep(Duration::from_secs(1)).await; - - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) - .await - .unwrap(); - drop(iroh); - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await; - assert!(iroh.is_ok()); - iroh.unwrap().shutdown().await.unwrap(); - } - - // check that the default author can be set manually and is persisted. - let default_author = { - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await - .unwrap(); - let author = iroh.authors().create().await.unwrap(); - iroh.authors().set_default(author).await.unwrap(); - assert_eq!(iroh.authors().default().await.unwrap(), author); - iroh.shutdown().await.unwrap(); - author - }; - { - let iroh = Node::persistent(iroh_root) - .await - .unwrap() - .enable_docs() - .spawn() - .await - .unwrap(); - assert_eq!(iroh.authors().default().await.unwrap(), default_author); - iroh.shutdown().await.unwrap(); - } - - Ok(()) - } } diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 0daa3a02eed..e6524f41e69 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -11,38 +11,35 @@ use futures_util::{FutureExt as _, TryFutureExt as _}; use iroh_base::key::SecretKey; use iroh_blobs::{ downloader::Downloader, + net_protocol::Blobs as BlobsProtocol, provider::EventSender, store::{Map, Store as BaoStore}, util::local_pool::{self, LocalPool, LocalPoolHandle, PanicMode}, }; -use iroh_docs::{engine::DefaultAuthorStorage, net::DOCS_ALPN}; +use iroh_docs::{ + engine::{DefaultAuthorStorage, Engine}, + net::DOCS_ALPN, +}; use iroh_gossip::net::{Gossip, GOSSIP_ALPN}; #[cfg(not(test))] use iroh_net::discovery::local_swarm_discovery::LocalSwarmDiscovery; use iroh_net::{ discovery::{dns::DnsDiscovery, pkarr::PkarrPublisher, ConcurrentDiscovery, Discovery}, dns::DnsResolver, - endpoint::TransportConfig, - relay::{force_staging_infra, RelayMode}, - Endpoint, + endpoint::{force_staging_infra, TransportConfig}, + Endpoint, RelayMode, }; -use quic_rpc::transport::{boxed::BoxableServerEndpoint, quinn::QuinnServerEndpoint}; +use iroh_router::{ProtocolHandler, RouterBuilder}; +use quic_rpc::transport::{boxed::BoxableListener, quinn::QuinnListener}; use serde::{Deserialize, Serialize}; use tokio::task::JoinError; use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{debug, error_span, trace, Instrument}; -use super::{ - docs::DocsEngine, rpc_status::RpcStatus, IrohServerEndpoint, JoinErrToStr, Node, NodeInner, -}; +use super::{rpc_status::RpcStatus, IrohServerEndpoint, JoinErrToStr, Node, NodeInner}; use crate::{ client::RPC_ALPN, - node::{ - nodes_storage::load_node_addrs, - protocol::{BlobsProtocol, ProtocolMap}, - ProtocolHandler, - }, - rpc_protocol::RpcService, + node::nodes_storage::load_node_addrs, util::{fs::load_secret_key, path::IrohPaths}, }; @@ -75,18 +72,32 @@ pub enum DocsStorage { Persistent(PathBuf), } -/// Storage backend for spaces. -#[derive(Debug, Clone)] -pub enum SpacesStorage { - /// Disable docs completely. - Disabled, - /// In-memory storage. - Memory, - /// File-based persistent storage. - Persistent(PathBuf), - /// (test only) persistent storage with in-memory redb backend. - #[cfg(feature = "test-utils")] - PersistentTest, +/// Start the engine, and prepare the selected storage version. +async fn spawn_docs( + storage: DocsStorage, + blobs_store: S, + default_author_storage: DefaultAuthorStorage, + endpoint: Endpoint, + gossip: Gossip, + downloader: Downloader, + local_pool_handle: LocalPoolHandle, +) -> anyhow::Result>> { + let docs_store = match storage { + DocsStorage::Disabled => return Ok(None), + DocsStorage::Memory => iroh_docs::store::fs::Store::memory(), + DocsStorage::Persistent(path) => iroh_docs::store::fs::Store::persistent(path)?, + }; + let engine = Engine::spawn( + endpoint, + gossip, + docs_store, + blobs_store, + downloader, + default_author_storage, + local_pool_handle, + ) + .await?; + Ok(Some(engine)) } /// Builder for the [`Node`]. @@ -131,7 +142,6 @@ where dns_resolver: Option, node_discovery: DiscoveryConfig, docs_storage: DocsStorage, - spaces_storage: SpacesStorage, #[cfg(any(test, feature = "test-utils"))] insecure_skip_relay_cert_verify: bool, /// Callback to register when a gc loop is done @@ -212,13 +222,12 @@ impl From> for DiscoveryConfig { #[derive(Debug, Default)] struct DummyServerEndpoint; -impl BoxableServerEndpoint +impl BoxableListener for DummyServerEndpoint { fn clone_box( &self, - ) -> Box> - { + ) -> Box> { Box::new(DummyServerEndpoint) } @@ -237,7 +246,7 @@ impl BoxableServerEndpoint IrohServerEndpoint { - quic_rpc::transport::boxed::ServerEndpoint::new(DummyServerEndpoint) + quic_rpc::transport::boxed::BoxedListener::new(DummyServerEndpoint) } impl Default for Builder { @@ -261,7 +270,6 @@ impl Default for Builder { rpc_addr: None, gc_policy: GcPolicy::Disabled, docs_storage: DocsStorage::Disabled, - spaces_storage: SpacesStorage::Memory, node_discovery: Default::default(), #[cfg(any(test, feature = "test-utils"))] insecure_skip_relay_cert_verify: false, @@ -298,8 +306,6 @@ impl Builder { rpc_addr: None, gc_policy: GcPolicy::Disabled, docs_storage, - // TODO: Expose in function - spaces_storage: SpacesStorage::Disabled, node_discovery: Default::default(), #[cfg(any(test, feature = "test-utils"))] insecure_skip_relay_cert_verify: false, @@ -343,7 +349,6 @@ where } DocsStorage::Disabled => DocsStorage::Disabled, }; - let spaces_storage = SpacesStorage::Persistent(IrohPaths::SpacesDatabase.with_root(root)); let secret_key_path = IrohPaths::SecretKey.with_root(root); let secret_key = load_secret_key(secret_key_path).await?; @@ -361,7 +366,6 @@ where dns_resolver: self.dns_resolver, gc_policy: self.gc_policy, docs_storage, - spaces_storage, node_discovery: self.node_discovery, #[cfg(any(test, feature = "test-utils"))] insecure_skip_relay_cert_verify: false, @@ -371,19 +375,6 @@ where }) } - /// Enables the persistent store, but with an in-memory backend. - /// - /// There's significant differences between the naive spaces in-memory store - /// and the persistent store, even if it's using the in-memory backend, - /// making it useful to test these two implementations against each other. - #[cfg(feature = "test-utils")] - pub fn enable_spaces_persist_test_mode(mut self, test_mode: bool) -> Self { - if test_mode == true { - self.spaces_storage = SpacesStorage::PersistentTest; - } - self - } - /// Configure rpc endpoint. pub fn rpc_endpoint(self, value: IrohServerEndpoint, rpc_addr: Option) -> Self { Self { @@ -403,7 +394,7 @@ where let (ep, actual_rpc_port) = make_rpc_endpoint(&self.secret_key, rpc_addr)?; rpc_addr.set_port(actual_rpc_port); - let ep = quic_rpc::transport::boxed::ServerEndpoint::new(ep); + let ep = quic_rpc::transport::boxed::BoxedListener::new(ep); if let StorageConfig::Persistent(ref root) = self.storage { // store rpc endpoint RpcStatus::store(root, actual_rpc_port).await?; @@ -435,12 +426,6 @@ where self } - /// Disables spaces support on this node completely. - pub fn disable_spaces(mut self) -> Self { - self.spaces_storage = SpacesStorage::Disabled; - self - } - /// Sets the relay servers to assist in establishing connectivity. /// /// Relay servers are used to discover other nodes by `PublicKey` and also help @@ -448,7 +433,7 @@ where /// assisting in holepunching to establish a direct connection between peers. /// /// When using [`RelayMode::Custom`], the provided `relay_map` must contain at least one - /// configured relay node. If an invalid [`iroh_net::relay::RelayMode`] is provided + /// configured relay node. If an invalid [`iroh_net::RelayMode`] is provided /// [`Self::spawn`] will result in an error. /// /// # Usage during tests @@ -693,83 +678,38 @@ where let downloader = Downloader::new(self.blobs_store.clone(), endpoint.clone(), lp.clone()); // Spawn the docs engine, if enabled. - // This returns None for DocsStorage::Disabled, otherwise Some(DocsEngine). - let docs = DocsEngine::spawn( + // This returns None for DocsStorage::Disabled, otherwise Some(DocsProtocol). + let docs = spawn_docs( self.docs_storage, self.blobs_store.clone(), self.storage.default_author_storage(), endpoint.clone(), gossip.clone(), downloader.clone(), + lp.handle().clone(), ) .await?; - let blobs_store = self.blobs_store.clone(); - let willow = match self.spaces_storage { - SpacesStorage::Disabled => None, - SpacesStorage::Memory => { - let create_store = move || { - // iroh_willow::store::memory::Store::new(blobs_store) - iroh_willow::store::persistent::Store::new_memory(blobs_store) - .expect("couldn't initialize store") - }; - let engine = iroh_willow::Engine::spawn( - endpoint.clone(), - create_store, - iroh_willow::engine::AcceptOpts::default(), - ); - Some(engine) - } - SpacesStorage::Persistent(path) => { - let create_store = move || { - iroh_willow::store::persistent::Store::new(path, blobs_store) - .expect("failed to spawn persistent store") // TODO(matheus23): introduce fallibility? - }; - let engine = iroh_willow::Engine::spawn( - endpoint.clone(), - create_store, - iroh_willow::engine::AcceptOpts::default(), - ); - Some(engine) - } - #[cfg(feature = "test-utils")] - SpacesStorage::PersistentTest => { - let create_store = move || { - iroh_willow::store::persistent::Store::new_memory(blobs_store) - .expect("couldn't initialize store") - }; - let engine = iroh_willow::Engine::spawn( - endpoint.clone(), - create_store, - iroh_willow::engine::AcceptOpts::default(), - ); - Some(engine) - } - }; - // Spawn the willow engine. - // TODO: Allow to disable. - // Initialize the internal RPC connection. - let (internal_rpc, controller) = quic_rpc::transport::flume::connection::(32); - let internal_rpc = quic_rpc::transport::boxed::ServerEndpoint::new(internal_rpc); + let (internal_rpc, controller) = quic_rpc::transport::flume::channel(32); + let internal_rpc = quic_rpc::transport::boxed::BoxedListener::new(internal_rpc); // box the controller. Boxing has a special case for the flume channel that avoids allocations, // so this has zero overhead. - let controller = quic_rpc::transport::boxed::Connection::new(controller); + let controller = quic_rpc::transport::boxed::BoxedConnector::new(controller); let client = crate::client::Iroh::new(quic_rpc::RpcClient::new(controller.clone())); let inner = Arc::new(NodeInner { rpc_addr: self.rpc_addr, db: Default::default(), - endpoint, + endpoint: endpoint.clone(), client, cancel_token: CancellationToken::new(), local_pool_handle: lp.handle().clone(), - willow, }); let protocol_builder = ProtocolBuilder { inner, - protocols: Default::default(), + router: RouterBuilder::new(endpoint), internal_rpc, external_rpc: self.rpc_endpoint, gc_policy: self.gc_policy, @@ -803,7 +743,7 @@ pub struct ProtocolBuilder { inner: Arc>, internal_rpc: IrohServerEndpoint, external_rpc: IrohServerEndpoint, - protocols: ProtocolMap, + router: RouterBuilder, #[debug("callback")] gc_done_callback: Option>, gc_policy: GcPolicy, @@ -825,7 +765,7 @@ impl ProtocolBuilder { /// # use std::sync::Arc; /// # use anyhow::Result; /// # use futures_lite::future::Boxed as BoxedFuture; - /// # use iroh::{node::{Node, ProtocolHandler}, net::endpoint::Connecting, client::Iroh}; + /// # use iroh::{node::{Node}, net::endpoint::Connecting, client::Iroh, router::ProtocolHandler}; /// # /// # #[tokio::main] /// # async fn main() -> Result<()> { @@ -858,10 +798,8 @@ impl ProtocolBuilder { /// # Ok(()) /// # } /// ``` - /// - /// pub fn accept(mut self, alpn: Vec, handler: Arc) -> Self { - self.protocols.insert(alpn, handler); + self.router = self.router.accept(alpn, handler); self } @@ -888,7 +826,7 @@ impl ProtocolBuilder { /// This downcasts to the concrete type and returns `None` if the handler registered for `alpn` /// does not match the passed type. pub fn get_protocol(&self, alpn: &[u8]) -> Option> { - self.protocols.get_typed(alpn) + self.router.get_protocol::

(alpn) } /// Registers the core iroh protocols (blobs, gossip, docs). @@ -898,7 +836,7 @@ impl ProtocolBuilder { store: D, gossip: Gossip, downloader: Downloader, - docs: Option, + docs: Option>, ) -> Self { // Register blobs. let blobs_proto = BlobsProtocol::new_with_events( @@ -906,6 +844,7 @@ impl ProtocolBuilder { self.local_pool_handle().clone(), blob_events, downloader, + self.endpoint().clone(), ); self = self.accept(iroh_blobs::protocol::ALPN.to_vec(), Arc::new(blobs_proto)); @@ -917,10 +856,6 @@ impl ProtocolBuilder { self = self.accept(DOCS_ALPN.to_vec(), Arc::new(docs)); } - if let Some(engine) = self.inner.willow.clone() { - self = self.accept(iroh_willow::ALPN.to_vec(), Arc::new(engine)); - } - self } @@ -930,24 +865,15 @@ impl ProtocolBuilder { inner, internal_rpc, external_rpc, - protocols, + router, gc_done_callback, gc_policy, nodes_data_path, local_pool: rt, } = self; - let protocols = Arc::new(protocols); let node_id = inner.endpoint.node_id(); - // Update the endpoint with our alpns. - let alpns = protocols - .alpns() - .map(|alpn| alpn.to_vec()) - .collect::>(); - if let Err(err) = inner.endpoint.set_alpns(alpns) { - inner.shutdown(protocols).await; - return Err(err); - } + let router = router.spawn().await?; // Spawn the main task and store it in the node for structured termination in shutdown. let fut = inner @@ -955,7 +881,7 @@ impl ProtocolBuilder { .run( external_rpc, internal_rpc, - protocols.clone(), + router.clone(), gc_policy, gc_done_callback, nodes_data_path, @@ -966,7 +892,7 @@ impl ProtocolBuilder { let node = Node { inner, - protocols, + router, task: AbortOnDropHandle::new(task) .map_err(Box::new(|e: JoinError| e.to_string()) as JoinErrToStr) .shared(), @@ -1024,7 +950,10 @@ pub const DEFAULT_RPC_ADDR: SocketAddr = fn make_rpc_endpoint( secret_key: &SecretKey, mut rpc_addr: SocketAddr, -) -> Result<(QuinnServerEndpoint, u16)> { +) -> Result<( + QuinnListener, + u16, +)> { let mut transport_config = quinn::TransportConfig::default(); transport_config .max_concurrent_bidi_streams(MAX_RPC_STREAMS.into()) @@ -1055,7 +984,7 @@ fn make_rpc_endpoint( }; let actual_rpc_port = rpc_quinn_endpoint.local_addr()?.port(); - let rpc_endpoint = QuinnServerEndpoint::::new(rpc_quinn_endpoint)?; + let rpc_endpoint = QuinnListener::new(rpc_quinn_endpoint)?; Ok((rpc_endpoint, actual_rpc_port)) } diff --git a/iroh/src/node/docs.rs b/iroh/src/node/docs.rs deleted file mode 100644 index be6400c8ca7..00000000000 --- a/iroh/src/node/docs.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::{ops::Deref, sync::Arc}; - -use anyhow::Result; -use futures_lite::future::Boxed as BoxedFuture; -use iroh_blobs::downloader::Downloader; -use iroh_docs::engine::{DefaultAuthorStorage, Engine}; -use iroh_gossip::net::Gossip; -use iroh_net::{endpoint::Connecting, Endpoint}; - -use crate::node::{DocsStorage, ProtocolHandler}; - -/// Wrapper around [`Engine`] so that we can implement our RPC methods directly. -#[derive(Debug, Clone)] -pub(crate) struct DocsEngine(Engine); - -impl DocsEngine { - pub async fn spawn( - storage: DocsStorage, - blobs_store: S, - default_author_storage: DefaultAuthorStorage, - endpoint: Endpoint, - gossip: Gossip, - downloader: Downloader, - ) -> anyhow::Result> { - let docs_store = match storage { - DocsStorage::Disabled => return Ok(None), - DocsStorage::Memory => iroh_docs::store::fs::Store::memory(), - DocsStorage::Persistent(path) => iroh_docs::store::fs::Store::persistent(path)?, - }; - let engine = Engine::spawn( - endpoint, - gossip, - docs_store, - blobs_store, - downloader, - default_author_storage, - ) - .await?; - Ok(Some(DocsEngine(engine))) - } -} - -impl Deref for DocsEngine { - type Target = Engine; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl ProtocolHandler for DocsEngine { - fn accept(self: Arc, conn: Connecting) -> BoxedFuture> { - Box::pin(async move { self.handle_connection(conn).await }) - } - - fn shutdown(self: Arc) -> BoxedFuture<()> { - Box::pin(async move { - let this: &Self = &self; - if let Err(err) = this.shutdown().await { - tracing::warn!("shutdown error: {:?}", err); - } - }) - } -} diff --git a/iroh/src/node/protocol.rs b/iroh/src/node/protocol.rs deleted file mode 100644 index c6a17b5c35f..00000000000 --- a/iroh/src/node/protocol.rs +++ /dev/null @@ -1,359 +0,0 @@ -use std::{any::Any, collections::BTreeMap, fmt, sync::Arc}; - -use anyhow::{anyhow, Result}; -use futures_lite::future::Boxed as BoxedFuture; -use futures_util::future::join_all; -use iroh_blobs::{ - downloader::{DownloadRequest, Downloader}, - get::{ - db::{DownloadProgress, GetState}, - Stats, - }, - provider::EventSender, - util::{ - local_pool::LocalPoolHandle, - progress::{AsyncChannelProgressSender, ProgressSender}, - SetTagOption, - }, - HashAndFormat, TempTag, -}; -use iroh_net::{endpoint::Connecting, Endpoint, NodeAddr}; -use tracing::{debug, warn}; - -use crate::{ - client::blobs::DownloadMode, - rpc_protocol::blobs::{BatchId, DownloadRequest as BlobDownloadRequest}, -}; - -/// Handler for incoming connections. -/// -/// An iroh node can accept connections for arbitrary ALPN protocols. By default, the iroh node -/// only accepts connections for the ALPNs of the core iroh protocols (blobs, gossip, docs). -/// -/// With this trait, you can handle incoming connections for custom protocols. -/// -/// Implement this trait on a struct that should handle incoming connections. -/// The protocol handler must then be registered on the node for an ALPN protocol with -/// [`crate::node::builder::ProtocolBuilder::accept`]. -pub trait ProtocolHandler: Send + Sync + IntoArcAny + fmt::Debug + 'static { - /// Handle an incoming connection. - /// - /// This runs on a freshly spawned tokio task so this can be long-running. - fn accept(self: Arc, conn: Connecting) -> BoxedFuture>; - - /// Called when the node shuts down. - fn shutdown(self: Arc) -> BoxedFuture<()> { - Box::pin(async move {}) - } -} - -/// Helper trait to facilite casting from `Arc` to `Arc`. -/// -/// This trait has a blanket implementation so there is no need to implement this yourself. -pub trait IntoArcAny { - fn into_arc_any(self: Arc) -> Arc; -} - -impl IntoArcAny for T { - fn into_arc_any(self: Arc) -> Arc { - self - } -} - -#[derive(Debug, Clone, Default)] -pub(super) struct ProtocolMap(BTreeMap, Arc>); - -impl ProtocolMap { - /// Returns the registered protocol handler for an ALPN as a concrete type. - pub(super) fn get_typed(&self, alpn: &[u8]) -> Option> { - let protocol: Arc = self.0.get(alpn)?.clone(); - let protocol_any: Arc = protocol.into_arc_any(); - let protocol_ref = Arc::downcast(protocol_any).ok()?; - Some(protocol_ref) - } - - /// Returns the registered protocol handler for an ALPN as a [`Arc`]. - pub(super) fn get(&self, alpn: &[u8]) -> Option> { - self.0.get(alpn).cloned() - } - - /// Inserts a protocol handler. - pub(super) fn insert(&mut self, alpn: Vec, handler: Arc) { - self.0.insert(alpn, handler); - } - - /// Returns an iterator of all registered ALPN protocol identifiers. - pub(super) fn alpns(&self) -> impl Iterator> { - self.0.keys() - } - - /// Shuts down all protocol handlers. - /// - /// Calls and awaits [`ProtocolHandler::shutdown`] for all registered handlers concurrently. - pub(super) async fn shutdown(&self) { - let handlers = self.0.values().cloned().map(ProtocolHandler::shutdown); - join_all(handlers).await; - } -} - -#[derive(Debug)] -pub(crate) struct BlobsProtocol { - rt: LocalPoolHandle, - store: S, - events: EventSender, - downloader: Downloader, - batches: tokio::sync::Mutex, -} - -/// Name used for logging when new node addresses are added from gossip. -const BLOB_DOWNLOAD_SOURCE_NAME: &str = "blob_download"; - -/// Keeps track of all the currently active batch operations of the blobs api. -#[derive(Debug, Default)] -pub(crate) struct BlobBatches { - /// Currently active batches - batches: BTreeMap, - /// Used to generate new batch ids. - max: u64, -} - -/// A single batch of blob operations -#[derive(Debug, Default)] -struct BlobBatch { - /// The tags in this batch. - tags: BTreeMap>, -} - -impl BlobBatches { - /// Create a new unique batch id. - pub(crate) fn create(&mut self) -> BatchId { - let id = self.max; - self.max += 1; - BatchId(id) - } - - /// Store a temp tag in a batch identified by a batch id. - pub(crate) fn store(&mut self, batch: BatchId, tt: TempTag) { - let entry = self.batches.entry(batch).or_default(); - entry.tags.entry(tt.hash_and_format()).or_default().push(tt); - } - - /// Remove a tag from a batch. - pub(crate) fn remove_one(&mut self, batch: BatchId, content: &HashAndFormat) -> Result<()> { - if let Some(batch) = self.batches.get_mut(&batch) { - if let Some(tags) = batch.tags.get_mut(content) { - tags.pop(); - if tags.is_empty() { - batch.tags.remove(content); - } - return Ok(()); - } - } - // this can happen if we try to upgrade a tag from an expired batch - anyhow::bail!("tag not found in batch"); - } - - /// Remove an entire batch. - pub(crate) fn remove(&mut self, batch: BatchId) { - self.batches.remove(&batch); - } -} - -impl BlobsProtocol { - pub(crate) fn new_with_events( - store: S, - rt: LocalPoolHandle, - events: EventSender, - downloader: Downloader, - ) -> Self { - Self { - rt, - store, - events, - downloader, - batches: Default::default(), - } - } - - pub(crate) fn store(&self) -> &S { - &self.store - } - - pub(crate) async fn batches(&self) -> tokio::sync::MutexGuard<'_, BlobBatches> { - self.batches.lock().await - } - - pub(crate) async fn download( - &self, - endpoint: Endpoint, - req: BlobDownloadRequest, - progress: AsyncChannelProgressSender, - ) -> Result<()> { - let BlobDownloadRequest { - hash, - format, - nodes, - tag, - mode, - } = req; - let hash_and_format = HashAndFormat { hash, format }; - let temp_tag = self.store.temp_tag(hash_and_format); - let stats = match mode { - DownloadMode::Queued => { - self.download_queued(endpoint, hash_and_format, nodes, progress.clone()) - .await? - } - DownloadMode::Direct => { - self.download_direct_from_nodes(endpoint, hash_and_format, nodes, progress.clone()) - .await? - } - }; - - progress.send(DownloadProgress::AllDone(stats)).await.ok(); - match tag { - SetTagOption::Named(tag) => { - self.store.set_tag(tag, Some(hash_and_format)).await?; - } - SetTagOption::Auto => { - self.store.create_tag(hash_and_format).await?; - } - } - drop(temp_tag); - - Ok(()) - } - - async fn download_queued( - &self, - endpoint: Endpoint, - hash_and_format: HashAndFormat, - nodes: Vec, - progress: AsyncChannelProgressSender, - ) -> Result { - let mut node_ids = Vec::with_capacity(nodes.len()); - let mut any_added = false; - for node in nodes { - node_ids.push(node.node_id); - if !node.info.is_empty() { - endpoint.add_node_addr_with_source(node, BLOB_DOWNLOAD_SOURCE_NAME)?; - any_added = true; - } - } - let can_download = !node_ids.is_empty() && (any_added || endpoint.discovery().is_some()); - anyhow::ensure!(can_download, "no way to reach a node for download"); - let req = DownloadRequest::new(hash_and_format, node_ids).progress_sender(progress); - let handle = self.downloader.queue(req).await; - let stats = handle.await?; - Ok(stats) - } - - #[tracing::instrument("download_direct", skip_all, fields(hash=%hash_and_format.hash.fmt_short()))] - async fn download_direct_from_nodes( - &self, - endpoint: Endpoint, - hash_and_format: HashAndFormat, - nodes: Vec, - progress: AsyncChannelProgressSender, - ) -> Result { - let mut last_err = None; - let mut remaining_nodes = nodes.len(); - let mut nodes_iter = nodes.into_iter(); - 'outer: loop { - match iroh_blobs::get::db::get_to_db_in_steps( - self.store.clone(), - hash_and_format, - progress.clone(), - ) - .await? - { - GetState::Complete(stats) => return Ok(stats), - GetState::NeedsConn(needs_conn) => { - let (conn, node_id) = 'inner: loop { - match nodes_iter.next() { - None => break 'outer, - Some(node) => { - remaining_nodes -= 1; - let node_id = node.node_id; - if node_id == endpoint.node_id() { - debug!( - ?remaining_nodes, - "skip node {} (it is the node id of ourselves)", - node_id.fmt_short() - ); - continue 'inner; - } - match endpoint.connect(node, iroh_blobs::protocol::ALPN).await { - Ok(conn) => break 'inner (conn, node_id), - Err(err) => { - debug!( - ?remaining_nodes, - "failed to connect to {}: {err}", - node_id.fmt_short() - ); - continue 'inner; - } - } - } - } - }; - match needs_conn.proceed(conn).await { - Ok(stats) => return Ok(stats), - Err(err) => { - warn!( - ?remaining_nodes, - "failed to download from {}: {err}", - node_id.fmt_short() - ); - last_err = Some(err); - } - } - } - } - } - match last_err { - Some(err) => Err(err.into()), - None => Err(anyhow!("No nodes to download from provided")), - } - } -} - -impl ProtocolHandler for BlobsProtocol { - fn accept(self: Arc, conn: Connecting) -> BoxedFuture> { - Box::pin(async move { - iroh_blobs::provider::handle_connection( - conn.await?, - self.store.clone(), - self.events.clone(), - self.rt.clone(), - ) - .await; - Ok(()) - }) - } - - fn shutdown(self: Arc) -> BoxedFuture<()> { - Box::pin(async move { - self.store.shutdown().await; - }) - } -} - -impl ProtocolHandler for iroh_gossip::net::Gossip { - fn accept(self: Arc, conn: Connecting) -> BoxedFuture> { - Box::pin(async move { self.handle_connection(conn.await?).await }) - } -} - -impl ProtocolHandler for iroh_willow::Engine { - fn accept(self: Arc, conn: Connecting) -> BoxedFuture> { - Box::pin(async move { self.handle_connection(conn.await?).await }) - } - - fn shutdown(self: Arc) -> BoxedFuture<()> { - Box::pin(async move { - if let Err(e) = (&**self).shutdown().await { - tracing::error!(?e, "Error while shutting down willow engine"); - } - }) - } -} diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index 9b99146297b..ae2a0ec0d78 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -1,152 +1,65 @@ -use std::{ - fmt::Debug, - io, - sync::{Arc, Mutex}, - time::Duration, -}; +use std::{fmt::Debug, sync::Arc, time::Duration}; -use anyhow::{anyhow, Result}; -use futures_buffered::BufferedStreamExt; -use futures_lite::{Stream, StreamExt}; -use futures_util::FutureExt; -use genawaiter::sync::{Co, Gen}; -use iroh_base::rpc::{RpcError, RpcResult}; +use anyhow::Result; +use futures_lite::Stream; use iroh_blobs::{ - export::ExportProgress, - format::collection::Collection, - get::db::DownloadProgress, - provider::{AddProgress, BatchAddPathProgress}, - store::{ - ConsistencyCheckProgress, ExportFormat, ImportProgress, MapEntry, Store as BaoStore, - ValidateProgress, - }, - util::{ - local_pool::LocalPoolHandle, - progress::{AsyncChannelProgressSender, ProgressSender}, - SetTagOption, - }, - BlobFormat, HashAndFormat, Tag, + net_protocol::Blobs as BlobsProtocol, store::Store as BaoStore, + util::local_pool::LocalPoolHandle, }; use iroh_docs::net::DOCS_ALPN; use iroh_gossip::net::{Gossip, GOSSIP_ALPN}; -use iroh_io::AsyncSliceReader; -use iroh_net::{relay::RelayUrl, NodeAddr, NodeId}; +use iroh_net::{NodeAddr, NodeId}; +use iroh_router::Router; use quic_rpc::server::{RpcChannel, RpcServerError}; use tokio::task::JoinSet; -use tokio_util::either::Either; use tracing::{debug, info, warn}; -use super::{protocol::ProtocolMap, IrohServerEndpoint}; +use super::IrohServerEndpoint; use crate::{ - client::{ - blobs::{BlobInfo, BlobStatus, IncompleteBlobInfo, WrapOption}, - tags::TagInfo, - NodeStatus, - }, - node::{docs::DocsEngine, protocol::BlobsProtocol, NodeInner}, + base::node_addr::RelayUrl, + client::NodeStatus, + node::NodeInner, rpc_protocol::{ - authors, blobs, - blobs::{ - AddPathRequest, AddPathResponse, AddStreamRequest, AddStreamResponse, AddStreamUpdate, - BatchAddPathRequest, BatchAddPathResponse, BatchAddStreamRequest, - BatchAddStreamResponse, BatchAddStreamUpdate, BatchCreateRequest, BatchCreateResponse, - BatchCreateTempTagRequest, BatchUpdate, BlobStatusRequest, BlobStatusResponse, - ConsistencyCheckRequest, CreateCollectionRequest, CreateCollectionResponse, - DeleteRequest, DownloadRequest as BlobDownloadRequest, DownloadResponse, ExportRequest, - ExportResponse, ListIncompleteRequest, ListRequest, ReadAtRequest, ReadAtResponse, - ValidateRequest, - }, - docs::{ - ExportFileRequest, ExportFileResponse, ImportFileRequest, ImportFileResponse, - Request as DocsRequest, SetHashRequest, - }, - gossip, net, net::{ - AddAddrRequest, AddrRequest, IdRequest, NodeWatchRequest, RelayRequest, + self, AddAddrRequest, AddrRequest, IdRequest, NodeWatchRequest, RelayRequest, RemoteInfoRequest, RemoteInfoResponse, RemoteInfosIterRequest, RemoteInfosIterResponse, WatchResponse, }, - node, - node::{ShutdownRequest, StatsRequest, StatsResponse, StatusRequest}, - tags, - tags::{DeleteRequest as TagDeleteRequest, ListRequest as ListTagsRequest, SyncMode}, + node::{self, ShutdownRequest, StatsRequest, StatsResponse, StatusRequest}, Request, RpcService, }, }; -mod docs; -mod spaces; - const HEALTH_POLL_WAIT: Duration = Duration::from_secs(1); -/// Chunk size for getting blobs over RPC -const RPC_BLOB_GET_CHUNK_SIZE: usize = 1024 * 64; -/// Channel cap for getting blobs over RPC -const RPC_BLOB_GET_CHANNEL_CAP: usize = 2; +pub(crate) type RpcError = serde_error::Error; +pub(crate) type RpcResult = Result; #[derive(Debug, Clone)] pub(crate) struct Handler { pub(crate) inner: Arc>, - pub(crate) protocols: Arc, + pub(crate) router: Router, } impl Handler { - pub fn new(inner: Arc>, protocols: Arc) -> Self { - Self { inner, protocols } + pub fn new(inner: Arc>, router: Router) -> Self { + Self { inner, router } } } impl Handler { - fn docs(&self) -> Option> { - self.protocols.get_typed::(DOCS_ALPN) - } - fn blobs(&self) -> Arc> { - self.protocols - .get_typed::>(iroh_blobs::protocol::ALPN) + self.router + .get_protocol::>(iroh_blobs::protocol::ALPN) .expect("missing blobs") } - fn blobs_store(&self) -> D { - self.blobs().store().clone() - } - - fn spaces(&self) -> Result<&iroh_willow::Engine, RpcError> { - self.inner.willow.as_ref().ok_or_else(spaces_disabled) - } - - async fn with_docs(self, f: F) -> RpcResult - where - T: Send + 'static, - F: FnOnce(Arc) -> Fut, - Fut: std::future::Future>, - { - if let Some(docs) = self.docs() { - f(docs).await - } else { - Err(docs_disabled()) - } - } - - fn with_docs_stream(self, f: F) -> impl Stream> - where - T: Send + 'static, - F: FnOnce(Arc) -> S, - S: Stream>, - { - if let Some(docs) = self.docs() { - Either::Left(f(docs)) - } else { - Either::Right(futures_lite::stream::once(Err(docs_disabled()))) - } - } - pub(crate) fn spawn_rpc_request( inner: Arc>, join_set: &mut JoinSet>, accepting: quic_rpc::server::Accepting, - protocols: Arc, + router: Router, ) { - let handler = Self::new(inner, protocols); + let handler = Self::new(inner, router); join_set.spawn(async move { let (msg, chan) = accepting.read_first().await?; if let Err(err) = handler.handle_rpc_request(msg, chan).await { @@ -191,275 +104,50 @@ impl Handler { } } - async fn handle_blobs_request( + async fn handle_blobs_and_tags_request( self, - msg: blobs::Request, - chan: RpcChannel, + msg: iroh_blobs::rpc::proto::Request, + chan: RpcChannel, ) -> Result<(), RpcServerError> { - use blobs::Request::*; - debug!("handling blob request: {msg}"); - match msg { - List(msg) => chan.server_streaming(msg, self, Self::blob_list).await, - ListIncomplete(msg) => { - chan.server_streaming(msg, self, Self::blob_list_incomplete) - .await - } - CreateCollection(msg) => chan.rpc(msg, self, Self::create_collection).await, - Delete(msg) => chan.rpc(msg, self, Self::blob_delete_blob).await, - AddPath(msg) => { - chan.server_streaming(msg, self, Self::blob_add_from_path) - .await - } - Download(msg) => chan.server_streaming(msg, self, Self::blob_download).await, - Export(msg) => chan.server_streaming(msg, self, Self::blob_export).await, - Validate(msg) => chan.server_streaming(msg, self, Self::blob_validate).await, - Fsck(msg) => { - chan.server_streaming(msg, self, Self::blob_consistency_check) - .await - } - ReadAt(msg) => chan.server_streaming(msg, self, Self::blob_read_at).await, - AddStream(msg) => chan.bidi_streaming(msg, self, Self::blob_add_stream).await, - AddStreamUpdate(_msg) => Err(RpcServerError::UnexpectedUpdateMessage), - BlobStatus(msg) => chan.rpc(msg, self, Self::blob_status).await, - BatchCreate(msg) => chan.bidi_streaming(msg, self, Self::batch_create).await, - BatchUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - BatchAddStream(msg) => chan.bidi_streaming(msg, self, Self::batch_add_stream).await, - BatchAddStreamUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - BatchAddPath(msg) => { - chan.server_streaming(msg, self, Self::batch_add_from_path) - .await - } - BatchCreateTempTag(msg) => chan.rpc(msg, self, Self::batch_create_temp_tag).await, - } - } - - async fn handle_tags_request( - self, - msg: tags::Request, - chan: RpcChannel, - ) -> Result<(), RpcServerError> { - use tags::Request::*; - match msg { - ListTags(msg) => chan.server_streaming(msg, self, Self::blob_list_tags).await, - DeleteTag(msg) => chan.rpc(msg, self, Self::blob_delete_tag).await, - Create(msg) => chan.rpc(msg, self, Self::tags_create).await, - Set(msg) => chan.rpc(msg, self, Self::tags_set).await, - } + self.blobs() + .handle_rpc_request(msg, chan) + .await + .map_err(|e| e.errors_into()) } async fn handle_gossip_request( self, - msg: gossip::Request, - chan: RpcChannel, - ) -> Result<(), RpcServerError> { - use gossip::Request::*; - match msg { - Subscribe(msg) => { - chan.bidi_streaming(msg, self, |handler, req, updates| { - let stream = handler - .protocols - .get_typed::(GOSSIP_ALPN) - .expect("missing gossip") - .join_with_stream( - req.topic, - iroh_gossip::net::JoinOptions { - bootstrap: req.bootstrap, - subscription_capacity: req.subscription_capacity, - }, - Box::pin(updates), - ); - futures_util::TryStreamExt::map_err(stream, RpcError::from) - }) - .await - } - Update(_msg) => Err(RpcServerError::UnexpectedUpdateMessage), - } - } - - async fn handle_authors_request( - self, - msg: authors::Request, + msg: iroh_gossip::RpcRequest, chan: RpcChannel, ) -> Result<(), RpcServerError> { - use authors::Request::*; - match msg { - List(msg) => { - chan.server_streaming(msg, self, |handler, req: authors::ListRequest| { - handler.with_docs_stream(|docs| docs.author_list(req)) - }) - .await - } - Create(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.author_create(req).await }) - }) - .await - } - Import(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.author_import(req).await }) - }) - .await - } - Export(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.author_export(req).await }) - }) - .await - } - Delete(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.author_delete(req).await }) - }) - .await - } - GetDefault(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { Ok(docs.author_default(req)) }) - }) - .await - } - SetDefault(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.author_set_default(req).await }) - }) - .await - } - } + let gossip = self + .router + .get_protocol::(GOSSIP_ALPN) + .expect("missing gossip"); + let chan = chan.map::(); + gossip + .handle_rpc_request(msg, chan) + .await + .map_err(|e| e.errors_into()) } async fn handle_docs_request( self, - msg: DocsRequest, + msg: iroh_docs::rpc::proto::Request, chan: RpcChannel, ) -> Result<(), RpcServerError> { - use DocsRequest::*; - match msg { - Open(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_open(req).await }) - }) + if let Some(docs) = self + .router + .get_protocol::>(DOCS_ALPN) + { + let chan = chan.map::(); + docs.handle_rpc_request(msg, chan) .await - } - Close(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_close(req).await }) - }) - .await - } - Status(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_status(req).await }) - }) - .await - } - List(msg) => { - chan.server_streaming(msg, self, |handler, req| { - handler.with_docs_stream(|docs| docs.doc_list(req)) - }) - .await - } - Create(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_create(req).await }) - }) - .await - } - Drop(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_drop(req).await }) - }) - .await - } - Import(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_import(req).await }) - }) - .await - } - Set(msg) => { - let blobs_store = self.blobs_store(); - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_set(&blobs_store, req).await }) - }) - .await - } - ImportFile(msg) => { - chan.server_streaming(msg, self, Self::doc_import_file) - .await - } - ExportFile(msg) => { - chan.server_streaming(msg, self, Self::doc_export_file) - .await - } - Del(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_del(req).await }) - }) - .await - } - SetHash(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_set_hash(req).await }) - }) - .await - } - Get(msg) => { - chan.server_streaming(msg, self, |handler, req| { - handler.with_docs_stream(|docs| docs.doc_get_many(req)) - }) - .await - } - GetExact(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_get_exact(req).await }) - }) - .await - } - StartSync(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_start_sync(req).await }) - }) - .await - } - Leave(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_leave(req).await }) - }) - .await - } - Share(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_share(req).await }) - }) - .await - } - Subscribe(msg) => { - chan.try_server_streaming(msg, self, |handler, req| async move { - handler - .with_docs(|docs| async move { docs.doc_subscribe(req).await }) - .await - }) - .await - } - SetDownloadPolicy(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_set_download_policy(req).await }) - }) - .await - } - GetDownloadPolicy(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_get_download_policy(req).await }) - }) - .await - } - GetSyncPeers(msg) => { - chan.rpc(msg, self, |handler, req| { - handler.with_docs(|docs| async move { docs.doc_get_sync_peers(req).await }) - }) - .await - } + .map_err(|e| e.errors_into()) + } else { + Err(RpcServerError::SendError(anyhow::anyhow!( + "Docs is not enabled" + ))) } } @@ -473,491 +161,36 @@ impl Handler { match msg { Net(msg) => self.handle_net_request(msg, chan).await, Node(msg) => self.handle_node_request(msg, chan).await, - Blobs(msg) => self.handle_blobs_request(msg, chan).await, - Tags(msg) => self.handle_tags_request(msg, chan).await, - Authors(msg) => self.handle_authors_request(msg, chan).await, + BlobsAndTags(msg) => { + self.handle_blobs_and_tags_request(msg, chan.map().boxed()) + .await + } Docs(msg) => self.handle_docs_request(msg, chan).await, Gossip(msg) => self.handle_gossip_request(msg, chan).await, - Spaces(msg) => self.handle_spaces_request(msg, chan).await, } } - fn local_pool_handle(&self) -> LocalPoolHandle { - self.inner.local_pool_handle.clone() - } - - async fn blob_status(self, msg: BlobStatusRequest) -> RpcResult { - let blobs = self.blobs(); - let entry = blobs.store().get(&msg.hash).await?; - Ok(BlobStatusResponse(match entry { - Some(entry) => { - if entry.is_complete() { - BlobStatus::Complete { - size: entry.size().value(), - } - } else { - BlobStatus::Partial { size: entry.size() } - } - } - None => BlobStatus::NotFound, - })) - } - - async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { - use bao_tree::io::fsm::Outboard; - - let blobs = self.blobs(); - let db = blobs.store(); - for blob in db.blobs().await? { - let blob = blob?; - let Some(entry) = db.get(&blob).await? else { - continue; - }; - let hash = entry.hash(); - let size = entry.outboard().await?.tree().size(); - let path = "".to_owned(); - co.yield_(Ok(BlobInfo { hash, size, path })).await; - } - Ok(()) - } - - async fn blob_list_incomplete_impl( - self, - co: &Co>, - ) -> io::Result<()> { - let blobs = self.blobs(); - let db = blobs.store(); - for hash in db.partial_blobs().await? { - let hash = hash?; - let Ok(Some(entry)) = db.get_mut(&hash).await else { - continue; - }; - if entry.is_complete() { - continue; - } - let size = 0; - let expected_size = entry.size().value(); - co.yield_(Ok(IncompleteBlobInfo { - hash, - size, - expected_size, - })) - .await; - } - Ok(()) - } - - fn blob_list( - self, - _msg: ListRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(|co| async move { - if let Err(e) = self.blob_list_impl(&co).await { - co.yield_(Err(e.into())).await; - } - }) - } - - fn blob_list_incomplete( - self, - _msg: ListIncompleteRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(move |co| async move { - if let Err(e) = self.blob_list_incomplete_impl(&co).await { - co.yield_(Err(e.into())).await; - } - }) - } - - async fn blob_delete_tag(self, msg: TagDeleteRequest) -> RpcResult<()> { - self.blobs_store().set_tag(msg.name, None).await?; - Ok(()) - } - - async fn blob_delete_blob(self, msg: DeleteRequest) -> RpcResult<()> { - self.blobs_store().delete(vec![msg.hash]).await?; - Ok(()) - } - - fn blob_list_tags(self, msg: ListTagsRequest) -> impl Stream + Send + 'static { - tracing::info!("blob_list_tags"); - let blobs = self.blobs(); - Gen::new(|co| async move { - let tags = blobs.store().tags().await.unwrap(); - #[allow(clippy::manual_flatten)] - for item in tags { - if let Ok((name, HashAndFormat { hash, format })) = item { - if (format.is_raw() && msg.raw) || (format.is_hash_seq() && msg.hash_seq) { - co.yield_(TagInfo { name, hash, format }).await; - } - } - } - }) - } - - /// Invoke validate on the database and stream out the result - fn blob_validate( - self, - msg: ValidateRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .validate(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ValidateProgress::Abort(e.into())).await.ok(); - } - }); - rx - } - - /// Invoke validate on the database and stream out the result - fn blob_consistency_check( - self, - msg: ConsistencyCheckRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .consistency_check(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ConsistencyCheckProgress::Abort(e.into())) - .await - .ok(); - } - }); - rx - } - - fn blob_add_from_path(self, msg: AddPathRequest) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.blob_add_from_path0(msg, tx).await { - tx2.send(AddProgress::Abort(e.into())).await.ok(); - } - }); - rx.map(AddPathResponse) - } - - fn doc_import_file(self, msg: ImportFileRequest) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.doc_import_file0(msg, tx).await { - tx2.send(crate::client::docs::ImportProgress::Abort(e.into())) - .await - .ok(); - } - }); - rx.map(ImportFileResponse) - } - - async fn doc_import_file0( - self, - msg: ImportFileRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let docs = self.docs().ok_or_else(|| anyhow!("docs are disabled"))?; - use std::collections::BTreeMap; - - use iroh_blobs::store::ImportMode; - - use crate::client::docs::ImportProgress as DocImportProgress; - - let progress = AsyncChannelProgressSender::new(progress); - let names = Arc::new(Mutex::new(BTreeMap::new())); - // convert import progress to provide progress - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Found { id, name } => { - names.lock().unwrap().insert(id, name); - None - } - ImportProgress::Size { id, size } => { - let name = names.lock().unwrap().remove(&id)?; - Some(DocImportProgress::Found { id, name, size }) - } - ImportProgress::OutboardProgress { id, offset } => { - Some(DocImportProgress::Progress { id, offset }) - } - ImportProgress::OutboardDone { hash, id } => { - Some(DocImportProgress::IngestDone { hash, id }) - } - _ => None, - }); - let ImportFileRequest { - doc_id, - author_id, - key, - path: root, - in_place, - } = msg; - // Check that the path is absolute and exists. - anyhow::ensure!(root.is_absolute(), "path must be absolute"); - anyhow::ensure!( - root.exists(), - "trying to add missing path: {}", - root.display() - ); - - let import_mode = match in_place { - true => ImportMode::TryReference, - false => ImportMode::Copy, - }; - - let blobs = self.blobs(); - let (temp_tag, size) = blobs - .store() - .import_file(root, import_mode, BlobFormat::Raw, import_progress) - .await?; - - let hash_and_format = temp_tag.inner(); - let HashAndFormat { hash, .. } = *hash_and_format; - docs.doc_set_hash(SetHashRequest { - doc_id, - author_id, - key: key.clone(), - hash, - size, - }) - .await?; - drop(temp_tag); - progress.send(DocImportProgress::AllDone { key }).await?; - Ok(()) - } - - fn doc_export_file(self, msg: ExportFileRequest) -> impl Stream { - let (tx, rx) = async_channel::bounded(1024); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.doc_export_file0(msg, tx).await { - tx2.send(ExportProgress::Abort(e.into())).await.ok(); - } - }); - rx.map(ExportFileResponse) - } - - async fn doc_export_file0( - self, - msg: ExportFileRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let _docs = self.docs().ok_or_else(|| anyhow!("docs are disabled"))?; - let progress = AsyncChannelProgressSender::new(progress); - let ExportFileRequest { entry, path, mode } = msg; - let key = bytes::Bytes::from(entry.key().to_vec()); - let export_progress = progress.clone().with_map(move |mut x| { - // assign the doc key to the `meta` field of the initial progress event - if let ExportProgress::Found { meta, .. } = &mut x { - *meta = Some(key.clone()) - } - x - }); - let blobs = self.blobs(); - iroh_blobs::export::export( - blobs.store(), - entry.content_hash(), - path, - ExportFormat::Blob, - mode, - export_progress, - ) - .await?; - progress.send(ExportProgress::AllDone).await?; - Ok(()) - } - - fn blob_download(self, msg: BlobDownloadRequest) -> impl Stream { - let (sender, receiver) = async_channel::bounded(1024); - let endpoint = self.inner.endpoint.clone(); - let progress = AsyncChannelProgressSender::new(sender); - - let blobs_protocol = self - .protocols - .get_typed::>(iroh_blobs::protocol::ALPN) - .expect("missing blobs"); - - self.local_pool_handle().spawn_detached(move || async move { - if let Err(err) = blobs_protocol - .download(endpoint, msg, progress.clone()) - .await - { - progress - .send(DownloadProgress::Abort(err.into())) - .await - .ok(); - } - }); - - receiver.map(DownloadResponse) - } - - fn blob_export(self, msg: ExportRequest) -> impl Stream { - let (tx, rx) = async_channel::bounded(1024); - let progress = AsyncChannelProgressSender::new(tx); - self.local_pool_handle().spawn_detached(move || async move { - let res = iroh_blobs::export::export( - self.blobs().store(), - msg.hash, - msg.path, - msg.format, - msg.mode, - progress.clone(), - ) - .await; - match res { - Ok(()) => progress.send(ExportProgress::AllDone).await.ok(), - Err(err) => progress.send(ExportProgress::Abort(err.into())).await.ok(), - }; - }); - rx.map(ExportResponse) - } - - async fn blob_add_from_path0( - self, - msg: AddPathRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - use std::collections::BTreeMap; - - use iroh_blobs::store::ImportMode; - - let blobs = self.blobs(); - let progress = AsyncChannelProgressSender::new(progress); - let names = Arc::new(Mutex::new(BTreeMap::new())); - // convert import progress to provide progress - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Found { id, name } => { - names.lock().unwrap().insert(id, name); - None - } - ImportProgress::Size { id, size } => { - let name = names.lock().unwrap().remove(&id)?; - Some(AddProgress::Found { id, name, size }) - } - ImportProgress::OutboardProgress { id, offset } => { - Some(AddProgress::Progress { id, offset }) - } - ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }), - _ => None, - }); - let AddPathRequest { - wrap, - path: root, - in_place, - tag, - } = msg; - // Check that the path is absolute and exists. - anyhow::ensure!(root.is_absolute(), "path must be absolute"); - anyhow::ensure!( - root.exists(), - "trying to add missing path: {}", - root.display() - ); - - let import_mode = match in_place { - true => ImportMode::TryReference, - false => ImportMode::Copy, - }; - - let create_collection = match wrap { - WrapOption::Wrap { .. } => true, - WrapOption::NoWrap => root.is_dir(), - }; - - let temp_tag = if create_collection { - // import all files below root recursively - let data_sources = crate::util::fs::scan_path(root, wrap)?; - let blobs = self.blobs(); - - const IO_PARALLELISM: usize = 4; - let result: Vec<_> = futures_lite::stream::iter(data_sources) - .map(|source| { - let import_progress = import_progress.clone(); - let blobs = blobs.clone(); - async move { - let name = source.name().to_string(); - let (tag, size) = blobs - .store() - .import_file( - source.path().to_owned(), - import_mode, - BlobFormat::Raw, - import_progress, - ) - .await?; - let hash = *tag.hash(); - io::Result::Ok((name, hash, size, tag)) - } - }) - .buffered_ordered(IO_PARALLELISM) - .try_collect() - .await?; - - // create a collection - let (collection, _child_tags): (Collection, Vec<_>) = result - .into_iter() - .map(|(name, hash, _, tag)| ((name, hash), tag)) - .unzip(); - - collection.store(blobs.store()).await? - } else { - // import a single file - let (tag, _size) = blobs - .store() - .import_file(root, import_mode, BlobFormat::Raw, import_progress) - .await?; - tag - }; - - let hash_and_format = temp_tag.inner(); - let HashAndFormat { hash, format } = *hash_and_format; - let tag = match tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(*hash_and_format)) - .await?; - tag - } - SetTagOption::Auto => blobs.store().create_tag(*hash_and_format).await?, - }; - progress - .send(AddProgress::AllDone { - hash, - format, - tag: tag.clone(), - }) - .await?; - Ok(()) - } - #[allow(clippy::unused_async)] async fn node_stats(self, _req: StatsRequest) -> RpcResult { #[cfg(feature = "metrics")] let res = Ok(StatsResponse { - stats: crate::metrics::get_metrics()?, + stats: crate::metrics::get_metrics().map_err(|e| RpcError::new(&*e))?, }); #[cfg(not(feature = "metrics"))] - let res = Err(anyhow::anyhow!("metrics are disabled").into()); + let res = Err(RpcError::new(&*anyhow::anyhow!("metrics are disabled"))); res } async fn node_status(self, _: StatusRequest) -> RpcResult { Ok(NodeStatus { - addr: self.inner.endpoint.node_addr().await?, + addr: self + .inner + .endpoint + .node_addr() + .await + .map_err(|e| RpcError::new(&*e))?, listen_addrs: self .inner .local_endpoint_addresses() @@ -974,7 +207,12 @@ impl Handler { } async fn node_addr(self, _: AddrRequest) -> RpcResult { - let addr = self.inner.endpoint.node_addr().await?; + let addr = self + .inner + .endpoint + .node_addr() + .await + .map_err(|e| RpcError::new(&*e))?; Ok(addr) } @@ -995,32 +233,6 @@ impl Handler { } } - async fn tags_set(self, msg: tags::SetRequest) -> RpcResult<()> { - let blobs = self.blobs(); - blobs.store().set_tag(msg.name, msg.value).await?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await?; - } - if let Some(batch) = msg.batch { - if let Some(content) = msg.value.as_ref() { - blobs.batches().await.remove_one(batch, content)?; - } - } - Ok(()) - } - - async fn tags_create(self, msg: tags::CreateRequest) -> RpcResult { - let blobs = self.blobs(); - let tag = blobs.store().create_tag(msg.value).await?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await?; - } - if let Some(batch) = msg.batch { - blobs.batches().await.remove_one(batch, &msg.value)?; - } - Ok(tag) - } - fn node_watch(self, _: NodeWatchRequest) -> impl Stream { futures_lite::stream::unfold((), |()| async move { tokio::time::sleep(HEALTH_POLL_WAIT).await; @@ -1033,296 +245,8 @@ impl Handler { }) } - async fn batch_create_temp_tag(self, msg: BatchCreateTempTagRequest) -> RpcResult<()> { - let blobs = self.blobs(); - let tag = blobs.store().temp_tag(msg.content); - blobs.batches().await.store(msg.batch, tag); - Ok(()) - } - - fn batch_add_stream( - self, - msg: BatchAddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let (tx, rx) = async_channel::bounded(32); - let this = self.clone(); - - self.local_pool_handle().spawn_detached(|| async move { - if let Err(err) = this.batch_add_stream0(msg, stream, tx.clone()).await { - tx.send(BatchAddStreamResponse::Abort(err.into())) - .await - .ok(); - } - }); - rx - } - - fn batch_add_from_path( - self, - msg: BatchAddPathRequest, - ) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.batch_add_from_path0(msg, tx).await { - tx2.send(BatchAddPathProgress::Abort(e.into())).await.ok(); - } - }); - rx.map(BatchAddPathResponse) - } - - async fn batch_add_stream0( - self, - msg: BatchAddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let blobs = self.blobs(); - let progress = AsyncChannelProgressSender::new(progress); - - let stream = stream.map(|item| match item { - BatchAddStreamUpdate::Chunk(chunk) => Ok(chunk), - BatchAddStreamUpdate::Abort => { - Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort")) - } - }); - - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::OutboardProgress { offset, .. } => { - Some(BatchAddStreamResponse::OutboardProgress { offset }) - } - _ => None, - }); - let (temp_tag, _len) = blobs - .store() - .import_stream(stream, msg.format, import_progress) - .await?; - let hash = temp_tag.inner().hash; - blobs.batches().await.store(msg.batch, temp_tag); - progress - .send(BatchAddStreamResponse::Result { hash }) - .await?; - Ok(()) - } - - async fn batch_add_from_path0( - self, - msg: BatchAddPathRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let progress = AsyncChannelProgressSender::new(progress); - // convert import progress to provide progress - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Size { size, .. } => Some(BatchAddPathProgress::Found { size }), - ImportProgress::OutboardProgress { offset, .. } => { - Some(BatchAddPathProgress::Progress { offset }) - } - ImportProgress::OutboardDone { hash, .. } => Some(BatchAddPathProgress::Done { hash }), - _ => None, - }); - let BatchAddPathRequest { - path: root, - import_mode, - format, - batch, - } = msg; - // Check that the path is absolute and exists. - anyhow::ensure!(root.is_absolute(), "path must be absolute"); - anyhow::ensure!( - root.exists(), - "trying to add missing path: {}", - root.display() - ); - let blobs = self.blobs(); - let (tag, _) = blobs - .store() - .import_file(root, import_mode, format, import_progress) - .await?; - let hash = *tag.hash(); - blobs.batches().await.store(batch, tag); - - progress.send(BatchAddPathProgress::Done { hash }).await?; - Ok(()) - } - - fn blob_add_stream( - self, - msg: AddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let (tx, rx) = async_channel::bounded(32); - let this = self.clone(); - - self.local_pool_handle().spawn_detached(|| async move { - if let Err(err) = this.blob_add_stream0(msg, stream, tx.clone()).await { - tx.send(AddProgress::Abort(err.into())).await.ok(); - } - }); - - rx.map(AddStreamResponse) - } - - async fn blob_add_stream0( - self, - msg: AddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let progress = AsyncChannelProgressSender::new(progress); - - let stream = stream.map(|item| match item { - AddStreamUpdate::Chunk(chunk) => Ok(chunk), - AddStreamUpdate::Abort => { - Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort")) - } - }); - - let name_cache = Arc::new(Mutex::new(None)); - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Found { id: _, name } => { - let _ = name_cache.lock().unwrap().insert(name); - None - } - ImportProgress::Size { id, size } => { - let name = name_cache.lock().unwrap().take()?; - Some(AddProgress::Found { id, name, size }) - } - ImportProgress::OutboardProgress { id, offset } => { - Some(AddProgress::Progress { id, offset }) - } - ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }), - _ => None, - }); - let blobs = self.blobs(); - let (temp_tag, _len) = blobs - .store() - .import_stream(stream, BlobFormat::Raw, import_progress) - .await?; - let hash_and_format = *temp_tag.inner(); - let HashAndFormat { hash, format } = hash_and_format; - let tag = match msg.tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(hash_and_format)) - .await?; - tag - } - SetTagOption::Auto => blobs.store().create_tag(hash_and_format).await?, - }; - progress - .send(AddProgress::AllDone { hash, tag, format }) - .await?; - Ok(()) - } - - fn blob_read_at( - self, - req: ReadAtRequest, - ) -> impl Stream> + Send + 'static { - let (tx, rx) = async_channel::bounded(RPC_BLOB_GET_CHANNEL_CAP); - let db = self.blobs_store(); - self.local_pool_handle().spawn_detached(move || async move { - if let Err(err) = read_loop(req, db, tx.clone(), RPC_BLOB_GET_CHUNK_SIZE).await { - tx.send(RpcResult::Err(err.into())).await.ok(); - } - }); - - async fn read_loop( - req: ReadAtRequest, - db: D, - tx: async_channel::Sender>, - max_chunk_size: usize, - ) -> anyhow::Result<()> { - let entry = db.get(&req.hash).await?; - let entry = entry.ok_or_else(|| anyhow!("Blob not found"))?; - let size = entry.size(); - - anyhow::ensure!( - req.offset <= size.value(), - "requested offset is out of range: {} > {:?}", - req.offset, - size - ); - - let len: usize = req - .len - .as_result_len(size.value() - req.offset) - .try_into()?; - - anyhow::ensure!( - req.offset + len as u64 <= size.value(), - "requested range is out of bounds: offset: {}, len: {} > {:?}", - req.offset, - len, - size - ); - - tx.send(Ok(ReadAtResponse::Entry { - size, - is_complete: entry.is_complete(), - })) - .await?; - let mut reader = entry.data_reader().await?; - - let (num_chunks, chunk_size) = if len <= max_chunk_size { - (1, len) - } else { - let num_chunks = len / max_chunk_size + (len % max_chunk_size != 0) as usize; - (num_chunks, max_chunk_size) - }; - - let mut read = 0u64; - for i in 0..num_chunks { - let chunk_size = if i == num_chunks - 1 { - // last chunk might be smaller - len - read as usize - } else { - chunk_size - }; - let chunk = reader.read_at(req.offset + read, chunk_size).await?; - let chunk_len = chunk.len(); - if !chunk.is_empty() { - tx.send(Ok(ReadAtResponse::Data { chunk })).await?; - } - if chunk_len < chunk_size { - break; - } else { - read += chunk_len as u64; - } - } - Ok(()) - } - - rx - } - - fn batch_create( - self, - _: BatchCreateRequest, - mut updates: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let blobs = self.blobs(); - async move { - let batch = blobs.batches().await.create(); - tokio::spawn(async move { - while let Some(item) = updates.next().await { - match item { - BatchUpdate::Drop(content) => { - // this can not fail, since we keep the batch alive. - // therefore it is safe to ignore the result. - let _ = blobs.batches().await.remove_one(batch, &content); - } - BatchUpdate::Ping => {} - } - } - blobs.batches().await.remove(batch); - }); - BatchCreateResponse::Id(batch) - } - .into_stream() + fn local_pool_handle(&self) -> LocalPoolHandle { + self.inner.local_pool_handle.clone() } fn remote_infos_iter( @@ -1353,48 +277,10 @@ impl Handler { #[allow(clippy::unused_async)] async fn node_add_addr(self, req: AddAddrRequest) -> RpcResult<()> { let AddAddrRequest { addr } = req; - self.inner.endpoint.add_node_addr(addr)?; + self.inner + .endpoint + .add_node_addr(addr) + .map_err(|e| RpcError::new(&*e))?; Ok(()) } - - async fn create_collection( - self, - req: CreateCollectionRequest, - ) -> RpcResult { - let CreateCollectionRequest { - collection, - tag, - tags_to_delete, - } = req; - - let blobs = self.blobs(); - - let temp_tag = collection.store(blobs.store()).await?; - let hash_and_format = temp_tag.inner(); - let HashAndFormat { hash, .. } = *hash_and_format; - let tag = match tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(*hash_and_format)) - .await?; - tag - } - SetTagOption::Auto => blobs.store().create_tag(*hash_and_format).await?, - }; - - for tag in tags_to_delete { - blobs.store().set_tag(tag, None).await?; - } - - Ok(CreateCollectionResponse { hash, tag }) - } -} - -fn docs_disabled() -> RpcError { - anyhow!("docs are disabled").into() -} - -fn spaces_disabled() -> RpcError { - anyhow::anyhow!("spaces are disabled").into() } diff --git a/iroh/src/node/rpc/docs.rs b/iroh/src/node/rpc/docs.rs deleted file mode 100644 index a777aff4523..00000000000 --- a/iroh/src/node/rpc/docs.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! This module contains an impl block on [`DocsEngine`] with handlers for RPC requests - -use anyhow::anyhow; -use futures_lite::{Stream, StreamExt}; -use iroh_base::rpc::RpcResult; -use iroh_blobs::{store::Store as BaoStore, BlobFormat}; -use iroh_docs::{Author, DocTicket, NamespaceSecret}; - -use crate::{ - client::docs::ShareMode, - node::DocsEngine, - rpc_protocol::{ - authors::{ - CreateRequest, CreateResponse, DeleteRequest, DeleteResponse, ExportRequest, - ExportResponse, GetDefaultRequest, GetDefaultResponse, ImportRequest, ImportResponse, - ListRequest as AuthorListRequest, ListResponse as AuthorListResponse, - SetDefaultRequest, SetDefaultResponse, - }, - docs::{ - CloseRequest, CloseResponse, CreateRequest as DocCreateRequest, - CreateResponse as DocCreateResponse, DelRequest, DelResponse, DocListRequest, - DocSubscribeRequest, DocSubscribeResponse, DropRequest, DropResponse, - GetDownloadPolicyRequest, GetDownloadPolicyResponse, GetExactRequest, GetExactResponse, - GetManyRequest, GetManyResponse, GetSyncPeersRequest, GetSyncPeersResponse, - ImportRequest as DocImportRequest, ImportResponse as DocImportResponse, LeaveRequest, - LeaveResponse, ListResponse as DocListResponse, OpenRequest, OpenResponse, - SetDownloadPolicyRequest, SetDownloadPolicyResponse, SetHashRequest, SetHashResponse, - SetRequest, SetResponse, ShareRequest, ShareResponse, StartSyncRequest, - StartSyncResponse, StatusRequest, StatusResponse, - }, - }, -}; - -/// Capacity for the flume channels to forward sync store iterators to async RPC streams. -const ITER_CHANNEL_CAP: usize = 64; - -#[allow(missing_docs)] -impl DocsEngine { - pub async fn author_create(&self, _req: CreateRequest) -> RpcResult { - // TODO: pass rng - let author = Author::new(&mut rand::rngs::OsRng {}); - self.sync.import_author(author.clone()).await?; - Ok(CreateResponse { - author_id: author.id(), - }) - } - - pub fn author_default(&self, _req: GetDefaultRequest) -> GetDefaultResponse { - let author_id = self.default_author.get(); - GetDefaultResponse { author_id } - } - - pub async fn author_set_default( - &self, - req: SetDefaultRequest, - ) -> RpcResult { - self.default_author.set(req.author_id, &self.sync).await?; - Ok(SetDefaultResponse) - } - - pub fn author_list( - &self, - _req: AuthorListRequest, - ) -> impl Stream> + Unpin { - let (tx, rx) = async_channel::bounded(ITER_CHANNEL_CAP); - let sync = self.sync.clone(); - // we need to spawn a task to send our request to the sync handle, because the method - // itself must be sync. - tokio::task::spawn(async move { - let tx2 = tx.clone(); - if let Err(err) = sync.list_authors(tx).await { - tx2.send(Err(err)).await.ok(); - } - }); - rx.boxed().map(|r| { - r.map(|author_id| AuthorListResponse { author_id }) - .map_err(Into::into) - }) - } - - pub async fn author_import(&self, req: ImportRequest) -> RpcResult { - let author_id = self.sync.import_author(req.author).await?; - Ok(ImportResponse { author_id }) - } - - pub async fn author_export(&self, req: ExportRequest) -> RpcResult { - let author = self.sync.export_author(req.author).await?; - - Ok(ExportResponse { author }) - } - - pub async fn author_delete(&self, req: DeleteRequest) -> RpcResult { - if req.author == self.default_author.get() { - return Err(anyhow!("Deleting the default author is not supported").into()); - } - self.sync.delete_author(req.author).await?; - Ok(DeleteResponse) - } - - pub async fn doc_create(&self, _req: DocCreateRequest) -> RpcResult { - let namespace = NamespaceSecret::new(&mut rand::rngs::OsRng {}); - let id = namespace.id(); - self.sync.import_namespace(namespace.into()).await?; - self.sync.open(id, Default::default()).await?; - Ok(DocCreateResponse { id }) - } - - pub async fn doc_drop(&self, req: DropRequest) -> RpcResult { - let DropRequest { doc_id } = req; - self.leave(doc_id, true).await?; - self.sync.drop_replica(doc_id).await?; - Ok(DropResponse {}) - } - - pub fn doc_list( - &self, - _req: DocListRequest, - ) -> impl Stream> + Unpin { - let (tx, rx) = async_channel::bounded(ITER_CHANNEL_CAP); - let sync = self.sync.clone(); - // we need to spawn a task to send our request to the sync handle, because the method - // itself must be sync. - tokio::task::spawn(async move { - let tx2 = tx.clone(); - if let Err(err) = sync.list_replicas(tx).await { - tx2.send(Err(err)).await.ok(); - } - }); - rx.boxed().map(|r| { - r.map(|(id, capability)| DocListResponse { id, capability }) - .map_err(Into::into) - }) - } - - pub async fn doc_open(&self, req: OpenRequest) -> RpcResult { - self.sync.open(req.doc_id, Default::default()).await?; - Ok(OpenResponse {}) - } - - pub async fn doc_close(&self, req: CloseRequest) -> RpcResult { - self.sync.close(req.doc_id).await?; - Ok(CloseResponse {}) - } - - pub async fn doc_status(&self, req: StatusRequest) -> RpcResult { - let status = self.sync.get_state(req.doc_id).await?; - Ok(StatusResponse { status }) - } - - pub async fn doc_share(&self, req: ShareRequest) -> RpcResult { - let ShareRequest { - doc_id, - mode, - addr_options, - } = req; - let mut me = self.endpoint.node_addr().await?; - me.apply_options(addr_options); - - let capability = match mode { - ShareMode::Read => iroh_docs::Capability::Read(doc_id), - ShareMode::Write => { - let secret = self.sync.export_secret_key(doc_id).await?; - iroh_docs::Capability::Write(secret) - } - }; - self.start_sync(doc_id, vec![]).await?; - - Ok(ShareResponse(DocTicket { - capability, - nodes: vec![me], - })) - } - - pub async fn doc_subscribe( - &self, - req: DocSubscribeRequest, - ) -> RpcResult>> { - let stream = self.subscribe(req.doc_id).await?; - - Ok(stream.map(|el| { - el.map(|event| DocSubscribeResponse { event }) - .map_err(Into::into) - })) - } - - pub async fn doc_import(&self, req: DocImportRequest) -> RpcResult { - let DocImportRequest { capability } = req; - let doc_id = self.sync.import_namespace(capability).await?; - self.sync.open(doc_id, Default::default()).await?; - Ok(DocImportResponse { doc_id }) - } - - pub async fn doc_start_sync(&self, req: StartSyncRequest) -> RpcResult { - let StartSyncRequest { doc_id, peers } = req; - self.start_sync(doc_id, peers).await?; - Ok(StartSyncResponse {}) - } - - pub async fn doc_leave(&self, req: LeaveRequest) -> RpcResult { - let LeaveRequest { doc_id } = req; - self.leave(doc_id, false).await?; - Ok(LeaveResponse {}) - } - - pub async fn doc_set( - &self, - bao_store: &B, - req: SetRequest, - ) -> RpcResult { - let SetRequest { - doc_id, - author_id, - key, - value, - } = req; - let len = value.len(); - let tag = bao_store.import_bytes(value, BlobFormat::Raw).await?; - self.sync - .insert_local(doc_id, author_id, key.clone(), *tag.hash(), len as u64) - .await?; - let entry = self - .sync - .get_exact(doc_id, author_id, key, false) - .await? - .ok_or_else(|| anyhow!("failed to get entry after insertion"))?; - Ok(SetResponse { entry }) - } - - pub async fn doc_del(&self, req: DelRequest) -> RpcResult { - let DelRequest { - doc_id, - author_id, - prefix, - } = req; - let removed = self.sync.delete_prefix(doc_id, author_id, prefix).await?; - Ok(DelResponse { removed }) - } - - pub async fn doc_set_hash(&self, req: SetHashRequest) -> RpcResult { - let SetHashRequest { - doc_id, - author_id, - key, - hash, - size, - } = req; - self.sync - .insert_local(doc_id, author_id, key.clone(), hash, size) - .await?; - Ok(SetHashResponse {}) - } - - pub fn doc_get_many( - &self, - req: GetManyRequest, - ) -> impl Stream> + Unpin { - let GetManyRequest { doc_id, query } = req; - let (tx, rx) = async_channel::bounded(ITER_CHANNEL_CAP); - let sync = self.sync.clone(); - // we need to spawn a task to send our request to the sync handle, because the method - // itself must be sync. - tokio::task::spawn(async move { - let tx2 = tx.clone(); - if let Err(err) = sync.get_many(doc_id, query, tx).await { - tx2.send(Err(err)).await.ok(); - } - }); - rx.boxed() - .map(|r| r.map(|entry| GetManyResponse { entry }).map_err(Into::into)) - } - - pub async fn doc_get_exact(&self, req: GetExactRequest) -> RpcResult { - let GetExactRequest { - doc_id, - author, - key, - include_empty, - } = req; - let entry = self - .sync - .get_exact(doc_id, author, key, include_empty) - .await?; - Ok(GetExactResponse { entry }) - } - - pub async fn doc_set_download_policy( - &self, - req: SetDownloadPolicyRequest, - ) -> RpcResult { - self.sync - .set_download_policy(req.doc_id, req.policy) - .await?; - Ok(SetDownloadPolicyResponse {}) - } - pub async fn doc_get_download_policy( - &self, - req: GetDownloadPolicyRequest, - ) -> RpcResult { - let policy = self.sync.get_download_policy(req.doc_id).await?; - Ok(GetDownloadPolicyResponse { policy }) - } - - pub async fn doc_get_sync_peers( - &self, - req: GetSyncPeersRequest, - ) -> RpcResult { - let peers = self.sync.get_sync_peers(req.doc_id).await?; - Ok(GetSyncPeersResponse { peers }) - } -} diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 5b81f32225e..1f116904579 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -17,14 +17,8 @@ //! macro. use serde::{Deserialize, Serialize}; -pub mod authors; -pub mod blobs; -pub mod docs; -pub mod gossip; pub mod net; pub mod node; -pub mod spaces; -pub mod tags; /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] @@ -37,12 +31,9 @@ pub struct RpcService; pub enum Request { Node(node::Request), Net(net::Request), - Blobs(blobs::Request), - Docs(docs::Request), - Tags(tags::Request), - Authors(authors::Request), - Gossip(gossip::Request), - Spaces(spaces::Request), + Gossip(iroh_gossip::RpcRequest), + Docs(iroh_docs::rpc::proto::Request), + BlobsAndTags(iroh_blobs::rpc::proto::Request), } /// The response enum, listing all possible responses. @@ -52,12 +43,9 @@ pub enum Request { pub enum Response { Node(node::Response), Net(net::Response), - Blobs(blobs::Response), - Tags(tags::Response), - Docs(docs::Response), - Authors(authors::Response), - Gossip(gossip::Response), - Spaces(spaces::Response), + Gossip(iroh_gossip::RpcResponse), + Docs(iroh_docs::rpc::proto::Response), + BlobsAndTags(iroh_blobs::rpc::proto::Response), } impl quic_rpc::Service for RpcService { diff --git a/iroh/src/rpc_protocol/authors.rs b/iroh/src/rpc_protocol/authors.rs deleted file mode 100644 index fec3b93ddbb..00000000000 --- a/iroh/src/rpc_protocol/authors.rs +++ /dev/null @@ -1,123 +0,0 @@ -use iroh_base::rpc::RpcResult; -use iroh_docs::{Author, AuthorId}; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[server_streaming(response = RpcResult)] - List(ListRequest), - #[rpc(response = RpcResult)] - Create(CreateRequest), - #[rpc(response = RpcResult)] - GetDefault(GetDefaultRequest), - #[rpc(response = RpcResult)] - SetDefault(SetDefaultRequest), - #[rpc(response = RpcResult)] - Import(ImportRequest), - #[rpc(response = RpcResult)] - Export(ExportRequest), - #[rpc(response = RpcResult)] - Delete(DeleteRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - List(RpcResult), - Create(RpcResult), - GetDefault(RpcResult), - SetDefault(RpcResult), - Import(RpcResult), - Export(RpcResult), - Delete(RpcResult), -} - -/// List document authors for which we have a secret key. -#[derive(Serialize, Deserialize, Debug)] -pub struct ListRequest {} - -/// Response for [`ListRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ListResponse { - /// The author id - pub author_id: AuthorId, -} - -/// Create a new document author. -#[derive(Serialize, Deserialize, Debug)] -pub struct CreateRequest; - -/// Response for [`CreateRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct CreateResponse { - /// The id of the created author - pub author_id: AuthorId, -} - -/// Get the default author. -#[derive(Serialize, Deserialize, Debug)] -pub struct GetDefaultRequest; - -/// Response for [`GetDefaultRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct GetDefaultResponse { - /// The id of the author - pub author_id: AuthorId, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct SetDefaultRequest { - /// The id of the author - pub author_id: AuthorId, -} - -/// Response for [`GetDefaultRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct SetDefaultResponse; - -/// Delete an author -#[derive(Serialize, Deserialize, Debug)] -pub struct DeleteRequest { - /// The id of the author to delete - pub author: AuthorId, -} - -/// Response for [`DeleteRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct DeleteResponse; - -/// Exports an author -#[derive(Serialize, Deserialize, Debug)] -pub struct ExportRequest { - /// The id of the author to delete - pub author: AuthorId, -} - -/// Response for [`ExportRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ExportResponse { - /// The author - pub author: Option, -} - -/// Import author from secret key -#[derive(Serialize, Deserialize, Debug)] -pub struct ImportRequest { - /// The author to import - pub author: Author, -} - -/// Response to [`ImportRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ImportResponse { - /// The author id of the imported author - pub author_id: AuthorId, -} diff --git a/iroh/src/rpc_protocol/blobs.rs b/iroh/src/rpc_protocol/blobs.rs deleted file mode 100644 index d167dceb81f..00000000000 --- a/iroh/src/rpc_protocol/blobs.rs +++ /dev/null @@ -1,345 +0,0 @@ -use std::path::PathBuf; - -use bytes::Bytes; -use iroh_base::{ - hash::Hash, - rpc::{RpcError, RpcResult}, -}; -use iroh_blobs::{ - export::ExportProgress, - format::collection::Collection, - get::db::DownloadProgress, - provider::{AddProgress, BatchAddPathProgress}, - store::{ - BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ImportMode, - ValidateProgress, - }, - util::SetTagOption, - BlobFormat, HashAndFormat, Tag, -}; -use iroh_net::NodeAddr; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; -use crate::client::blobs::{ - BlobInfo, BlobStatus, DownloadMode, IncompleteBlobInfo, ReadAtLen, WrapOption, -}; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[server_streaming(response = RpcResult)] - ReadAt(ReadAtRequest), - #[bidi_streaming(update = AddStreamUpdate, response = AddStreamResponse)] - AddStream(AddStreamRequest), - AddStreamUpdate(AddStreamUpdate), - #[server_streaming(response = AddPathResponse)] - AddPath(AddPathRequest), - #[server_streaming(response = DownloadResponse)] - Download(DownloadRequest), - #[server_streaming(response = ExportResponse)] - Export(ExportRequest), - #[server_streaming(response = RpcResult)] - List(ListRequest), - #[server_streaming(response = RpcResult)] - ListIncomplete(ListIncompleteRequest), - #[rpc(response = RpcResult<()>)] - Delete(DeleteRequest), - #[server_streaming(response = ValidateProgress)] - Validate(ValidateRequest), - #[server_streaming(response = ConsistencyCheckProgress)] - Fsck(ConsistencyCheckRequest), - #[rpc(response = RpcResult)] - CreateCollection(CreateCollectionRequest), - #[rpc(response = RpcResult)] - BlobStatus(BlobStatusRequest), - - #[bidi_streaming(update = BatchUpdate, response = BatchCreateResponse)] - BatchCreate(BatchCreateRequest), - BatchUpdate(BatchUpdate), - #[bidi_streaming(update = BatchAddStreamUpdate, response = BatchAddStreamResponse)] - BatchAddStream(BatchAddStreamRequest), - BatchAddStreamUpdate(BatchAddStreamUpdate), - #[server_streaming(response = BatchAddPathResponse)] - BatchAddPath(BatchAddPathRequest), - #[rpc(response = RpcResult<()>)] - BatchCreateTempTag(BatchCreateTempTagRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - ReadAt(RpcResult), - AddStream(AddStreamResponse), - AddPath(AddPathResponse), - List(RpcResult), - ListIncomplete(RpcResult), - Download(DownloadResponse), - Fsck(ConsistencyCheckProgress), - Export(ExportResponse), - Validate(ValidateProgress), - CreateCollection(RpcResult), - BlobStatus(RpcResult), - BatchCreate(BatchCreateResponse), - BatchAddStream(BatchAddStreamResponse), - BatchAddPath(BatchAddPathResponse), -} - -/// A request to the node to provide the data at the given path -/// -/// Will produce a stream of [`AddProgress`] messages. -#[derive(Debug, Serialize, Deserialize)] -pub struct AddPathRequest { - /// The path to the data to provide. - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. Usually the cli will run on the same machine as the - /// node, so this should be an absolute path on the cli machine. - pub path: PathBuf, - /// True if the provider can assume that the data will not change, so it - /// can be shared in place. - pub in_place: bool, - /// Tag to tag the data with. - pub tag: SetTagOption, - /// Whether to wrap the added data in a collection - pub wrap: WrapOption, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct AddPathResponse(pub AddProgress); - -/// A request to the node to download and share the data specified by the hash. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DownloadRequest { - /// This mandatory field contains the hash of the data to download and share. - pub hash: Hash, - /// If the format is [`BlobFormat::HashSeq`], all children are downloaded and shared as - /// well. - pub format: BlobFormat, - /// This mandatory field specifies the nodes to download the data from. - /// - /// If set to more than a single node, they will all be tried. If `mode` is set to - /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds. - /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel, - /// if the concurrency limits permit. - pub nodes: Vec, - /// Optional tag to tag the data with. - pub tag: SetTagOption, - /// Whether to directly start the download or add it to the download queue. - pub mode: DownloadMode, -} - -/// Progress response for [`DownloadRequest`] -#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct DownloadResponse(pub DownloadProgress); - -/// A request to the node to download and share the data specified by the hash. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExportRequest { - /// The hash of the blob to export. - pub hash: Hash, - /// The filepath to where the data should be saved - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. - pub path: PathBuf, - /// Set to [`ExportFormat::Collection`] if the `hash` refers to a [`Collection`] and you want - /// to export all children of the collection into individual files. - pub format: ExportFormat, - /// The mode of exporting. - /// - /// The default is [`ExportMode::Copy`]. See [`ExportMode`] for details. - pub mode: ExportMode, -} - -/// Progress response for [`ExportRequest`] -#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct ExportResponse(pub ExportProgress); - -/// A request to the node to validate the integrity of all provided data -#[derive(Debug, Serialize, Deserialize)] -pub struct ConsistencyCheckRequest { - /// repair the store by dropping inconsistent blobs - pub repair: bool, -} - -/// A request to the node to validate the integrity of all provided data -#[derive(Debug, Serialize, Deserialize)] -pub struct ValidateRequest { - /// repair the store by downgrading blobs from complete to partial - pub repair: bool, -} - -/// List all blobs, including collections -#[derive(Debug, Serialize, Deserialize)] -pub struct ListRequest; - -/// List all blobs, including collections -#[derive(Debug, Serialize, Deserialize)] -pub struct ListIncompleteRequest; - -/// Get the bytes for a hash -#[derive(Serialize, Deserialize, Debug)] -pub struct ReadAtRequest { - /// Hash to get bytes for - pub hash: Hash, - /// Offset to start reading at - pub offset: u64, - /// Length of the data to get - pub len: ReadAtLen, -} - -/// Response to [`ReadAtRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub enum ReadAtResponse { - /// The entry header. - Entry { - /// The size of the blob - size: BaoBlobSize, - /// Whether the blob is complete - is_complete: bool, - }, - /// Chunks of entry data. - Data { - /// The data chunk - chunk: Bytes, - }, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct AddStreamRequest { - /// Tag to tag the data with. - pub tag: SetTagOption, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub enum AddStreamUpdate { - /// A chunk of stream data - Chunk(Bytes), - /// Abort the request due to an error on the client side - Abort, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct AddStreamResponse(pub AddProgress); - -/// Delete a blob -#[derive(Debug, Serialize, Deserialize)] -pub struct DeleteRequest { - /// Name of the tag - pub hash: Hash, -} - -/// Create a collection. -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateCollectionRequest { - /// The collection - pub collection: Collection, - /// Tag option. - pub tag: SetTagOption, - /// Tags that should be deleted after creation. - pub tags_to_delete: Vec, -} - -/// A response to a create collection request -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateCollectionResponse { - /// The resulting hash. - pub hash: Hash, - /// The resulting tag. - pub tag: Tag, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobStatusRequest { - /// The hash of the blob - pub hash: Hash, -} - -/// The response to a status request -#[derive(Debug, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct BlobStatusResponse(pub BlobStatus); - -/// Request to create a new scope for temp tags -#[derive(Debug, Serialize, Deserialize)] -pub struct BatchCreateRequest; - -/// Update to a temp tag scope -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchUpdate { - /// Drop of a remote temp tag - Drop(HashAndFormat), - /// Message to check that the connection is still alive - Ping, -} - -/// Response to a temp tag scope request -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchCreateResponse { - /// We got the id of the scope - Id(BatchId), -} - -/// Create a temp tag with a given hash and format -#[derive(Debug, Serialize, Deserialize)] -pub struct BatchCreateTempTagRequest { - /// Content to protect - pub content: HashAndFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddStreamRequest { - /// What format to use for the blob - pub format: BlobFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub enum BatchAddStreamUpdate { - /// A chunk of stream data - Chunk(Bytes), - /// Abort the request due to an error on the client side - Abort, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchAddStreamResponse { - Abort(RpcError), - OutboardProgress { offset: u64 }, - Result { hash: Hash }, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddPathRequest { - /// The path to the data to provide. - pub path: PathBuf, - /// Add the data in place - pub import_mode: ImportMode, - /// What format to use for the blob - pub format: BlobFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Response to a batch add path request -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddPathResponse(pub BatchAddPathProgress); - -#[derive(Debug, PartialEq, Eq, PartialOrd, Serialize, Deserialize, Ord, Clone, Copy, Hash)] -pub struct BatchId(pub(crate) u64); diff --git a/iroh/src/rpc_protocol/docs.rs b/iroh/src/rpc_protocol/docs.rs deleted file mode 100644 index 83975b6f301..00000000000 --- a/iroh/src/rpc_protocol/docs.rs +++ /dev/null @@ -1,425 +0,0 @@ -use std::path::PathBuf; - -use bytes::Bytes; -use iroh_base::{ - node_addr::AddrInfoOptions, - rpc::{RpcError, RpcResult}, -}; -use iroh_blobs::{export::ExportProgress, store::ExportMode, Hash}; -use iroh_docs::{ - actor::OpenState, - engine::LiveEvent, - store::{DownloadPolicy, Query}, - AuthorId, Capability, CapabilityKind, DocTicket, Entry, NamespaceId, PeerIdBytes, SignedEntry, -}; -use iroh_net::NodeAddr; -use nested_enum_utils::enum_conversions; -use quic_rpc::pattern::try_server_streaming::StreamCreated; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; -use crate::client::docs::{ImportProgress, ShareMode}; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[rpc(response = RpcResult)] - Open(OpenRequest), - #[rpc(response = RpcResult)] - Close(CloseRequest), - #[rpc(response = RpcResult)] - Status(StatusRequest), - #[server_streaming(response = RpcResult)] - List(DocListRequest), - #[rpc(response = RpcResult)] - Create(CreateRequest), - #[rpc(response = RpcResult)] - Drop(DropRequest), - #[rpc(response = RpcResult)] - Import(ImportRequest), - #[rpc(response = RpcResult)] - Set(SetRequest), - #[rpc(response = RpcResult)] - SetHash(SetHashRequest), - #[server_streaming(response = RpcResult)] - Get(GetManyRequest), - #[rpc(response = RpcResult)] - GetExact(GetExactRequest), - #[server_streaming(response = ImportFileResponse)] - ImportFile(ImportFileRequest), - #[server_streaming(response = ExportFileResponse)] - ExportFile(ExportFileRequest), - #[rpc(response = RpcResult)] - Del(DelRequest), - #[rpc(response = RpcResult)] - StartSync(StartSyncRequest), - #[rpc(response = RpcResult)] - Leave(LeaveRequest), - #[rpc(response = RpcResult)] - Share(ShareRequest), - #[try_server_streaming(create_error = RpcError, item_error = RpcError, item = DocSubscribeResponse)] - Subscribe(DocSubscribeRequest), - #[rpc(response = RpcResult)] - GetDownloadPolicy(GetDownloadPolicyRequest), - #[rpc(response = RpcResult)] - SetDownloadPolicy(SetDownloadPolicyRequest), - #[rpc(response = RpcResult)] - GetSyncPeers(GetSyncPeersRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - Open(RpcResult), - Close(RpcResult), - Status(RpcResult), - List(RpcResult), - Create(RpcResult), - Drop(RpcResult), - Import(RpcResult), - Set(RpcResult), - SetHash(RpcResult), - Get(RpcResult), - GetExact(RpcResult), - ImportFile(ImportFileResponse), - ExportFile(ExportFileResponse), - Del(RpcResult), - Share(RpcResult), - StartSync(RpcResult), - Leave(RpcResult), - Subscribe(RpcResult), - GetDownloadPolicy(RpcResult), - SetDownloadPolicy(RpcResult), - GetSyncPeers(RpcResult), - StreamCreated(RpcResult), -} - -/// Subscribe to events for a document. -#[derive(Serialize, Deserialize, Debug)] -pub struct DocSubscribeRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`DocSubscribeRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct DocSubscribeResponse { - /// The event that occurred on the document - pub event: LiveEvent, -} - -/// List all documents -#[derive(Serialize, Deserialize, Debug)] -pub struct DocListRequest {} - -/// Response to [`DocListRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ListResponse { - /// The document id - pub id: NamespaceId, - /// The capability over the document. - pub capability: CapabilityKind, -} - -/// Create a new document -#[derive(Serialize, Deserialize, Debug)] -pub struct CreateRequest {} - -/// Response to [`CreateRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct CreateResponse { - /// The document id - pub id: NamespaceId, -} - -/// Import a document from a capability. -#[derive(Serialize, Deserialize, Debug)] -pub struct ImportRequest { - /// The namespace capability. - pub capability: Capability, -} - -/// Response to [`ImportRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ImportResponse { - /// the document id - pub doc_id: NamespaceId, -} - -/// Share a document with peers over a ticket. -#[derive(Serialize, Deserialize, Debug)] -pub struct ShareRequest { - /// The document id - pub doc_id: NamespaceId, - /// Whether to share read or write access to the document - pub mode: ShareMode, - /// Configuration of the addresses in the ticket. - pub addr_options: AddrInfoOptions, -} - -/// The response to [`ShareRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct ShareResponse(pub DocTicket); - -/// Get info on a document -#[derive(Serialize, Deserialize, Debug)] -pub struct StatusRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`StatusRequest`] -// TODO: actually provide info -#[derive(Serialize, Deserialize, Debug)] -pub struct StatusResponse { - /// Live sync status - pub status: OpenState, -} - -/// Open a document -#[derive(Serialize, Deserialize, Debug)] -pub struct OpenRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`OpenRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct OpenResponse {} - -/// Open a document -#[derive(Serialize, Deserialize, Debug)] -pub struct CloseRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`CloseRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct CloseResponse {} - -/// Start to sync a doc with peers. -#[derive(Serialize, Deserialize, Debug)] -pub struct StartSyncRequest { - /// The document id - pub doc_id: NamespaceId, - /// List of peers to join - pub peers: Vec, -} - -/// Response to [`StartSyncRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct StartSyncResponse {} - -/// Stop the live sync for a doc, and optionally delete the document. -#[derive(Serialize, Deserialize, Debug)] -pub struct LeaveRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`LeaveRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct LeaveResponse {} - -/// Stop the live sync for a doc, and optionally delete the document. -#[derive(Serialize, Deserialize, Debug)] -pub struct DropRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`DropRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct DropResponse {} - -/// Set an entry in a document -#[derive(Serialize, Deserialize, Debug)] -pub struct SetRequest { - /// The document id - pub doc_id: NamespaceId, - /// Author of this entry. - pub author_id: AuthorId, - /// Key of this entry. - pub key: Bytes, - /// Value of this entry. - // TODO: Allow to provide the hash directly - // TODO: Add a way to provide content as stream - pub value: Bytes, -} - -/// Response to [`SetRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct SetResponse { - /// The newly-created entry. - pub entry: SignedEntry, -} - -/// A request to the node to add the data at the given filepath as an entry to the document -/// -/// Will produce a stream of [`ImportProgress`] messages. -#[derive(Debug, Serialize, Deserialize)] -pub struct ImportFileRequest { - /// The document id - pub doc_id: NamespaceId, - /// Author of this entry. - pub author_id: AuthorId, - /// Key of this entry. - pub key: Bytes, - /// The filepath to the data - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. Usually the cli will run on the same machine as the - /// node, so this should be an absolute path on the cli machine. - pub path: PathBuf, - /// True if the provider can assume that the data will not change, so it - /// can be shared in place. - pub in_place: bool, -} - -/// Wrapper around [`ImportProgress`]. -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct ImportFileResponse(pub ImportProgress); - -/// A request to the node to save the data of the entry to the given filepath -/// -/// Will produce a stream of [`ExportFileResponse`] messages. -#[derive(Debug, Serialize, Deserialize)] -pub struct ExportFileRequest { - /// The entry you want to export - pub entry: Entry, - /// The filepath to where the data should be saved - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. Usually the cli will run on the same machine as the - /// node, so this should be an absolute path on the cli machine. - pub path: PathBuf, - /// The mode of exporting. Setting to `ExportMode::TryReference` means attempting - /// to use references for keeping file - pub mode: ExportMode, -} - -/// Progress messages for an doc export operation -/// -/// An export operation involves reading the entry from the database ans saving the entry to the -/// given `outpath` -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct ExportFileResponse(pub ExportProgress); - -/// Delete entries in a document -#[derive(Serialize, Deserialize, Debug)] -pub struct DelRequest { - /// The document id. - pub doc_id: NamespaceId, - /// Author of this entry. - pub author_id: AuthorId, - /// Prefix to delete. - pub prefix: Bytes, -} - -/// Response to [`DelRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct DelResponse { - /// The number of entries that were removed. - pub removed: usize, -} - -/// Set an entry in a document via its hash -#[derive(Serialize, Deserialize, Debug)] -pub struct SetHashRequest { - /// The document id - pub doc_id: NamespaceId, - /// Author of this entry. - pub author_id: AuthorId, - /// Key of this entry. - pub key: Bytes, - /// Hash of this entry. - pub hash: Hash, - /// Size of this entry. - pub size: u64, -} - -/// Response to [`SetHashRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct SetHashResponse {} - -/// Get entries from a document -#[derive(Serialize, Deserialize, Debug)] -pub struct GetManyRequest { - /// The document id - pub doc_id: NamespaceId, - /// Query to run - pub query: Query, -} - -/// Response to [`GetManyRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct GetManyResponse { - /// The document entry - pub entry: SignedEntry, -} - -/// Get entries from a document -#[derive(Serialize, Deserialize, Debug)] -pub struct GetExactRequest { - /// The document id - pub doc_id: NamespaceId, - /// Key matcher - pub key: Bytes, - /// Author matcher - pub author: AuthorId, - /// Whether to include empty entries (prefix deletion markers) - pub include_empty: bool, -} - -/// Response to [`GetExactRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct GetExactResponse { - /// The document entry - pub entry: Option, -} - -/// Set a download policy -#[derive(Serialize, Deserialize, Debug)] -pub struct SetDownloadPolicyRequest { - /// The document id - pub doc_id: NamespaceId, - /// Download policy - pub policy: DownloadPolicy, -} - -/// Response to [`SetDownloadPolicyRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct SetDownloadPolicyResponse {} - -/// Get a download policy -#[derive(Serialize, Deserialize, Debug)] -pub struct GetDownloadPolicyRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`GetDownloadPolicyRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct GetDownloadPolicyResponse { - /// The download policy - pub policy: DownloadPolicy, -} - -/// Get peers for document -#[derive(Serialize, Deserialize, Debug)] -pub struct GetSyncPeersRequest { - /// The document id - pub doc_id: NamespaceId, -} - -/// Response to [`GetSyncPeersRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct GetSyncPeersResponse { - /// List of peers ids - pub peers: Option>, -} diff --git a/iroh/src/rpc_protocol/gossip.rs b/iroh/src/rpc_protocol/gossip.rs deleted file mode 100644 index f2666335ed7..00000000000 --- a/iroh/src/rpc_protocol/gossip.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::collections::BTreeSet; - -use iroh_base::rpc::RpcResult; -pub use iroh_gossip::net::{Command as SubscribeUpdate, Event as SubscribeResponse}; -use iroh_gossip::proto::TopicId; -use iroh_net::NodeId; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[bidi_streaming(update = SubscribeUpdate, response = RpcResult)] - Subscribe(SubscribeRequest), - Update(SubscribeUpdate), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - Subscribe(RpcResult), -} - -/// A request to the node to subscribe to gossip events. -/// -/// This is basically a topic and additional options -#[derive(Serialize, Deserialize, Debug)] -pub struct SubscribeRequest { - /// The topic to subscribe to - pub topic: TopicId, - /// The nodes to bootstrap the subscription from - pub bootstrap: BTreeSet, - /// The capacity of the subscription - pub subscription_capacity: usize, -} diff --git a/iroh/src/rpc_protocol/net.rs b/iroh/src/rpc_protocol/net.rs index d9cb4e1e40b..143392ee6ec 100644 --- a/iroh/src/rpc_protocol/net.rs +++ b/iroh/src/rpc_protocol/net.rs @@ -1,10 +1,10 @@ -use iroh_base::rpc::RpcResult; -use iroh_net::{endpoint::RemoteInfo, key::PublicKey, relay::RelayUrl, NodeAddr, NodeId}; +use iroh_net::{endpoint::RemoteInfo, key::PublicKey, NodeAddr, NodeId, RelayUrl}; use nested_enum_utils::enum_conversions; use quic_rpc_derive::rpc_requests; use serde::{Deserialize, Serialize}; use super::RpcService; +use crate::node::RpcResult; #[allow(missing_docs)] #[derive(strum::Display, Debug, Serialize, Deserialize)] @@ -37,6 +37,7 @@ pub enum Response { RemoteInfosIter(RpcResult), RemoteInfo(RpcResult), Watch(WatchResponse), + Unit(RpcResult<()>), } /// List network path information about all the remote nodes known by this node. diff --git a/iroh/src/rpc_protocol/node.rs b/iroh/src/rpc_protocol/node.rs index e0e6d415c55..b615ea47d81 100644 --- a/iroh/src/rpc_protocol/node.rs +++ b/iroh/src/rpc_protocol/node.rs @@ -1,12 +1,11 @@ use std::collections::BTreeMap; -use iroh_base::rpc::RpcResult; use nested_enum_utils::enum_conversions; use quic_rpc_derive::rpc_requests; use serde::{Deserialize, Serialize}; use super::RpcService; -use crate::client::NodeStatus; +use crate::{client::NodeStatus, node::RpcResult}; #[allow(missing_docs)] #[derive(strum::Display, Debug, Serialize, Deserialize)] diff --git a/iroh/src/rpc_protocol/tags.rs b/iroh/src/rpc_protocol/tags.rs deleted file mode 100644 index c63243d381e..00000000000 --- a/iroh/src/rpc_protocol/tags.rs +++ /dev/null @@ -1,110 +0,0 @@ -use iroh_base::rpc::RpcResult; -use iroh_blobs::{HashAndFormat, Tag}; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::{blobs::BatchId, RpcService}; -use crate::client::tags::TagInfo; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[rpc(response = RpcResult)] - Create(CreateRequest), - #[rpc(response = RpcResult<()>)] - Set(SetRequest), - #[rpc(response = RpcResult<()>)] - DeleteTag(DeleteRequest), - #[server_streaming(response = TagInfo)] - ListTags(ListRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - Create(RpcResult), - ListTags(TagInfo), - DeleteTag(RpcResult<()>), -} - -/// Determine how to sync the db after a modification operation -#[derive(Debug, Serialize, Deserialize, Default)] -pub enum SyncMode { - /// Fully sync the db - #[default] - Full, - /// Do not sync the db - None, -} - -/// Create a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateRequest { - /// Value of the tag - pub value: HashAndFormat, - /// Batch to use, none for global - pub batch: Option, - /// Sync mode - pub sync: SyncMode, -} - -/// Set or delete a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct SetRequest { - /// Name of the tag - pub name: Tag, - /// Value of the tag, None to delete - pub value: Option, - /// Batch to use, none for global - pub batch: Option, - /// Sync mode - pub sync: SyncMode, -} - -/// List all collections -/// -/// Lists all collections that have been explicitly added to the database. -#[derive(Debug, Serialize, Deserialize)] -pub struct ListRequest { - /// List raw tags - pub raw: bool, - /// List hash seq tags - pub hash_seq: bool, -} - -impl ListRequest { - /// List all tags - pub fn all() -> Self { - Self { - raw: true, - hash_seq: true, - } - } - - /// List raw tags - pub fn raw() -> Self { - Self { - raw: true, - hash_seq: false, - } - } - - /// List hash seq tags - pub fn hash_seq() -> Self { - Self { - raw: false, - hash_seq: true, - } - } -} - -/// Delete a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct DeleteRequest { - /// Name of the tag - pub name: Tag, -} diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index d91416ef9dc..061be9a7627 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -1,6 +1,5 @@ //! Utilities for filesystem operations. use std::{ - borrow::Cow, fs::read_dir, path::{Component, Path, PathBuf}, }; @@ -9,115 +8,6 @@ use anyhow::{bail, Context}; use bytes::Bytes; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; -use walkdir::WalkDir; - -use crate::client::blobs::WrapOption; - -/// A data source -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -pub struct DataSource { - /// Custom name - name: String, - /// Path to the file - path: PathBuf, -} - -impl DataSource { - /// Creates a new [`DataSource`] from a [`PathBuf`]. - pub fn new(path: PathBuf) -> Self { - let name = path - .file_name() - .map(|s| s.to_string_lossy().to_string()) - .unwrap_or_default(); - DataSource { path, name } - } - /// Creates a new [`DataSource`] from a [`PathBuf`] and a custom name. - pub fn with_name(path: PathBuf, name: String) -> Self { - DataSource { path, name } - } - - /// Returns blob name for this data source. - /// - /// If no name was provided when created it is derived from the path name. - pub fn name(&self) -> Cow<'_, str> { - Cow::Borrowed(&self.name) - } - - /// Returns the path of this data source. - pub fn path(&self) -> &Path { - &self.path - } -} - -impl From for DataSource { - fn from(value: PathBuf) -> Self { - DataSource::new(value) - } -} - -impl From<&std::path::Path> for DataSource { - fn from(value: &std::path::Path) -> Self { - DataSource::new(value.to_path_buf()) - } -} - -/// Create data sources from a path. -pub fn scan_path(path: PathBuf, wrap: WrapOption) -> anyhow::Result> { - if path.is_dir() { - scan_dir(path, wrap) - } else { - let name = match wrap { - WrapOption::NoWrap => bail!("Cannot scan a file without wrapping"), - WrapOption::Wrap { name: None } => file_name(&path)?, - WrapOption::Wrap { name: Some(name) } => name, - }; - Ok(vec![DataSource { name, path }]) - } -} - -fn file_name(path: &Path) -> anyhow::Result { - relative_canonicalized_path_to_string(path.file_name().context("path is invalid")?) -} - -/// Create data sources from a directory. -pub fn scan_dir(root: PathBuf, wrap: WrapOption) -> anyhow::Result> { - if !root.is_dir() { - bail!("Expected {} to be a file", root.to_string_lossy()); - } - let prefix = match wrap { - WrapOption::NoWrap => None, - WrapOption::Wrap { name: None } => Some(file_name(&root)?), - WrapOption::Wrap { name: Some(name) } => Some(name), - }; - let files = WalkDir::new(&root).into_iter(); - let data_sources = files - .map(|entry| { - let entry = entry?; - if !entry.file_type().is_file() { - // Skip symlinks. Directories are handled by WalkDir. - return Ok(None); - } - let path = entry.into_path(); - let mut name = relative_canonicalized_path_to_string(path.strip_prefix(&root)?)?; - if let Some(prefix) = &prefix { - name = format!("{prefix}/{name}"); - } - anyhow::Ok(Some(DataSource { name, path })) - }) - .filter_map(Result::transpose); - let data_sources: Vec> = data_sources.collect::>(); - data_sources.into_iter().collect::>>() -} - -/// This function converts a canonicalized relative path to a string, returning -/// an error if the path is not valid unicode. -/// -/// This function will also fail if the path is non canonical, i.e. contains -/// `..` or `.`, or if the path components contain any windows or unix path -/// separators. -pub fn relative_canonicalized_path_to_string(path: impl AsRef) -> anyhow::Result { - canonicalized_path_to_string(path, true) -} /// Loads a [`SecretKey`] from the provided file, or stores a newly generated one /// at the given location. @@ -167,7 +57,6 @@ pub struct PathContent { } /// Walks the directory to get the total size and number of files in directory or file -/// // TODO: possible combine with `scan_dir` pub fn path_content_info(path: impl AsRef) -> anyhow::Result { path_content_info0(path) diff --git a/iroh/src/util/path.rs b/iroh/src/util/path.rs index 8d6aa706f4e..f7ee91af409 100644 --- a/iroh/src/util/path.rs +++ b/iroh/src/util/path.rs @@ -15,9 +15,6 @@ pub enum IrohPaths { /// Path to the [iroh-docs document database](iroh_docs::store::fs::Store) #[strum(serialize = "docs.redb")] DocsDatabase, - /// Path to the iroh-willow database - #[strum(serialize = "spaces.redb")] - SpacesDatabase, /// Path to the console state #[strum(serialize = "console")] Console, diff --git a/iroh/tests/client.rs b/iroh/tests/client.rs index a250e5836ed..bf56a61a54c 100644 --- a/iroh/tests/client.rs +++ b/iroh/tests/client.rs @@ -21,7 +21,7 @@ async fn spawn_node() -> (NodeAddr, Iroh) { let secret_key = SecretKey::generate(); let node = iroh::node::Builder::default() .secret_key(secret_key) - .relay_mode(iroh_net::relay::RelayMode::Disabled) + .relay_mode(iroh_net::RelayMode::Disabled) .node_discovery(iroh::node::DiscoveryConfig::None) .spawn() .await?; diff --git a/iroh/tests/gc.rs b/iroh/tests/gc.rs index d194f2b7793..857de7e19f9 100644 --- a/iroh/tests/gc.rs +++ b/iroh/tests/gc.rs @@ -185,13 +185,11 @@ mod file { io::fsm::{BaoContentItem, ResponseDecoderNext}, BaoTree, }; - use futures_lite::StreamExt; use iroh_blobs::{ - store::{BaoBatchWriter, ConsistencyCheckProgress, Map, MapEntryMut, ReportLevel}, + store::{BaoBatchWriter, ConsistencyCheckProgress, MapEntryMut, ReportLevel}, util::progress::{AsyncChannelProgressSender, ProgressSender as _}, TempTag, }; - use iroh_io::AsyncSliceReaderExt; use testdir::testdir; use tokio::io::AsyncReadExt; @@ -228,45 +226,6 @@ mod file { Ok(max_level) } - #[tokio::test] - async fn redb_doc_import_stress() -> Result<()> { - let _ = tracing_subscriber::fmt::try_init(); - let dir = testdir!(); - let bao_store = iroh_blobs::store::fs::Store::load(dir.join("store")).await?; - let (node, _) = wrap_in_node(bao_store.clone(), Duration::from_secs(10)).await; - let client = node.client(); - let doc = client.docs().create().await?; - let author = client.authors().create().await?; - let temp_path = dir.join("temp"); - tokio::fs::create_dir_all(&temp_path).await?; - let mut to_import = Vec::new(); - for i in 0..100 { - let data = create_test_data(16 * 1024 * 3 + 1); - let path = temp_path.join(format!("file{}", i)); - tokio::fs::write(&path, &data).await?; - let key = Bytes::from(format!("{}", path.display())); - to_import.push((key, path, data)); - } - for (key, path, _) in to_import.iter() { - let mut progress = doc.import_file(author, key.clone(), path, true).await?; - while let Some(msg) = progress.next().await { - tracing::info!("import progress {:?}", msg); - } - } - for (i, (key, _, expected)) in to_import.iter().enumerate() { - let Some(entry) = doc.get_exact(author, key.clone(), true).await? else { - anyhow::bail!("doc entry not found {}", i); - }; - let hash = entry.content_hash(); - let Some(content) = bao_store.get(&hash).await? else { - anyhow::bail!("content not found {} {}", i, &hash.to_hex()[..8]); - }; - let data = content.data_reader().read_to_end().await?; - assert_eq!(data, expected); - } - Ok(()) - } - /// Test gc for sequences of hashes that protect their children from deletion. #[tokio::test] async fn gc_file_basics() -> Result<()> { diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 7d380dc4bb2..87ae66f3842 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -10,7 +10,7 @@ use bao_tree::{blake3, ChunkNum, ChunkRanges}; use bytes::Bytes; use futures_lite::FutureExt; use iroh::node::{Builder, DocsStorage}; -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use iroh_blobs::{ format::collection::Collection, get::{ @@ -19,7 +19,7 @@ use iroh_blobs::{ }, protocol::{GetRequest, RangeSpecSeq}, store::{MapMut, Store}, - BlobFormat, Hash, + Hash, }; use iroh_net::{defaults::staging::default_relay_map, key::SecretKey, NodeAddr, NodeId}; use rand::RngCore; @@ -386,15 +386,11 @@ async fn test_run_ticket() { let node = test_node(db).spawn().await.unwrap(); let _drop_guard = node.cancel_token().drop_guard(); - let ticket = node - .blobs() - .share( - hash, - BlobFormat::HashSeq, - AddrInfoOptions::RelayAndAddresses, - ) - .await - .unwrap(); + let mut addr = node.net().node_addr().await.unwrap(); + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, iroh_blobs::BlobFormat::HashSeq) + .expect("ticket creation failed"); + tokio::time::timeout(Duration::from_secs(10), async move { let request = GetRequest::all(hash); run_collection_get_request(SecretKey::generate(), ticket.node_addr().clone(), request).await diff --git a/iroh/tests/spaces.proptest-regressions b/iroh/tests/spaces.proptest-regressions deleted file mode 100644 index 156d17ee07f..00000000000 --- a/iroh/tests/spaces.proptest-regressions +++ /dev/null @@ -1,30 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc b247c5db7888ec8f993852033ea7d612f5a7cc5e51d6dc80cbbf0b370f1bf9df # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [])] } -cc 10758efcbd4145b23bb48a35a5a93b13f42cc71457b18e8b2f521fb66537e94e # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("gamma", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma")]), (Betty, [Write("beta", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("beta", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta")]), (Betty, [Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "beta")]), (Betty, [Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta")])] } -cc bad55ca9718ab95bc85e0ee4581fcf9ca019f10ae8cd8b1c30acd2ab7fd03a7f # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta")]), (Betty, [Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "beta")]), (Alfie, [Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "beta")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma")]), (Betty, [Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma")])] } -cc 9c1851f6773562a9d437743f7033d15df566ef3ee865533a4d197120af731891 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "gamma")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha")]), (Alfie, [])] } -cc 2bd80650f13377a3e39bbbf73c0fcf1f17b056880651abfc68e75f66a7f3c130 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta")]), (Betty, []), (Alfie, [Write("gamma", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha")]), (Alfie, [Write("alpha", "gamma")])] } -cc 48f0a951e77785086a8625b0e5afeb4a25e49fd1923e707eebc42d30c430a144 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "beta"), Write("beta", "beta")]), (Alfie, [Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "alpha")]), (Betty, [Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta")]), (Alfie, [Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "beta")]), (Betty, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "gamma")]), (Betty, [Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha")]), (Betty, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta")])] } -cc fd7a666da43de4a6647fd7a5b7c543c4f11abde19a3d8592d3845226bc964f1c # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [] } -cc 42fc5284840d3b6e58d5650c131a3ab6e7528fc98fb4c4fb77097e29715326f5 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [] } -cc cef4354611d13f5c0cdecbc409eb54eea1ef59512c272875d01040b187db84ea # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta")]), (Betty, []), (Alfie, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha")]), (Betty, [Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("alpha", "beta")])] } -cc cfd6874efc9b42a5cad679512edfb09332852f4919920b2dde117e7039edff5a # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta")]), (Betty, [Write("beta", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma")]), (Alfie, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "beta"), Write("gamma", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha")]), (Betty, [Write("gamma", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "alpha")])] } -cc 61e5a9d3c5dc02a1fe0ebb0a357d33c0c7eb5340a6b0982a9e6d187103366062 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "gamma"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Betty, [Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma")]), (Betty, [Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("alpha", "beta")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma")]), (Alfie, [Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta")]), (Betty, [Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta")])] } -cc 7ad26b3a87698eedb995dfcb549ddf6e730b900d794e3645e9193a3d0ba942af # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta")]), (Betty, [Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "beta")]), (Betty, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "beta"), Write("beta", "gamma")])] } -cc 66d537ccb5b41cbc39884aef2c35e3c6242a37973888a30f87d9918f3e626fca # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Alfie, [Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("gamma", "gamma")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma")]), (Alfie, [Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha")]), (Alfie, [Write("alpha", "gamma"), Write("beta", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("gamma", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "beta")]), (Alfie, []), (Alfie, [Write("alpha", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma")]), (Alfie, [Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("beta", "alpha"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("beta", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "gamma"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "gamma")])] } -cc a7a7d234a4fbe2d760f1cc6000ba506da4e1a36e06ec8303a1877ff006242c84 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("beta", "beta")]), (Alfie, [Write("gamma", "alpha"), Write("alpha", "alpha"), Write("beta", "beta"), Write("alpha", "gamma"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("beta", "beta")]), (Alfie, [Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "gamma"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "beta")]), (Betty, [Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("gamma", "gamma")]), (Alfie, [Write("gamma", "gamma"), Write("gamma", "alpha"), Write("beta", "alpha")]), (Betty, [Write("beta", "gamma"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha")]), (Betty, [Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "alpha"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "alpha")])] } -cc 8345ae470aaefbc75795860875a4a379be7a30a96c8108c87d0f7473278f55b8 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "beta"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "beta"), Write("beta", "gamma")]), (Alfie, [Write("gamma", "beta"), Write("beta", "alpha"), Write("gamma", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "beta"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("alpha", "gamma")]), (Betty, [Write("alpha", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("alpha", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("beta", "beta"), Write("beta", "gamma"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "alpha"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "gamma")]), (Betty, [Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "alpha"), Write("gamma", "beta"), Write("beta", "beta"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("gamma", "alpha")]), (Betty, [Write("alpha", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("gamma", "alpha")]), (Alfie, [Write("beta", "gamma"), Write("alpha", "beta")]), (Betty, [Write("beta", "gamma"), Write("alpha", "beta")]), (Alfie, [Write("gamma", "gamma"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "beta")])] } -cc f4f91399efad219908f9467397d047a6319e1111571cf08574f93fa8da8d1f06 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Betty, [Write("beta", "alpha"), Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "alpha"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "beta")]), (Betty, [Write("alpha", "beta"), Write("gamma", "alpha"), Write("gamma", "alpha"), Write("beta", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "alpha"), Write("gamma", "beta"), Write("gamma", "beta"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("gamma", "beta"), Write("alpha", "gamma"), Write("gamma", "alpha"), Write("gamma", "beta")]), (Alfie, [Write("alpha", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("beta", "alpha")]), (Alfie, [Write("beta", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("alpha", "gamma"), Write("gamma", "beta"), Write("alpha", "alpha"), Write("beta", "alpha")]), (Alfie, [Write("beta", "beta"), Write("beta", "beta")]), (Alfie, [Write("gamma", "gamma"), Write("beta", "beta"), Write("alpha", "alpha"), Write("beta", "beta"), Write("beta", "alpha")]), (Alfie, [Write("beta", "alpha"), Write("gamma", "alpha"), Write("alpha", "alpha"), Write("alpha", "alpha"), Write("alpha", "beta"), Write("gamma", "gamma"), Write("alpha", "beta"), Write("gamma", "beta"), Write("gamma", "beta"), Write("beta", "beta"), Write("gamma", "beta"), Write("gamma", "alpha"), Write("beta", "gamma"), Write("beta", "gamma")]), (Betty, [Write("gamma", "alpha"), Write("gamma", "gamma"), Write("gamma", "alpha"), Write("alpha", "gamma"), Write("alpha", "gamma"), Write("beta", "gamma"), Write("beta", "beta"), Write("gamma", "beta"), Write("beta", "alpha")]), (Alfie, [Write("gamma", "alpha"), Write("gamma", "beta")])] } -cc d08210148ec737525059d04cef06d7c8911c32d67ecc9be9c2d9036493a6b0f8 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "alpha"), Write("gamma", "alpha")]), (X, [Write("beta", "gamma"), Write("beta", "alpha"), Write("beta", "beta"), Write("alpha", "beta"), Write("beta", "beta"), Write("gamma", "gamma"), Write("alpha", "gamma"), Write("alpha", "beta"), Write("gamma", "alpha"), Write("beta", "alpha"), Write("beta", "gamma"), Write("gamma", "alpha"), Write("alpha", "beta"), Write("beta", "gamma")]), (Y, [Write("alpha", "alpha"), Write("alpha", "gamma"), Write("beta", "alpha"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "gamma"), Write("alpha", "beta"), Write("beta", "alpha")])] } -cc d1712b6b1cbada8a7fb7793bf1f53d562a2b2abef0842c2f533ec140da37f763 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "green"), Write("beta", "green"), Write("gamma", "green")]), (Y, []), (Y, [Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("alpha", "red")])] } -cc 52d30497f1066cca6e7f81e2e643ed6db7f7308708401f88e238bb0df583f2c1 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("gamma", "blue"), Write("gamma", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("alpha", "green")])] } -cc 81d02a44e9205146e30df256f51c3e306dbcc71ed54b8c5f6d5b9c6b011d73b6 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "red"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue")]), (Y, [Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "red")]), (X, [Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("alpha", "red")]), (X, [Write("beta", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red")]), (Y, [Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "green")]), (Y, [Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("gamma", "green")]), (X, [Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "blue")]), (Y, [Write("beta", "red"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue")]), (X, [Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "red"), Write("alpha", "green")]), (X, [Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("alpha", "red"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "red")]), (X, [Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "red"), Write("gamma", "blue")]), (X, [Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "red")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "green")]), (X, [Write("gamma", "blue")])] } -cc 1bf6826f5eed0f39830227d43f7dfad1e043474fe580ffd44ddbb396631860eb # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "blue"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "red"), Write("alpha", "blue")]), (Y, [Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "blue")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "green")]), (Y, [Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red")]), (Y, [Write("beta", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green")]), (Y, [Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "green")]), (Y, [Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "red")]), (X, [Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green")]), (X, [Write("beta", "red"), Write("alpha", "green"), Write("alpha", "green")]), (X, [Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "blue"), Write("beta", "blue")]), (X, [Write("beta", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("alpha", "red")]), (X, [Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("alpha", "red")]), (X, [Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("beta", "blue")]), (Y, [Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "blue")]), (Y, [Write("beta", "blue"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("gamma", "red")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green"), Write("beta", "red"), Write("gamma", "red"), Write("gamma", "red")]), (X, [Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green")])] } -cc b3451ba74bf6d3578cd59d523609b25e6d26e72d542bfd2928106263ab0c0317 # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("alpha", "blue"), Write("gamma", "red")]), (Y, [Write("beta", "green"), Write("beta", "blue"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "blue")]), (X, []), (X, [Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("gamma", "green"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue")]), (X, [Write("alpha", "green"), Write("gamma", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "blue")]), (Y, [Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("beta", "red"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "green"), Write("beta", "blue")]), (X, [Write("gamma", "green"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red"), Write("alpha", "red")]), (Y, [Write("alpha", "red"), Write("beta", "red")]), (Y, [Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "blue")]), (Y, [Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("beta", "red")]), (Y, [Write("beta", "red"), Write("beta", "green"), Write("beta", "green")]), (X, [Write("gamma", "blue"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("gamma", "green"), Write("gamma", "red")]), (Y, [Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "green"), Write("beta", "green"), Write("alpha", "red"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "green")])] } -cc ba16827de785794376a4a534790de30db562c8ae30d79ce9e9c113215aa7ec4b # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(Y, [Write("beta", "red"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "green")]), (Y, [Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "green"), Write("beta", "green")]), (X, [Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "red")]), (X, [Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "green")]), (Y, [Write("gamma", "blue"), Write("beta", "blue")]), (X, [Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "green")]), (Y, [Write("beta", "blue"), Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "blue"), Write("beta", "green")]), (Y, [Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("alpha", "green"), Write("gamma", "red")]), (Y, [Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "red")]), (Y, [Write("gamma", "green"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("gamma", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "red")]), (Y, [Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("beta", "green")]), (Y, [Write("beta", "green"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("gamma", "red")]), (Y, [Write("gamma", "green"), Write("beta", "blue"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "blue")]), (Y, [Write("alpha", "green"), Write("beta", "red"), Write("gamma", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "green")]), (X, [Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("beta", "red"), Write("beta", "blue"), Write("gamma", "red"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("gamma", "red"), Write("alpha", "red"), Write("beta", "green"), Write("gamma", "red"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "blue"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "red"), Write("beta", "red"), Write("beta", "red"), Write("alpha", "green")]), (Y, [Write("gamma", "green")])] } -cc f7035816a33aa12aab5db8a46f69789d339c4610dc800ced7d359806511a4b7a # shrinks to input = _TestGetManyWeirdResultArgs { rounds: [(X, [Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "green"), Write("beta", "red")]), (X, [Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "green"), Write("beta", "red"), Write("alpha", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("beta", "blue"), Write("alpha", "green")]), (Y, [Write("gamma", "blue"), Write("gamma", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "green"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "red"), Write("alpha", "red"), Write("gamma", "red"), Write("beta", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "red")]), (X, [Write("beta", "red"), Write("alpha", "red"), Write("gamma", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "blue"), Write("alpha", "red"), Write("beta", "red"), Write("beta", "red")]), (X, [Write("beta", "red"), Write("gamma", "red"), Write("beta", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "green")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "blue"), Write("alpha", "red"), Write("beta", "red")]), (X, [Write("beta", "green"), Write("alpha", "blue"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "red"), Write("beta", "green"), Write("alpha", "red"), Write("alpha", "blue"), Write("beta", "green"), Write("beta", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "red"), Write("beta", "green"), Write("gamma", "green"), Write("beta", "green"), Write("alpha", "green")]), (X, [Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "red"), Write("alpha", "green"), Write("beta", "blue"), Write("beta", "red"), Write("beta", "blue"), Write("beta", "red"), Write("alpha", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "red")]), (Y, [Write("alpha", "green"), Write("beta", "blue"), Write("alpha", "red"), Write("gamma", "blue"), Write("alpha", "red"), Write("alpha", "green"), Write("alpha", "green"), Write("beta", "red"), Write("alpha", "red"), Write("gamma", "green"), Write("alpha", "red"), Write("alpha", "green")]), (X, [Write("alpha", "red"), Write("alpha", "blue"), Write("alpha", "blue")]), (X, [Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "blue"), Write("gamma", "red"), Write("alpha", "blue"), Write("beta", "red"), Write("beta", "red"), Write("beta", "green"), Write("alpha", "green"), Write("beta", "red"), Write("beta", "green")]), (X, [Write("gamma", "blue"), Write("beta", "green"), Write("gamma", "green"), Write("alpha", "blue"), Write("alpha", "green"), Write("beta", "green"), Write("alpha", "red"), Write("gamma", "green")]), (X, [Write("alpha", "blue"), Write("alpha", "blue"), Write("alpha", "green"), Write("gamma", "red"), Write("alpha", "blue"), Write("gamma", "green"), Write("gamma", "green")]), (X, [Write("gamma", "green"), Write("alpha", "blue"), Write("beta", "blue")]), (X, [Write("gamma", "green"), Write("gamma", "green")]), (Y, [Write("alpha", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "red"), Write("beta", "blue")]), (Y, [Write("beta", "red"), Write("gamma", "blue"), Write("beta", "blue"), Write("alpha", "blue"), Write("beta", "green"), Write("alpha", "green"), Write("alpha", "green"), Write("alpha", "red"), Write("gamma", "red"), Write("gamma", "red"), Write("gamma", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "blue"), Write("gamma", "blue"), Write("beta", "red"), Write("gamma", "blue")]), (Y, [Write("beta", "green"), Write("alpha", "red"), Write("alpha", "green"), Write("gamma", "green"), Write("alpha", "green"), Write("beta", "green"), Write("gamma", "blue"), Write("beta", "blue"), Write("gamma", "blue"), Write("alpha", "blue"), Write("beta", "red"), Write("gamma", "blue"), Write("gamma", "blue"), Write("gamma", "green"), Write("gamma", "red"), Write("alpha", "red"), Write("gamma", "red")])] } diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs deleted file mode 100644 index ab924aa5d7d..00000000000 --- a/iroh/tests/sync.rs +++ /dev/null @@ -1,1375 +0,0 @@ -use std::{ - collections::HashMap, - future::Future, - sync::Arc, - time::{Duration, Instant}, -}; - -use anyhow::{anyhow, bail, Context, Result}; -use bytes::Bytes; -use futures_lite::Stream; -use futures_util::{FutureExt, StreamExt, TryStreamExt}; -use iroh::{ - base::node_addr::AddrInfoOptions, - client::{ - docs::{Entry, LiveEvent, ShareMode}, - Doc, - }, - net::key::{PublicKey, SecretKey}, - node::{Builder, Node}, -}; -use iroh_blobs::Hash; -use iroh_docs::{ - store::{DownloadPolicy, FilterKind, Query}, - AuthorId, ContentStatus, -}; -use iroh_net::relay::RelayMode; -use rand::{CryptoRng, Rng, SeedableRng}; -use tracing::{debug, error_span, info, Instrument}; -use tracing_subscriber::{prelude::*, EnvFilter}; - -const TIMEOUT: Duration = Duration::from_secs(60); - -fn test_node(secret_key: SecretKey) -> Builder { - Node::memory() - .secret_key(secret_key) - .enable_docs() - .relay_mode(RelayMode::Disabled) -} - -// The function is not `async fn` so that we can take a `&mut` borrow on the `rng` without -// capturing that `&mut` lifetime in the returned future. This allows to call it in a loop while -// still collecting the futures before awaiting them altogether (see [`spawn_nodes`]) -fn spawn_node( - i: usize, - rng: &mut (impl CryptoRng + Rng), -) -> impl Future>> + 'static { - let secret_key = SecretKey::generate_with_rng(rng); - async move { - let node = test_node(secret_key); - let node = node.spawn().await?; - info!(?i, me = %node.node_id().fmt_short(), "node spawned"); - Ok(node) - } -} - -async fn spawn_nodes( - n: usize, - mut rng: &mut (impl CryptoRng + Rng), -) -> anyhow::Result>> { - let mut futs = vec![]; - for i in 0..n { - futs.push(spawn_node(i, &mut rng)); - } - futures_buffered::join_all(futs).await.into_iter().collect() -} - -pub fn test_rng(seed: &[u8]) -> rand_chacha::ChaCha12Rng { - rand_chacha::ChaCha12Rng::from_seed(*Hash::new(seed).as_bytes()) -} - -macro_rules! match_event { - ($pattern:pat $(if $guard:expr)? $(,)?) => { - Box::new(move |e| matches!(e, $pattern $(if $guard)?)) - }; -} - -/// This tests the simplest scenario: A node connects to another node, and performs sync. -#[tokio::test] -async fn sync_simple() -> Result<()> { - setup_logging(); - let mut rng = test_rng(b"sync_simple"); - let nodes = spawn_nodes(2, &mut rng).await?; - let clients = nodes.iter().map(|node| node.client()).collect::>(); - - // create doc on node0 - let peer0 = nodes[0].node_id(); - let author0 = clients[0].authors().create().await?; - let doc0 = clients[0].docs().create().await?; - let hash0 = doc0 - .set_bytes(author0, b"k1".to_vec(), b"v1".to_vec()) - .await?; - assert_latest(&doc0, b"k1", b"v1").await; - let ticket = doc0 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - - let mut events0 = doc0.subscribe().await?; - - info!("node1: join"); - let peer1 = nodes[1].node_id(); - let doc1 = clients[1].docs().import(ticket.clone()).await?; - let mut events1 = doc1.subscribe().await?; - info!("node1: assert 5 events"); - assert_next_unordered( - &mut events1, - TIMEOUT, - vec![ - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer0)), - Box::new(move |e| matches!(e, LiveEvent::InsertRemote { from, .. } if *from == peer0 )), - Box::new(move |e| match_sync_finished(e, peer0)), - Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - assert_latest(&doc1, b"k1", b"v1").await; - - info!("node0: assert 2 events"); - assert_next( - &mut events0, - TIMEOUT, - vec![ - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer1)), - Box::new(move |e| match_sync_finished(e, peer1)), - ], - ) - .await; - - for node in nodes { - node.shutdown().await?; - } - Ok(()) -} - -/// Test subscribing to replica events (without sync) -#[tokio::test] -async fn sync_subscribe_no_sync() -> Result<()> { - let mut rng = test_rng(b"sync_subscribe"); - setup_logging(); - let node = spawn_node(0, &mut rng).await?; - let client = node.client(); - let doc = client.docs().create().await?; - let mut sub = doc.subscribe().await?; - let author = client.authors().create().await?; - doc.set_bytes(author, b"k".to_vec(), b"v".to_vec()).await?; - let event = tokio::time::timeout(Duration::from_millis(100), sub.next()).await?; - assert!( - matches!(event, Some(Ok(LiveEvent::InsertLocal { .. }))), - "expected InsertLocal but got {event:?}" - ); - node.shutdown().await?; - Ok(()) -} - -#[tokio::test] -async fn sync_gossip_bulk() -> Result<()> { - let n_entries: usize = std::env::var("N_ENTRIES") - .map(|x| x.parse().expect("N_ENTRIES must be a number")) - .unwrap_or(100); - let mut rng = test_rng(b"sync_gossip_bulk"); - setup_logging(); - - let nodes = spawn_nodes(2, &mut rng).await?; - let clients = nodes.iter().map(|node| node.client()).collect::>(); - - let _peer0 = nodes[0].node_id(); - let author0 = clients[0].authors().create().await?; - let doc0 = clients[0].docs().create().await?; - let mut ticket = doc0 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - // unset peers to not yet start sync - let peers = ticket.nodes.clone(); - ticket.nodes = vec![]; - let doc1 = clients[1].docs().import(ticket).await?; - let mut events = doc1.subscribe().await?; - - // create entries for initial sync. - let now = Instant::now(); - let value = b"foo"; - for i in 0..n_entries { - let key = format!("init/{i}"); - doc0.set_bytes(author0, key.as_bytes().to_vec(), value.to_vec()) - .await?; - } - let elapsed = now.elapsed(); - info!( - "insert took {elapsed:?} for {n_entries} ({:?} per entry)", - elapsed / n_entries as u32 - ); - - let now = Instant::now(); - let mut count = 0; - doc0.start_sync(vec![]).await?; - doc1.start_sync(peers).await?; - while let Some(event) = events.next().await { - let event = event?; - if matches!(event, LiveEvent::InsertRemote { .. }) { - count += 1; - } - if count == n_entries { - break; - } - } - let elapsed = now.elapsed(); - info!( - "initial sync took {elapsed:?} for {n_entries} ({:?} per entry)", - elapsed / n_entries as u32 - ); - - // publish another 1000 entries - let mut count = 0; - let value = b"foo"; - let now = Instant::now(); - for i in 0..n_entries { - let key = format!("gossip/{i}"); - doc0.set_bytes(author0, key.as_bytes().to_vec(), value.to_vec()) - .await?; - } - let elapsed = now.elapsed(); - info!( - "insert took {elapsed:?} for {n_entries} ({:?} per entry)", - elapsed / n_entries as u32 - ); - - while let Some(event) = events.next().await { - let event = event?; - if matches!(event, LiveEvent::InsertRemote { .. }) { - count += 1; - } - if count == n_entries { - break; - } - } - let elapsed = now.elapsed(); - info!( - "gossip recv took {elapsed:?} for {n_entries} ({:?} per entry)", - elapsed / n_entries as u32 - ); - - Ok(()) -} - -/// This tests basic sync and gossip with 3 peers. -#[tokio::test] -#[ignore = "flaky"] -async fn sync_full_basic() -> Result<()> { - let mut rng = test_rng(b"sync_full_basic"); - setup_logging(); - let mut nodes = spawn_nodes(2, &mut rng).await?; - let mut clients = nodes - .iter() - .map(|node| node.client().clone()) - .collect::>(); - - // peer0: create doc and ticket - let peer0 = nodes[0].node_id(); - let author0 = clients[0].authors().create().await?; - let doc0 = clients[0].docs().create().await?; - let mut events0 = doc0.subscribe().await?; - let key0 = b"k1"; - let value0 = b"v1"; - let hash0 = doc0 - .set_bytes(author0, key0.to_vec(), value0.to_vec()) - .await?; - - info!("peer0: wait for 1 event (local insert)"); - let e = next(&mut events0).await; - assert!( - matches!(&e, LiveEvent::InsertLocal { entry } if entry.content_hash() == hash0), - "expected LiveEvent::InsertLocal but got {e:?}", - ); - assert_latest(&doc0, key0, value0).await; - let ticket = doc0 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - - info!("peer1: spawn"); - let peer1 = nodes[1].node_id(); - let author1 = clients[1].authors().create().await?; - info!("peer1: join doc"); - let doc1 = clients[1].docs().import(ticket.clone()).await?; - - info!("peer1: wait for 4 events (for sync and join with peer0)"); - let mut events1 = doc1.subscribe().await?; - assert_next_unordered( - &mut events1, - TIMEOUT, - vec![ - match_event!(LiveEvent::NeighborUp(peer) if *peer == peer0), - match_event!(LiveEvent::InsertRemote { from, .. } if *from == peer0 ), - Box::new(move |e| match_sync_finished(e, peer0)), - match_event!(LiveEvent::ContentReady { hash } if *hash == hash0), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - - info!("peer0: wait for 2 events (join & accept sync finished from peer1)"); - assert_next( - &mut events0, - TIMEOUT, - vec![ - match_event!(LiveEvent::NeighborUp(peer) if *peer == peer1), - Box::new(move |e| match_sync_finished(e, peer1)), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - - info!("peer1: insert entry"); - let key1 = b"k2"; - let value1 = b"v2"; - let hash1 = doc1 - .set_bytes(author1, key1.to_vec(), value1.to_vec()) - .await?; - assert_latest(&doc1, key1, value1).await; - info!("peer1: wait for 1 event (local insert, and pendingcontentready)"); - assert_next( - &mut events1, - TIMEOUT, - vec![match_event!(LiveEvent::InsertLocal { entry} if entry.content_hash() == hash1)], - ) - .await; - - // peer0: assert events for entry received via gossip - info!("peer0: wait for 2 events (gossip'ed entry from peer1)"); - assert_next( - &mut events0, - TIMEOUT, - vec![ - Box::new( - move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == peer1), - ), - Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash1)), - ], - ).await; - assert_latest(&doc0, key1, value1).await; - - // Note: If we could check gossip messages directly here (we can't easily), we would notice - // that peer1 will receive a `Op::ContentReady` gossip message, broadcast - // by peer0 with neighbor scope. This message is superfluous, and peer0 could know that, however - // our gossip implementation does not allow us to filter message receivers this way. - - info!("peer2: spawn"); - nodes.push(spawn_node(nodes.len(), &mut rng).await?); - clients.push(nodes.last().unwrap().client().clone()); - let doc2 = clients[2].docs().import(ticket).await?; - let peer2 = nodes[2].node_id(); - let mut events2 = doc2.subscribe().await?; - - info!("peer2: wait for 9 events (from sync with peers)"); - assert_next_unordered_with_optionals( - &mut events2, - TIMEOUT, - // required events - vec![ - // 2 NeighborUp events - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer0)), - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer1)), - // 2 SyncFinished events - Box::new(move |e| match_sync_finished(e, peer0)), - Box::new(move |e| match_sync_finished(e, peer1)), - // 2 InsertRemote events - Box::new( - move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing, .. } if entry.content_hash() == hash0), - ), - Box::new( - move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing, .. } if entry.content_hash() == hash1), - ), - // 2 ContentReady events - Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)), - Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash1)), - // at least 1 PendingContentReady - match_event!(LiveEvent::PendingContentReady), - ], - // optional events - // it may happen that we run sync two times against our two peers: - // if the first sync (as a result of us joining the peer manually through the ticket) completes - // before the peer shows up as a neighbor, we run sync again for the NeighborUp event. - vec![ - // 2 SyncFinished events - Box::new(move |e| match_sync_finished(e, peer0)), - Box::new(move |e| match_sync_finished(e, peer1)), - match_event!(LiveEvent::PendingContentReady), - match_event!(LiveEvent::PendingContentReady), - ] - ).await; - assert_latest(&doc2, b"k1", b"v1").await; - assert_latest(&doc2, b"k2", b"v2").await; - - info!("peer0: wait for 2 events (join & accept sync finished from peer2)"); - assert_next( - &mut events0, - TIMEOUT, - vec![ - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer2)), - Box::new(move |e| match_sync_finished(e, peer2)), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - - info!("peer1: wait for 2 events (join & accept sync finished from peer2)"); - assert_next( - &mut events1, - TIMEOUT, - vec![ - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer2)), - Box::new(move |e| match_sync_finished(e, peer2)), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - - info!("shutdown"); - for node in nodes { - node.shutdown().await?; - } - - Ok(()) -} - -#[tokio::test] -async fn sync_open_close() -> Result<()> { - let mut rng = test_rng(b"sync_subscribe_stop_close"); - setup_logging(); - let node = spawn_node(0, &mut rng).await?; - let client = node.client(); - - let doc = client.docs().create().await?; - let status = doc.status().await?; - assert_eq!(status.handles, 1); - - let doc2 = client.docs().open(doc.id()).await?.unwrap(); - let status = doc2.status().await?; - assert_eq!(status.handles, 2); - - doc.close().await?; - assert!(doc.status().await.is_err()); - - let status = doc2.status().await?; - assert_eq!(status.handles, 1); - - Ok(()) -} - -#[tokio::test] -async fn sync_subscribe_stop_close() -> Result<()> { - let mut rng = test_rng(b"sync_subscribe_stop_close"); - setup_logging(); - let node = spawn_node(0, &mut rng).await?; - let client = node.client(); - - let doc = client.docs().create().await?; - let author = client.authors().create().await?; - - let status = doc.status().await?; - assert_eq!(status.subscribers, 0); - assert_eq!(status.handles, 1); - assert!(!status.sync); - - doc.start_sync(vec![]).await?; - let status = doc.status().await?; - assert!(status.sync); - assert_eq!(status.handles, 2); - assert_eq!(status.subscribers, 1); - - let sub = doc.subscribe().await?; - let status = doc.status().await?; - assert_eq!(status.subscribers, 2); - drop(sub); - // trigger an event that makes the actor check if the event channels are still connected - doc.set_bytes(author, b"x".to_vec(), b"x".to_vec()).await?; - let status = doc.status().await?; - assert_eq!(status.subscribers, 1); - - doc.leave().await?; - let status = doc.status().await?; - assert_eq!(status.subscribers, 0); - assert_eq!(status.handles, 1); - assert!(!status.sync); - - Ok(()) -} - -#[tokio::test] -#[cfg(feature = "test-utils")] -async fn test_sync_via_relay() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let (relay_map, _relay_url, _guard) = iroh_net::test_utils::run_relay_server().await?; - - let node1 = Node::memory() - .relay_mode(RelayMode::Custom(relay_map.clone())) - .insecure_skip_relay_cert_verify(true) - .enable_docs() - .spawn() - .await?; - let node1_id = node1.node_id(); - let node2 = Node::memory() - .bind_random_port() - .relay_mode(RelayMode::Custom(relay_map.clone())) - .insecure_skip_relay_cert_verify(true) - .enable_docs() - .spawn() - .await?; - - let doc1 = node1.docs().create().await?; - let author1 = node1.authors().create().await?; - let inserted_hash = doc1 - .set_bytes(author1, b"foo".to_vec(), b"bar".to_vec()) - .await?; - let mut ticket = doc1 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - - // remove direct addrs to force connect via relay - ticket.nodes[0].info.direct_addresses = Default::default(); - - // join - let doc2 = node2.docs().import(ticket).await?; - let mut events = doc2.subscribe().await?; - - assert_next_unordered_with_optionals( - &mut events, - Duration::from_secs(2), - vec![ - Box::new(move |e| matches!(e, LiveEvent::NeighborUp(n) if *n== node1_id)), - Box::new(move |e| match_sync_finished(e, node1_id)), - Box::new( - move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == node1_id), - ), - Box::new( - move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == inserted_hash), - ), - match_event!(LiveEvent::PendingContentReady), - ], - vec![Box::new(move |e| match_sync_finished(e, node1_id))], - ).await; - let actual = doc2 - .get_exact(author1, b"foo", false) - .await? - .expect("entry to exist") - .content_bytes(&doc2) - .await?; - assert_eq!(actual.as_ref(), b"bar"); - - // update - let updated_hash = doc1 - .set_bytes(author1, b"foo".to_vec(), b"update".to_vec()) - .await?; - assert_next_unordered_with_optionals( - &mut events, - Duration::from_secs(2), - vec![ - Box::new( - move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == node1_id), - ), - Box::new( - move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == updated_hash), - ), - ], - vec![ - Box::new(move |e| match_sync_finished(e, node1_id)), - Box::new(move |e| matches!(e, LiveEvent::PendingContentReady)), - ], - ).await; - let actual = doc2 - .get_exact(author1, b"foo", false) - .await? - .expect("entry to exist") - .content_bytes(&doc2) - .await?; - assert_eq!(actual.as_ref(), b"update"); - Ok(()) -} - -#[tokio::test] -#[cfg(feature = "test-utils")] -#[ignore = "flaky"] -async fn sync_restart_node() -> Result<()> { - let mut rng = test_rng(b"sync_restart_node"); - setup_logging(); - let (relay_map, _relay_url, _guard) = iroh_net::test_utils::run_relay_server().await?; - - let discovery_server = iroh_net::test_utils::DnsPkarrServer::run().await?; - - let node1_dir = tempfile::TempDir::with_prefix("test-sync_restart_node-node1")?; - let secret_key_1 = SecretKey::generate_with_rng(&mut rng); - - let node1 = Node::persistent(&node1_dir) - .await? - .secret_key(secret_key_1.clone()) - .insecure_skip_relay_cert_verify(true) - .relay_mode(RelayMode::Custom(relay_map.clone())) - .dns_resolver(discovery_server.dns_resolver()) - .node_discovery(discovery_server.discovery(secret_key_1.clone()).into()) - .enable_docs() - .spawn() - .await?; - let id1 = node1.node_id(); - - // create doc & ticket on node1 - let doc1 = node1.docs().create().await?; - let mut events1 = doc1.subscribe().await?; - let ticket = doc1 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - - // create node2 - let secret_key_2 = SecretKey::generate_with_rng(&mut rng); - let node2 = Node::memory() - .secret_key(secret_key_2.clone()) - .relay_mode(RelayMode::Custom(relay_map.clone())) - .insecure_skip_relay_cert_verify(true) - .dns_resolver(discovery_server.dns_resolver()) - .node_discovery(discovery_server.discovery(secret_key_2.clone()).into()) - .enable_docs() - .spawn() - .await?; - let id2 = node2.node_id(); - let author2 = node2.authors().create().await?; - let doc2 = node2.docs().import(ticket.clone()).await?; - - info!("node2 set a"); - let hash_a = doc2.set_bytes(author2, "n2/a", "a").await?; - assert_latest(&doc2, b"n2/a", b"a").await; - - assert_next_unordered_with_optionals( - &mut events1, - Duration::from_secs(10), - vec![ - match_event!(LiveEvent::NeighborUp(n) if *n == id2), - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2), - match_event!(LiveEvent::ContentReady { hash } if *hash == hash_a), - match_event!(LiveEvent::PendingContentReady), - ], - vec![ - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::PendingContentReady), - ], - ) - .await; - assert_latest(&doc1, b"n2/a", b"a").await; - - info!(me = id1.fmt_short(), "node1 start shutdown"); - node1.shutdown().await?; - info!(me = id1.fmt_short(), "node1 down"); - - info!(me = id1.fmt_short(), "sleep 1s"); - tokio::time::sleep(Duration::from_secs(1)).await; - - info!(me = id2.fmt_short(), "node2 set b"); - let hash_b = doc2.set_bytes(author2, "n2/b", "b").await?; - - info!(me = id1.fmt_short(), "node1 respawn"); - let node1 = Node::persistent(&node1_dir) - .await? - .secret_key(secret_key_1.clone()) - .insecure_skip_relay_cert_verify(true) - .relay_mode(RelayMode::Custom(relay_map.clone())) - .dns_resolver(discovery_server.dns_resolver()) - .node_discovery(discovery_server.discovery(secret_key_1.clone()).into()) - .enable_docs() - .spawn() - .await?; - assert_eq!(id1, node1.node_id()); - - let doc1 = node1.docs().open(doc1.id()).await?.expect("doc to exist"); - let mut events1 = doc1.subscribe().await?; - assert_latest(&doc1, b"n2/a", b"a").await; - - // check that initial resync is working - doc1.start_sync(vec![]).await?; - assert_next_unordered_with_optionals( - &mut events1, - Duration::from_secs(10), - vec![ - match_event!(LiveEvent::NeighborUp(n) if *n== id2), - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2), - match_event!(LiveEvent::ContentReady { hash } if *hash == hash_b), - ], - vec![ - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::PendingContentReady), - ] - ).await; - assert_latest(&doc1, b"n2/b", b"b").await; - - // check that live conn is working - info!(me = id2.fmt_short(), "node2 set c"); - let hash_c = doc2.set_bytes(author2, "n2/c", "c").await?; - assert_next_unordered_with_optionals( - &mut events1, - Duration::from_secs(10), - vec![ - match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2), - match_event!(LiveEvent::ContentReady { hash } if *hash == hash_c), - ], - vec![ - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::PendingContentReady), - match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()), - match_event!(LiveEvent::PendingContentReady), - ] - ).await; - - assert_latest(&doc1, b"n2/c", b"c").await; - - Ok(()) -} - -/// Joins two nodes that write to the same document but have differing download policies and tests -/// that they both synced the key info but not the content. -#[tokio::test] -async fn test_download_policies() -> Result<()> { - // keys node a has - let star_wars_movies = &[ - "star_wars/prequel/the_phantom_menace", - "star_wars/prequel/attack_of_the_clones", - "star_wars/prequel/revenge_of_the_sith", - "star_wars/og/a_new_hope", - "star_wars/og/the_empire_strikes_back", - "star_wars/og/return_of_the_jedi", - ]; - // keys node b has - let lotr_movies = &[ - "lotr/fellowship_of_the_ring", - "lotr/the_two_towers", - "lotr/return_of_the_king", - ]; - - // content policy for what b wants - let policy_b = - DownloadPolicy::EverythingExcept(vec![FilterKind::Prefix("star_wars/og".into())]); - // content policy for what a wants - let policy_a = DownloadPolicy::NothingExcept(vec![FilterKind::Exact( - "lotr/fellowship_of_the_ring".into(), - )]); - - // a will sync all lotr keys but download a single key - const EXPECTED_A_SYNCED: usize = 3; - const EXPECTED_A_DOWNLOADED: usize = 1; - - // b will sync all star wars content but download only the prequel keys - const EXPECTED_B_SYNCED: usize = 6; - const EXPECTED_B_DOWNLOADED: usize = 3; - - let mut rng = test_rng(b"sync_download_policies"); - setup_logging(); - let nodes = spawn_nodes(2, &mut rng).await?; - let clients = nodes.iter().map(|node| node.client()).collect::>(); - - let doc_a = clients[0].docs().create().await?; - let author_a = clients[0].authors().create().await?; - let ticket = doc_a - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - - let doc_b = clients[1].docs().import(ticket).await?; - let author_b = clients[1].authors().create().await?; - - doc_a.set_download_policy(policy_a).await?; - doc_b.set_download_policy(policy_b).await?; - - let mut events_a = doc_a.subscribe().await?; - let mut events_b = doc_b.subscribe().await?; - - let mut key_hashes: HashMap = HashMap::default(); - - // set content in a - for k in star_wars_movies.iter() { - let hash = doc_a - .set_bytes(author_a, k.to_owned(), k.to_owned()) - .await?; - key_hashes.insert(hash, k); - } - - // set content in b - for k in lotr_movies.iter() { - let hash = doc_b - .set_bytes(author_b, k.to_owned(), k.to_owned()) - .await?; - key_hashes.insert(hash, k); - } - - assert_eq!(key_hashes.len(), star_wars_movies.len() + lotr_movies.len()); - - let fut = async { - use LiveEvent::*; - let mut downloaded_a: Vec<&'static str> = Vec::new(); - let mut downloaded_b: Vec<&'static str> = Vec::new(); - let mut synced_a = 0usize; - let mut synced_b = 0usize; - loop { - tokio::select! { - Some(Ok(ev)) = events_a.next() => { - match ev { - InsertRemote { content_status, entry, .. } => { - synced_a += 1; - if let ContentStatus::Complete = content_status { - downloaded_a.push(key_hashes.get(&entry.content_hash()).unwrap()) - } - }, - ContentReady { hash } => { - downloaded_a.push(key_hashes.get(&hash).unwrap()); - }, - _ => {} - } - } - Some(Ok(ev)) = events_b.next() => { - match ev { - InsertRemote { content_status, entry, .. } => { - synced_b += 1; - if let ContentStatus::Complete = content_status { - downloaded_b.push(key_hashes.get(&entry.content_hash()).unwrap()) - } - }, - ContentReady { hash } => { - downloaded_b.push(key_hashes.get(&hash).unwrap()); - }, - _ => {} - } - } - } - - if synced_a == EXPECTED_A_SYNCED - && downloaded_a.len() == EXPECTED_A_DOWNLOADED - && synced_b == EXPECTED_B_SYNCED - && downloaded_b.len() == EXPECTED_B_DOWNLOADED - { - break; - } - } - (downloaded_a, downloaded_b) - }; - - let (downloaded_a, mut downloaded_b) = tokio::time::timeout(TIMEOUT, fut) - .await - .context("timeout elapsed")?; - - downloaded_b.sort(); - assert_eq!(downloaded_a, vec!["lotr/fellowship_of_the_ring"]); - assert_eq!( - downloaded_b, - vec![ - "star_wars/prequel/attack_of_the_clones", - "star_wars/prequel/revenge_of_the_sith", - "star_wars/prequel/the_phantom_menace", - ] - ); - - Ok(()) -} - -/// Test sync between many nodes with propagation through sync reports. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "flaky"] -async fn sync_big() -> Result<()> { - setup_logging(); - let mut rng = test_rng(b"sync_big"); - let n_nodes = std::env::var("NODES") - .map(|v| v.parse().expect("NODES must be a number")) - .unwrap_or(10); - let n_entries_init = 1; - - tokio::task::spawn(async move { - for i in 0.. { - tokio::time::sleep(Duration::from_secs(1)).await; - info!("tick {i}"); - } - }); - - let nodes = spawn_nodes(n_nodes, &mut rng).await?; - let node_ids = nodes.iter().map(|node| node.node_id()).collect::>(); - let clients = nodes.iter().map(|node| node.client()).collect::>(); - let authors = collect_futures(clients.iter().map(|c| c.authors().create())).await?; - - let doc0 = clients[0].docs().create().await?; - let mut ticket = doc0 - .share(ShareMode::Write, AddrInfoOptions::RelayAndAddresses) - .await?; - // do not join for now, just import without any peer info - let peer0 = ticket.nodes[0].clone(); - ticket.nodes = vec![]; - - let mut docs = vec![]; - docs.push(doc0); - docs.extend_from_slice( - &collect_futures( - clients - .iter() - .skip(1) - .map(|c| c.docs().import(ticket.clone())), - ) - .await?, - ); - - let mut expected = vec![]; - - // create initial data on each node - publish(&docs, &mut expected, n_entries_init, |i, j| { - ( - authors[i], - format!("init/{}/{j}", node_ids[i].fmt_short()), - format!("init:{i}:{j}"), - ) - }) - .await?; - - // assert initial data - for (i, doc) in docs.iter().enumerate() { - let entries = get_all_with_content(doc).await?; - let mut expected = expected - .iter() - .filter(|e| e.author == authors[i]) - .cloned() - .collect::>(); - expected.sort(); - assert_eq!(entries, expected, "phase1 pre-sync correct"); - } - - // setup event streams - let events = collect_futures(docs.iter().map(|d| d.subscribe())).await?; - - // join nodes together - for (i, doc) in docs.iter().enumerate().skip(1) { - info!(me = %node_ids[i].fmt_short(), peer = %peer0.node_id.fmt_short(), "join"); - doc.start_sync(vec![peer0.clone()]).await?; - } - - // wait for InsertRemote events stuff to happen - info!("wait for all peers to receive insert events"); - let expected_inserts = (n_nodes - 1) * n_entries_init; - let mut tasks = tokio::task::JoinSet::default(); - for (i, events) in events.into_iter().enumerate() { - let doc = docs[i].clone(); - let me = doc.id().fmt_short(); - let expected = expected.clone(); - let fut = async move { - wait_for_events(events, expected_inserts, TIMEOUT, |e| { - matches!(e, LiveEvent::InsertRemote { .. }) - }) - .await?; - let entries = get_all(&doc).await?; - if entries != expected { - Err(anyhow!( - "node {i} failed (has {} entries but expected to have {})", - entries.len(), - expected.len() - )) - } else { - info!( - "received and checked all {} expected entries", - expected.len() - ); - Ok(()) - } - } - .instrument(error_span!("sync-test", %me)); - let fut = fut.map(move |r| r.with_context(move || format!("node {i} ({me})"))); - tasks.spawn(fut); - } - - while let Some(res) = tasks.join_next().await { - res??; - } - - assert_all_docs(&docs, &node_ids, &expected, "after initial sync").await; - - info!("shutdown"); - for node in nodes { - node.shutdown().await?; - } - - Ok(()) -} - -#[tokio::test] -#[cfg(feature = "test-utils")] -async fn test_list_docs_stream() -> Result<()> { - let node = Node::memory() - .node_discovery(iroh::node::DiscoveryConfig::None) - .relay_mode(iroh::net::relay::RelayMode::Disabled) - .enable_docs() - .spawn() - .await?; - let count = 200; - - // create docs - for _i in 0..count { - let doc = node.docs().create().await?; - doc.close().await?; - } - - // create doc stream - let mut stream = node.docs().list().await?; - - // process each doc and call into the docs actor. - // this makes sure that we don't deadlock the docs actor. - let mut i = 0; - let fut = async { - while let Some((id, _)) = stream.try_next().await.unwrap() { - let _doc = node.docs().open(id).await.unwrap().unwrap(); - i += 1; - } - }; - - tokio::time::timeout(Duration::from_secs(2), fut) - .await - .expect("not to timeout"); - - assert_eq!(i, count); - - Ok(()) -} - -/// Get all entries of a document. -async fn get_all(doc: &Doc) -> anyhow::Result> { - let entries = doc.get_many(Query::all()).await?; - let entries = entries.collect::>().await; - entries.into_iter().collect() -} - -/// Get all entries of a document with the blob content. -async fn get_all_with_content(doc: &Doc) -> anyhow::Result> { - let entries = doc.get_many(Query::all()).await?; - let entries = entries.and_then(|entry| async { - let content = entry.content_bytes(doc).await; - content.map(|c| (entry, c)) - }); - let entries = entries.collect::>().await; - let entries = entries.into_iter().collect::>>()?; - Ok(entries) -} - -async fn publish( - docs: &[Doc], - expected: &mut Vec, - n: usize, - cb: impl Fn(usize, usize) -> (AuthorId, String, String), -) -> anyhow::Result<()> { - for (i, doc) in docs.iter().enumerate() { - for j in 0..n { - let (author, key, value) = cb(i, j); - doc.set_bytes(author, key.as_bytes().to_vec(), value.as_bytes().to_vec()) - .await?; - expected.push(ExpectedEntry { author, key, value }); - } - } - expected.sort(); - Ok(()) -} - -/// Collect an iterator into futures by joining them all and failing if any future failed. -async fn collect_futures( - futs: impl IntoIterator>>, -) -> anyhow::Result> { - futures_buffered::join_all(futs) - .await - .into_iter() - .collect::>>() -} - -/// Collect `count` events from the `events` stream, only collecting events for which `matcher` -/// returns true. -async fn wait_for_events( - mut events: impl Stream> + Send + Unpin + 'static, - count: usize, - timeout: Duration, - matcher: impl Fn(&LiveEvent) -> bool, -) -> anyhow::Result> { - let mut res = Vec::with_capacity(count); - let sleep = tokio::time::sleep(timeout); - tokio::pin!(sleep); - while res.len() < count { - tokio::select! { - () = &mut sleep => { - bail!("Failed to collect {count} elements in {timeout:?} (collected only {})", res.len()); - }, - event = events.try_next() => { - let event = event?; - match event { - None => bail!("stream ended after {} items, but expected {count}", res.len()), - Some(event) => if matcher(&event) { - res.push(event); - debug!("recv event {} of {count}", res.len()); - } - } - } - } - } - Ok(res) -} - -async fn assert_all_docs( - docs: &[Doc], - node_ids: &[PublicKey], - expected: &Vec, - label: &str, -) { - info!("validate all peers: {label}"); - for (i, doc) in docs.iter().enumerate() { - let entries = get_all(doc).await.unwrap_or_else(|err| { - panic!("failed to get entries for peer {:?}: {err:?}", node_ids[i]) - }); - assert_eq!( - &entries, - expected, - "{label}: peer {i} {:?} failed (have {} but expected {})", - node_ids[i], - entries.len(), - expected.len() - ); - } -} - -#[derive(Debug, Ord, Eq, PartialEq, PartialOrd, Clone)] -struct ExpectedEntry { - author: AuthorId, - key: String, - value: String, -} - -impl PartialEq for ExpectedEntry { - fn eq(&self, other: &Entry) -> bool { - self.key.as_bytes() == other.key() - && Hash::new(&self.value) == other.content_hash() - && self.author == other.author() - } -} -impl PartialEq<(Entry, Bytes)> for ExpectedEntry { - fn eq(&self, (entry, content): &(Entry, Bytes)) -> bool { - self.key.as_bytes() == entry.key() - && Hash::new(&self.value) == entry.content_hash() - && self.author == entry.author() - && self.value.as_bytes() == content.as_ref() - } -} -impl PartialEq for Entry { - fn eq(&self, other: &ExpectedEntry) -> bool { - other.eq(self) - } -} -impl PartialEq for (Entry, Bytes) { - fn eq(&self, other: &ExpectedEntry) -> bool { - other.eq(self) - } -} - -#[tokio::test] -async fn doc_delete() -> Result<()> { - let node = Node::memory() - .gc_policy(iroh::node::GcPolicy::Interval(Duration::from_millis(100))) - .enable_docs() - .spawn() - .await?; - let client = node.client(); - let doc = client.docs().create().await?; - let author = client.authors().create().await?; - let hash = doc - .set_bytes(author, b"foo".to_vec(), b"hi".to_vec()) - .await?; - assert_latest(&doc, b"foo", b"hi").await; - let deleted = doc.del(author, b"foo".to_vec()).await?; - assert_eq!(deleted, 1); - - let entry = doc.get_exact(author, b"foo".to_vec(), false).await?; - assert!(entry.is_none()); - - // wait for gc - // TODO: allow to manually trigger gc - tokio::time::sleep(Duration::from_millis(200)).await; - let bytes = client.blobs().read_to_bytes(hash).await; - assert!(bytes.is_err()); - node.shutdown().await?; - Ok(()) -} - -#[tokio::test] -async fn sync_drop_doc() -> Result<()> { - let mut rng = test_rng(b"sync_drop_doc"); - setup_logging(); - let node = spawn_node(0, &mut rng).await?; - let client = node.client(); - - let doc = client.docs().create().await?; - let author = client.authors().create().await?; - - let mut sub = doc.subscribe().await?; - doc.set_bytes(author, b"foo".to_vec(), b"bar".to_vec()) - .await?; - let ev = sub.next().await; - assert!(matches!(ev, Some(Ok(LiveEvent::InsertLocal { .. })))); - - client.docs().drop_doc(doc.id()).await?; - let res = doc.get_exact(author, b"foo".to_vec(), true).await; - assert!(res.is_err()); - let res = doc - .set_bytes(author, b"foo".to_vec(), b"bar".to_vec()) - .await; - assert!(res.is_err()); - let res = client.docs().open(doc.id()).await; - assert!(res.is_err()); - let ev = sub.next().await; - assert!(ev.is_none()); - - Ok(()) -} - -async fn assert_latest(doc: &Doc, key: &[u8], value: &[u8]) { - let content = get_latest(doc, key).await.unwrap(); - assert_eq!(content, value.to_vec()); -} - -async fn get_latest(doc: &Doc, key: &[u8]) -> anyhow::Result> { - let query = Query::single_latest_per_key().key_exact(key); - let entry = doc - .get_many(query) - .await? - .next() - .await - .ok_or_else(|| anyhow!("entry not found"))??; - let content = entry.content_bytes(doc).await?; - Ok(content.to_vec()) -} - -fn setup_logging() { - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) - .with(EnvFilter::from_default_env()) - .try_init() - .ok(); -} - -async fn next(mut stream: impl Stream> + Unpin) -> T { - let event = stream - .next() - .await - .expect("stream ended") - .expect("stream produced error"); - debug!("Event: {event:?}"); - event -} - -#[allow(clippy::type_complexity)] -fn apply_matchers(item: &T, matchers: &mut Vec bool + Send>>) -> bool { - for i in 0..matchers.len() { - if matchers[i](item) { - let _ = matchers.remove(i); - return true; - } - } - false -} - -/// Receive the next `matchers.len()` elements from a stream and matches them against the functions -/// in `matchers`, in order. -/// -/// Returns all received events. -#[allow(clippy::type_complexity)] -async fn assert_next( - mut stream: impl Stream> + Unpin + Send, - timeout: Duration, - matchers: Vec bool + Send>>, -) -> Vec { - let fut = async { - let mut items = vec![]; - for (i, f) in matchers.iter().enumerate() { - let item = stream - .next() - .await - .expect("event stream ended prematurely") - .expect("event stream errored"); - if !(f)(&item) { - panic!("assertion failed for event {i} {item:?}"); - } - items.push(item); - } - items - }; - let res = tokio::time::timeout(timeout, fut).await; - res.expect("timeout reached") -} - -/// Receive `matchers.len()` elements from a stream and assert that each element matches one of the -/// functions in `matchers`. -/// -/// Order of the matchers is not relevant. -/// -/// Returns all received events. -#[allow(clippy::type_complexity)] -async fn assert_next_unordered( - stream: impl Stream> + Unpin + Send, - timeout: Duration, - matchers: Vec bool + Send>>, -) -> Vec { - assert_next_unordered_with_optionals(stream, timeout, matchers, vec![]).await -} - -/// Receive between `min` and `max` elements from the stream and assert that each element matches -/// either one of the matchers in `required_matchers` or in `optional_matchers`. -/// -/// Order of the matchers is not relevant. -/// -/// Will return an error if: -/// * Any element fails to match one of the required or optional matchers -/// * More than `max` elements were received, but not all required matchers were used yet -/// * The timeout completes before all required matchers were used -/// -/// Returns all received events. -#[allow(clippy::type_complexity)] -async fn assert_next_unordered_with_optionals( - mut stream: impl Stream> + Unpin + Send, - timeout: Duration, - mut required_matchers: Vec bool + Send>>, - mut optional_matchers: Vec bool + Send>>, -) -> Vec { - let max = required_matchers.len() + optional_matchers.len(); - let required = required_matchers.len(); - // we have to use a mutex because rustc is not intelligent enough to realize - // that the mutable borrow terminates when the future completes - let events = Arc::new(parking_lot::Mutex::new(vec![])); - let fut = async { - while let Some(event) = stream.next().await { - let event = event.context("failed to read from stream")?; - let len = { - let mut events = events.lock(); - events.push(event.clone()); - events.len() - }; - if !apply_matchers(&event, &mut required_matchers) - && !apply_matchers(&event, &mut optional_matchers) - { - bail!("Event didn't match any matcher: {event:?}"); - } - if required_matchers.is_empty() || len == max { - break; - } - } - if !required_matchers.is_empty() { - bail!( - "Matched only {} of {required} required matchers", - required - required_matchers.len() - ); - } - Ok(()) - }; - tokio::pin!(fut); - let res = tokio::time::timeout(timeout, fut) - .await - .map_err(|_| anyhow!("Timeout reached ({timeout:?})")) - .and_then(|res| res); - let events = events.lock().clone(); - if let Err(err) = &res { - println!("Received events: {events:#?}"); - println!( - "Received {} events, expected between {required} and {max}", - events.len() - ); - panic!("Failed to receive or match all events: {err:?}"); - } - events -} - -/// Asserts that the event is a [`LiveEvent::SyncFinished`] and that the contained [`SyncEvent`] -/// has no error and matches `peer` and `namespace`. -fn match_sync_finished(event: &LiveEvent, peer: PublicKey) -> bool { - let LiveEvent::SyncFinished(e) = event else { - return false; - }; - e.peer == peer && e.result.is_ok() -} diff --git a/net-tools/netwatch/Cargo.toml b/net-tools/netwatch/Cargo.toml new file mode 100644 index 00000000000..38637d45b6d --- /dev/null +++ b/net-tools/netwatch/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "netwatch" +version = "0.1.0" +readme = "README.md" +description = "Cross-platform monitoring for network interface changes" +license = "MIT OR Apache-2.0" +authors = ["n0 team"] +repository = "https://github.com/n0-computer/iroh" +keywords = ["networking", "interfaces"] +edition = "2021" + +[lints] +workspace = true + +[dependencies] +anyhow = { version = "1" } +bytes = "1.7" +futures-lite = "2.3" +futures-sink = "0.3.25" +futures-util = "0.3.25" +libc = "0.2.139" +netdev = "0.30.0" +once_cell = "1.18.0" +socket2 = "0.5.3" +thiserror = "1" +time = "0.3.20" +tokio = { version = "1", features = ["io-util", "macros", "sync", "rt", "net", "fs", "io-std", "signal", "process", "time"] } +tokio-util = { version = "0.7", features = ["rt"] } +tracing = "0.1" + +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] +netlink-packet-core = "0.7.0" +netlink-packet-route = "0.17.0" +netlink-sys = "0.8.5" +rtnetlink = "0.13.0" + +[target.'cfg(target_os = "windows")'.dependencies] +wmi = "0.13" +windows = { version = "0.51", features = ["Win32_NetworkManagement_IpHelper", "Win32_Foundation", "Win32_NetworkManagement_Ndis", "Win32_Networking_WinSock"] } +serde = { version = "1", features = ["derive"] } +derive_more = { version = "1.0.0", features = ["debug"] } + +[dev-dependencies] +tokio = { version = "1", features = ["io-util", "sync", "rt", "net", "fs", "macros", "time", "test-util"] } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "iroh_docsrs"] diff --git a/net-tools/netwatch/README.md b/net-tools/netwatch/README.md new file mode 100644 index 00000000000..e0c8f39b052 --- /dev/null +++ b/net-tools/netwatch/README.md @@ -0,0 +1,24 @@ +# Netwatch + +`netwatch` is a cross-platform library for monitoring of networking interfaces +and route changes. + +Used in [iroh](https://github.com/n0-computer/iroh), created with love by the +[n0 team](https://n0.computer/). + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-net/src/net/interfaces.rs b/net-tools/netwatch/src/interfaces.rs similarity index 90% rename from iroh-net/src/net/interfaces.rs rename to net-tools/netwatch/src/interfaces.rs index 759b0925f7b..f9f511d8794 100644 --- a/iroh-net/src/net/interfaces.rs +++ b/net-tools/netwatch/src/interfaces.rs @@ -29,11 +29,11 @@ use self::bsd::default_route; use self::linux::default_route; #[cfg(target_os = "windows")] use self::windows::default_route; -use crate::net::ip::{is_private_v6, is_up}; +use crate::ip::{is_private_v6, is_up}; /// Represents a network interface. #[derive(Debug)] -pub(crate) struct Interface { +pub struct Interface { iface: netdev::interface::Interface, } @@ -71,7 +71,7 @@ impl Interface { } /// A list of all ip addresses of this interface. - pub(crate) fn addrs(&self) -> impl Iterator + '_ { + pub fn addrs(&self) -> impl Iterator + '_ { self.iface .ipv4 .iter() @@ -82,14 +82,7 @@ impl Interface { /// Creates a fake interface for usage in tests. /// - /// Sometimes tests want to be deterministic, e.g. [`ProbePlan`] tests rely on the - /// interface state. This allows tests to be independent of the host interfaces. - /// - /// It is rather possible that we'll want more variations of this in the future, feel - /// free to add parameters or different alternative constructors. - /// - /// [`ProbePlan`]: crate::netcheck::reportgen::probes::ProbePlan - #[cfg(test)] + /// This allows tests to be independent of the host interfaces. pub(crate) fn fake() -> Self { use std::net::Ipv4Addr; @@ -126,7 +119,7 @@ impl Interface { /// Structure of an IP network, either IPv4 or IPv6. #[derive(Clone, Debug)] -pub(crate) enum IpNet { +pub enum IpNet { /// Structure of IPv4 Network. V4(Ipv4Net), /// Structure of IPv6 Network. @@ -161,16 +154,16 @@ impl IpNet { /// Intended to store the state of the machine's network interfaces, routing table, and /// other network configuration. For now it's pretty basic. #[derive(Debug, PartialEq, Eq)] -pub(crate) struct State { +pub struct State { /// Maps from an interface name interface. - pub(crate) interfaces: HashMap, + pub interfaces: HashMap, /// Whether this machine has an IPv6 Global or Unique Local Address /// which might provide connectivity. - pub(crate) have_v6: bool, + pub have_v6: bool, /// Whether the machine has some non-localhost, non-link-local IPv4 address. - pub(crate) have_v4: bool, + pub have_v4: bool, //// Whether the current network interface is considered "expensive", which currently means LTE/etc /// instead of Wifi. This field is not populated by `get_state`. @@ -255,15 +248,8 @@ impl State { /// Creates a fake interface state for usage in tests. /// - /// Sometimes tests want to be deterministic, e.g. [`ProbePlan`] tests rely on the - /// interface state. This allows tests to be independent of the host interfaces. - /// - /// It is rather possible that we'll want more variations of this in the future, feel - /// free to add parameters or different alternative constructors. - /// - /// [`ProbePlan`]: crate::netcheck::reportgen::probes::ProbePlan - #[cfg(test)] - pub(crate) fn fake() -> Self { + /// This allows tests to be independent of the host interfaces. + pub fn fake() -> Self { let fake = Interface::fake(); let ifname = fake.iface.name.clone(); Self { @@ -341,7 +327,7 @@ pub async fn default_route_interface() -> Option { /// Likely IPs of the residentla router, and the ip address of the current /// machine using it. #[derive(Debug, Clone)] -pub(crate) struct HomeRouter { +pub struct HomeRouter { /// Ip of the router. pub gateway: IpAddr, /// Our local Ip if known. @@ -354,7 +340,7 @@ impl HomeRouter { /// In addition, it returns the IP address of the current machine on /// the LAN using that gateway. /// This is used as the destination for UPnP, NAT-PMP, PCP, etc queries. - pub(crate) fn new() -> Option { + pub fn new() -> Option { let gateway = Self::get_default_gateway()?; let my_ip = netdev::interface::get_local_ipaddr(); diff --git a/iroh-net/src/net/interfaces/bsd.rs b/net-tools/netwatch/src/interfaces/bsd.rs similarity index 100% rename from iroh-net/src/net/interfaces/bsd.rs rename to net-tools/netwatch/src/interfaces/bsd.rs diff --git a/iroh-net/src/net/interfaces/bsd/freebsd.rs b/net-tools/netwatch/src/interfaces/bsd/freebsd.rs similarity index 100% rename from iroh-net/src/net/interfaces/bsd/freebsd.rs rename to net-tools/netwatch/src/interfaces/bsd/freebsd.rs diff --git a/iroh-net/src/net/interfaces/bsd/macos.rs b/net-tools/netwatch/src/interfaces/bsd/macos.rs similarity index 100% rename from iroh-net/src/net/interfaces/bsd/macos.rs rename to net-tools/netwatch/src/interfaces/bsd/macos.rs diff --git a/iroh-net/src/net/interfaces/bsd/netbsd.rs b/net-tools/netwatch/src/interfaces/bsd/netbsd.rs similarity index 100% rename from iroh-net/src/net/interfaces/bsd/netbsd.rs rename to net-tools/netwatch/src/interfaces/bsd/netbsd.rs diff --git a/iroh-net/src/net/interfaces/bsd/openbsd.rs b/net-tools/netwatch/src/interfaces/bsd/openbsd.rs similarity index 100% rename from iroh-net/src/net/interfaces/bsd/openbsd.rs rename to net-tools/netwatch/src/interfaces/bsd/openbsd.rs diff --git a/iroh-net/src/net/interfaces/linux.rs b/net-tools/netwatch/src/interfaces/linux.rs similarity index 100% rename from iroh-net/src/net/interfaces/linux.rs rename to net-tools/netwatch/src/interfaces/linux.rs diff --git a/iroh-net/src/net/interfaces/windows.rs b/net-tools/netwatch/src/interfaces/windows.rs similarity index 100% rename from iroh-net/src/net/interfaces/windows.rs rename to net-tools/netwatch/src/interfaces/windows.rs diff --git a/iroh-net/src/net/ip.rs b/net-tools/netwatch/src/ip.rs similarity index 100% rename from iroh-net/src/net/ip.rs rename to net-tools/netwatch/src/ip.rs diff --git a/iroh-net/src/net/ip_family.rs b/net-tools/netwatch/src/ip_family.rs similarity index 100% rename from iroh-net/src/net/ip_family.rs rename to net-tools/netwatch/src/ip_family.rs diff --git a/iroh-net/src/net.rs b/net-tools/netwatch/src/lib.rs similarity index 83% rename from iroh-net/src/net.rs rename to net-tools/netwatch/src/lib.rs index a010dc235c0..213fe78e009 100644 --- a/iroh-net/src/net.rs +++ b/net-tools/netwatch/src/lib.rs @@ -1,6 +1,6 @@ //! Networking related utilities -pub(crate) mod interfaces; +pub mod interfaces; pub mod ip; mod ip_family; pub mod netmon; diff --git a/iroh-net/src/net/netmon.rs b/net-tools/netwatch/src/netmon.rs similarity index 89% rename from iroh-net/src/net/netmon.rs rename to net-tools/netwatch/src/netmon.rs index f5a41b72bdb..4901bece372 100644 --- a/iroh-net/src/net/netmon.rs +++ b/net-tools/netwatch/src/netmon.rs @@ -2,10 +2,8 @@ use anyhow::Result; use futures_lite::future::Boxed as BoxFuture; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinHandle, -}; +use tokio::sync::{mpsc, oneshot}; +use tokio_util::task::AbortOnDropHandle; mod actor; #[cfg(target_os = "android")] @@ -30,16 +28,10 @@ use self::actor::{Actor, ActorMessage}; #[derive(Debug)] pub struct Monitor { /// Task handle for the monitor task. - handle: JoinHandle<()>, + _handle: AbortOnDropHandle<()>, actor_tx: mpsc::Sender, } -impl Drop for Monitor { - fn drop(&mut self) { - self.handle.abort(); - } -} - impl Monitor { /// Create a new monitor. pub async fn new() -> Result { @@ -50,7 +42,10 @@ impl Monitor { actor.run().await; }); - Ok(Monitor { handle, actor_tx }) + Ok(Monitor { + _handle: AbortOnDropHandle::new(handle), + actor_tx, + }) } /// Subscribe to network changes. @@ -91,8 +86,6 @@ mod tests { #[tokio::test] async fn test_smoke_monitor() { - let _guard = iroh_test::logging::setup(); - let mon = Monitor::new().await.unwrap(); let _token = mon .subscribe(|is_major| { diff --git a/iroh-net/src/net/netmon/actor.rs b/net-tools/netwatch/src/netmon/actor.rs similarity index 81% rename from iroh-net/src/net/netmon/actor.rs rename to net-tools/netwatch/src/netmon/actor.rs index f12f44c8730..8ce39105117 100644 --- a/iroh-net/src/net/netmon/actor.rs +++ b/net-tools/netwatch/src/netmon/actor.rs @@ -24,7 +24,7 @@ use super::bsd as os; use super::linux as os; #[cfg(target_os = "windows")] use super::windows as os; -use crate::net::{ +use crate::{ interfaces::{IpNet, State}, ip::is_link_local, }; @@ -81,7 +81,6 @@ impl Actor { let interface_state = State::new().await; let wall_time = Instant::now(); - // Use flume channels, as tokio::mpsc is not safe to use across ffi boundaries. let (mon_sender, mon_receiver) = mpsc::channel(MON_CHAN_CAPACITY); let route_monitor = RouteMonitor::new(mon_sender)?; let (actor_sender, actor_receiver) = mpsc::channel(ACTOR_CHAN_CAPACITY); @@ -112,6 +111,7 @@ impl Actor { loop { tokio::select! { biased; + _ = debounce_interval.tick() => { if let Some(time_jumped) = last_event.take() { if let Err(err) = self.handle_potential_change(time_jumped).await { @@ -127,29 +127,40 @@ impl Actor { debounce_interval.reset_immediately(); } } - Some(_event) = self.mon_receiver.recv() => { - trace!("network activity detected"); - last_event.replace(false); - debounce_interval.reset_immediately(); - } - Some(msg) = self.actor_receiver.recv() => match msg { - ActorMessage::Subscribe(callback, s) => { - let token = self.next_callback_token(); - self.callbacks.insert(token, Arc::new(callback)); - s.send(token).ok(); - } - ActorMessage::Unsubscribe(token, s) => { - self.callbacks.remove(&token); - s.send(()).ok(); + event = self.mon_receiver.recv() => { + match event { + Some(NetworkMessage::Change) => { + trace!("network activity detected"); + last_event.replace(false); + debounce_interval.reset_immediately(); + } + None => { + debug!("shutting down, network monitor receiver gone"); + break; + } } - ActorMessage::NetworkChange => { - trace!("external network activity detected"); - last_event.replace(false); - debounce_interval.reset_immediately(); + } + msg = self.actor_receiver.recv() => { + match msg { + Some(ActorMessage::Subscribe(callback, s)) => { + let token = self.next_callback_token(); + self.callbacks.insert(token, Arc::new(callback)); + s.send(token).ok(); + } + Some(ActorMessage::Unsubscribe(token, s)) => { + self.callbacks.remove(&token); + s.send(()).ok(); + } + Some(ActorMessage::NetworkChange) => { + trace!("external network activity detected"); + last_event.replace(false); + debounce_interval.reset_immediately(); + } + None => { + debug!("shutting down, actor receiver gone"); + break; + } } - }, - else => { - break; } } } diff --git a/iroh-net/src/net/netmon/android.rs b/net-tools/netwatch/src/netmon/android.rs similarity index 100% rename from iroh-net/src/net/netmon/android.rs rename to net-tools/netwatch/src/netmon/android.rs diff --git a/iroh-net/src/net/netmon/bsd.rs b/net-tools/netwatch/src/netmon/bsd.rs similarity index 68% rename from iroh-net/src/net/netmon/bsd.rs rename to net-tools/netwatch/src/netmon/bsd.rs index 20bab5aae71..61ebb346660 100644 --- a/iroh-net/src/net/netmon/bsd.rs +++ b/net-tools/netwatch/src/netmon/bsd.rs @@ -1,32 +1,34 @@ use anyhow::Result; #[cfg(any(target_os = "macos", target_os = "ios"))] use libc::{RTAX_DST, RTAX_IFP}; -use tokio::{io::AsyncReadExt, sync::mpsc, task::JoinHandle}; +use tokio::{io::AsyncReadExt, sync::mpsc}; +use tokio_util::task::AbortOnDropHandle; use tracing::{trace, warn}; use super::actor::NetworkMessage; #[cfg(any(target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))] -use crate::net::interfaces::bsd::{RTAX_DST, RTAX_IFP}; -use crate::net::{interfaces::bsd::WireMessage, ip::is_link_local}; +use crate::interfaces::bsd::{RTAX_DST, RTAX_IFP}; +use crate::{interfaces::bsd::WireMessage, ip::is_link_local}; #[derive(Debug)] pub(super) struct RouteMonitor { - handle: JoinHandle<()>, + _handle: AbortOnDropHandle<()>, } -impl Drop for RouteMonitor { - fn drop(&mut self) { - self.handle.abort(); - } +fn create_socket() -> Result { + let socket = socket2::Socket::new(libc::AF_ROUTE.into(), socket2::Type::RAW, None)?; + socket.set_nonblocking(true)?; + let socket_std: std::os::unix::net::UnixStream = socket.into(); + let socket: tokio::net::UnixStream = socket_std.try_into()?; + + trace!("AF_ROUTE socket bound"); + + Ok(socket) } impl RouteMonitor { pub(super) fn new(sender: mpsc::Sender) -> Result { - let socket = socket2::Socket::new(libc::AF_ROUTE.into(), socket2::Type::RAW, None)?; - socket.set_nonblocking(true)?; - let socket_std: std::os::unix::net::UnixStream = socket.into(); - let mut socket: tokio::net::UnixStream = socket_std.try_into()?; - + let mut socket = create_socket()?; let handle = tokio::task::spawn(async move { trace!("AF_ROUTE monitor started"); @@ -52,12 +54,25 @@ impl RouteMonitor { } Err(err) => { warn!("AF_ROUTE: error reading: {:?}", err); + // recreate socket, as it is likely in an invalid state + // TODO: distinguish between different errors? + match create_socket() { + Ok(new_socket) => { + socket = new_socket; + } + Err(err) => { + warn!("AF_ROUTE: unable to bind a new socket: {:?}", err); + // TODO: what to do here? + } + } } } } }); - Ok(RouteMonitor { handle }) + Ok(RouteMonitor { + _handle: AbortOnDropHandle::new(handle), + }) } } diff --git a/iroh-net/src/net/netmon/linux.rs b/net-tools/netwatch/src/netmon/linux.rs similarity index 99% rename from iroh-net/src/net/netmon/linux.rs rename to net-tools/netwatch/src/netmon/linux.rs index 2380ac69d4e..95fd8e35eb1 100644 --- a/iroh-net/src/net/netmon/linux.rs +++ b/net-tools/netwatch/src/netmon/linux.rs @@ -13,7 +13,7 @@ use tokio::{sync::mpsc, task::JoinHandle}; use tracing::{info, trace, warn}; use super::actor::NetworkMessage; -use crate::net::ip::is_link_local; +use crate::ip::is_link_local; #[derive(Debug)] pub(super) struct RouteMonitor { diff --git a/iroh-net/src/net/netmon/windows.rs b/net-tools/netwatch/src/netmon/windows.rs similarity index 100% rename from iroh-net/src/net/netmon/windows.rs rename to net-tools/netwatch/src/netmon/windows.rs diff --git a/iroh-net/src/net/udp.rs b/net-tools/netwatch/src/udp.rs similarity index 100% rename from iroh-net/src/net/udp.rs rename to net-tools/netwatch/src/udp.rs diff --git a/net-tools/portmapper/Cargo.toml b/net-tools/portmapper/Cargo.toml new file mode 100644 index 00000000000..f75d09fcae0 --- /dev/null +++ b/net-tools/portmapper/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "portmapper" +version = "0.1.0" +edition = "2021" +readme = "README.md" +description = "Portmapping utilities" +license = "MIT OR Apache-2.0" +authors = ["n0 team"] +repository = "https://github.com/n0-computer/iroh" +keywords = ["portmapping", "pmp", "pcp", "upnp"] + +[lints] +workspace = true + +[dependencies] +anyhow = { version = "1" } +base64 = "0.22.1" +bytes = "1.7" +derive_more = { version = "1.0.0", features = ["debug", "display", "from", "try_into", "deref"] } +futures-lite = "2.3" +futures-util = "0.3.25" +igd-next = { version = "0.15.1", features = ["aio_tokio"] } +iroh-metrics = { version = "0.28.0", default-features = false } +libc = "0.2.139" +netwatch = { version = "0.1.0", path = "../netwatch" } +num_enum = "0.7" +rand = "0.8" +serde = { version = "1", features = ["derive", "rc"] } +smallvec = "1.11.1" +socket2 = "0.5.3" +thiserror = "1" +time = "0.3.20" +tokio = { version = "1", features = ["io-util", "macros", "sync", "rt", "net", "fs", "io-std", "signal", "process"] } +tokio-util = { version = "0.7.12", features = ["io-util", "io", "codec", "rt"] } +tracing = "0.1" +url = { version = "2.4", features = ["serde"] } + +[dev-dependencies] +ntest = "0.9" +rand_chacha = "0.3.1" +tokio = { version = "1", features = ["io-util", "sync", "rt", "net", "fs", "macros", "time", "test-util"] } + +[features] +default = ["metrics"] +metrics = ["iroh-metrics/metrics"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "iroh_docsrs"] diff --git a/net-tools/portmapper/README.md b/net-tools/portmapper/README.md new file mode 100644 index 00000000000..7f819769cc5 --- /dev/null +++ b/net-tools/portmapper/README.md @@ -0,0 +1,24 @@ +# Portmapper + +`portmapper` is a library to ensure a mapping for a local port is maintained +despite network changes. Provides upnp, pcp and nat-pmp protocols support. + +Used in [iroh](https://github.com/n0-computer/iroh), created with love by the +[n0 team](https://n0.computer/). + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-net/src/portmapper/current_mapping.rs b/net-tools/portmapper/src/current_mapping.rs similarity index 100% rename from iroh-net/src/portmapper/current_mapping.rs rename to net-tools/portmapper/src/current_mapping.rs diff --git a/iroh-net/src/portmapper.rs b/net-tools/portmapper/src/lib.rs similarity index 98% rename from iroh-net/src/portmapper.rs rename to net-tools/portmapper/src/lib.rs index 0b54840213f..708d5720987 100644 --- a/iroh-net/src/portmapper.rs +++ b/net-tools/portmapper/src/lib.rs @@ -10,18 +10,30 @@ use anyhow::{anyhow, Result}; use current_mapping::CurrentMapping; use futures_lite::StreamExt; use iroh_metrics::inc; +use netwatch::interfaces::HomeRouter; use tokio::sync::{mpsc, oneshot, watch}; use tokio_util::task::AbortOnDropHandle; use tracing::{debug, info_span, trace, Instrument}; -use crate::{net::interfaces::HomeRouter, util}; - mod current_mapping; mod mapping; mod metrics; mod nat_pmp; mod pcp; mod upnp; +mod util; +mod defaults { + use std::time::Duration; + + /// Maximum duration a UPnP search can take before timing out. + pub(crate) const UPNP_SEARCH_TIMEOUT: Duration = Duration::from_secs(1); + + /// Timeout to receive a response from a PCP server. + pub(crate) const PCP_RECV_TIMEOUT: Duration = Duration::from_millis(500); + + /// Timeout to receive a response from a NAT-PMP server. + pub(crate) const NAT_PMP_RECV_TIMEOUT: Duration = Duration::from_millis(500); +} pub use metrics::Metrics; diff --git a/iroh-net/src/portmapper/mapping.rs b/net-tools/portmapper/src/mapping.rs similarity index 100% rename from iroh-net/src/portmapper/mapping.rs rename to net-tools/portmapper/src/mapping.rs diff --git a/iroh-net/src/portmapper/metrics.rs b/net-tools/portmapper/src/metrics.rs similarity index 100% rename from iroh-net/src/portmapper/metrics.rs rename to net-tools/portmapper/src/metrics.rs diff --git a/iroh-net/src/portmapper/nat_pmp.rs b/net-tools/portmapper/src/nat_pmp.rs similarity index 98% rename from iroh-net/src/portmapper/nat_pmp.rs rename to net-tools/portmapper/src/nat_pmp.rs index 911b4ace7b9..a44c4aeb7ee 100644 --- a/iroh-net/src/portmapper/nat_pmp.rs +++ b/net-tools/portmapper/src/nat_pmp.rs @@ -2,10 +2,11 @@ use std::{net::Ipv4Addr, num::NonZeroU16, time::Duration}; +use netwatch::UdpSocket; use tracing::{debug, trace}; use self::protocol::{MapProtocol, Request, Response}; -use crate::{defaults::timeouts::NAT_PMP_RECV_TIMEOUT as RECV_TIMEOUT, net::UdpSocket}; +use crate::defaults::NAT_PMP_RECV_TIMEOUT as RECV_TIMEOUT; mod protocol; diff --git a/iroh-net/src/portmapper/nat_pmp/protocol.rs b/net-tools/portmapper/src/nat_pmp/protocol.rs similarity index 100% rename from iroh-net/src/portmapper/nat_pmp/protocol.rs rename to net-tools/portmapper/src/nat_pmp/protocol.rs diff --git a/iroh-net/src/portmapper/nat_pmp/protocol/request.rs b/net-tools/portmapper/src/nat_pmp/protocol/request.rs similarity index 100% rename from iroh-net/src/portmapper/nat_pmp/protocol/request.rs rename to net-tools/portmapper/src/nat_pmp/protocol/request.rs diff --git a/iroh-net/src/portmapper/nat_pmp/protocol/response.rs b/net-tools/portmapper/src/nat_pmp/protocol/response.rs similarity index 100% rename from iroh-net/src/portmapper/nat_pmp/protocol/response.rs rename to net-tools/portmapper/src/nat_pmp/protocol/response.rs diff --git a/iroh-net/src/portmapper/pcp.rs b/net-tools/portmapper/src/pcp.rs similarity index 98% rename from iroh-net/src/portmapper/pcp.rs rename to net-tools/portmapper/src/pcp.rs index f911a341e5f..0f2fe789f50 100644 --- a/iroh-net/src/portmapper/pcp.rs +++ b/net-tools/portmapper/src/pcp.rs @@ -2,10 +2,11 @@ use std::{net::Ipv4Addr, num::NonZeroU16, time::Duration}; +use netwatch::UdpSocket; use rand::RngCore; use tracing::{debug, trace}; -use crate::{defaults::timeouts::PCP_RECV_TIMEOUT as RECV_TIMEOUT, net::UdpSocket}; +use crate::defaults::PCP_RECV_TIMEOUT as RECV_TIMEOUT; mod protocol; diff --git a/iroh-net/src/portmapper/pcp/protocol.rs b/net-tools/portmapper/src/pcp/protocol.rs similarity index 100% rename from iroh-net/src/portmapper/pcp/protocol.rs rename to net-tools/portmapper/src/pcp/protocol.rs diff --git a/iroh-net/src/portmapper/pcp/protocol/opcode_data.rs b/net-tools/portmapper/src/pcp/protocol/opcode_data.rs similarity index 100% rename from iroh-net/src/portmapper/pcp/protocol/opcode_data.rs rename to net-tools/portmapper/src/pcp/protocol/opcode_data.rs diff --git a/iroh-net/src/portmapper/pcp/protocol/request.rs b/net-tools/portmapper/src/pcp/protocol/request.rs similarity index 99% rename from iroh-net/src/portmapper/pcp/protocol/request.rs rename to net-tools/portmapper/src/pcp/protocol/request.rs index f893f726d9e..3cb0d6520ff 100644 --- a/iroh-net/src/portmapper/pcp/protocol/request.rs +++ b/net-tools/portmapper/src/pcp/protocol/request.rs @@ -10,7 +10,6 @@ use super::{ /// A PCP Request. /// /// See [RFC 6887 Request Header](https://datatracker.ietf.org/doc/html/rfc6887#section-7.1) -/// // NOTE: PCP Options are optional, and currently not used in this code, thus not implemented #[derive(Debug, PartialEq, Eq)] pub struct Request { diff --git a/iroh-net/src/portmapper/pcp/protocol/response.rs b/net-tools/portmapper/src/pcp/protocol/response.rs similarity index 99% rename from iroh-net/src/portmapper/pcp/protocol/response.rs rename to net-tools/portmapper/src/pcp/protocol/response.rs index f834ebef9fd..6469b227c5c 100644 --- a/iroh-net/src/portmapper/pcp/protocol/response.rs +++ b/net-tools/portmapper/src/pcp/protocol/response.rs @@ -105,7 +105,6 @@ impl From for u8 { /// A PCP successful Response/Notification. /// /// See [RFC 6887 Response Header](https://datatracker.ietf.org/doc/html/rfc6887#section-7.2) -/// // NOTE: first two fields are *currently* not used, but are useful for debug purposes #[allow(unused)] #[derive(Debug, PartialEq, Eq)] diff --git a/iroh-net/src/portmapper/upnp.rs b/net-tools/portmapper/src/upnp.rs similarity index 80% rename from iroh-net/src/portmapper/upnp.rs rename to net-tools/portmapper/src/upnp.rs index 6d202eed350..4960ceaee6f 100644 --- a/iroh-net/src/portmapper/upnp.rs +++ b/net-tools/portmapper/src/upnp.rs @@ -13,7 +13,7 @@ use super::Metrics; pub type Gateway = aigd::Gateway; -use crate::defaults::timeouts::UPNP_SEARCH_TIMEOUT as SEARCH_TIMEOUT; +use crate::defaults::UPNP_SEARCH_TIMEOUT as SEARCH_TIMEOUT; /// Seconds we ask the router to maintain the port mapping. 0 means infinite. const PORT_MAPPING_LEASE_DURATION_SECONDS: u32 = 0; @@ -49,11 +49,15 @@ impl Mapping { let gateway = if let Some(known_gateway) = gateway { known_gateway } else { - aigd::tokio::search_gateway(igd_next::SearchOptions { - timeout: Some(SEARCH_TIMEOUT), - ..Default::default() - }) - .await? + // Wrap in manual timeout, because igd_next doesn't respect the set timeout + tokio::time::timeout( + SEARCH_TIMEOUT, + aigd::tokio::search_gateway(igd_next::SearchOptions { + timeout: Some(SEARCH_TIMEOUT), + ..Default::default() + }), + ) + .await?? }; let std::net::IpAddr::V4(external_ip) = gateway.get_external_ip().await? else { @@ -126,14 +130,25 @@ impl Mapping { /// Searches for UPnP gateways. pub async fn probe_available() -> Option { inc!(Metrics, upnp_probes); - match aigd::tokio::search_gateway(igd_next::SearchOptions { - timeout: Some(SEARCH_TIMEOUT), - ..Default::default() - }) - .await - { - Ok(gateway) => Some(gateway), + + // Wrap in manual timeout, because igd_next doesn't respect the set timeout + let res = tokio::time::timeout( + SEARCH_TIMEOUT, + aigd::tokio::search_gateway(igd_next::SearchOptions { + timeout: Some(SEARCH_TIMEOUT), + ..Default::default() + }), + ) + .await; + + match res { + Ok(Ok(gateway)) => Some(gateway), Err(e) => { + inc!(Metrics, upnp_probes_failed); + debug!("upnp probe timed out: {e}"); + None + } + Ok(Err(e)) => { inc!(Metrics, upnp_probes_failed); debug!("upnp probe failed: {e}"); None diff --git a/net-tools/portmapper/src/util.rs b/net-tools/portmapper/src/util.rs new file mode 100644 index 00000000000..5dccd45c003 --- /dev/null +++ b/net-tools/portmapper/src/util.rs @@ -0,0 +1,30 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +/// Resolves to pending if the inner is `None`. +#[derive(Debug)] +pub(crate) struct MaybeFuture { + /// Future to be polled. + pub inner: Option, +} + +// NOTE: explicit implementation to bypass derive unnecessary bounds +impl Default for MaybeFuture { + fn default() -> Self { + MaybeFuture { inner: None } + } +} + +impl Future for MaybeFuture { + type Output = T::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.inner { + Some(ref mut t) => Pin::new(t).poll(cx), + None => Poll::Pending, + } + } +}