Merge remote-tracking branch 'remotes/origin/main' into dnut/repair3

src/accountsdb/snapshots.zig

@@ -1,25 +1,28 @@
 const std = @import("std");
+const zstd = @import("zstd");
+const sig = @import("../lib.zig");
+
+const bincode = sig.bincode;
+
 const ArrayList = std.ArrayList;
 const HashMap = std.AutoHashMap;
-
-const _genesis_config = @import("genesis_config.zig");
-const UnixTimestamp = _genesis_config.UnixTimestamp;
-const FeeRateGovernor = _genesis_config.FeeRateGovernor;
-const EpochSchedule = _genesis_config.EpochSchedule;
-const Rent = _genesis_config.Rent;
-const Inflation = _genesis_config.Inflation;
-
-const Account = @import("../core/account.zig").Account;
-const Hash = @import("../core/hash.zig").Hash;
-const Slot = @import("../core/time.zig").Slot;
-const Epoch = @import("../core/time.zig").Epoch;
-const Pubkey = @import("../core/pubkey.zig").Pubkey;
-const bincode = @import("../bincode/bincode.zig");
-const defaultArrayListOnEOFConfig = @import("../utils/arraylist.zig").defaultArrayListOnEOFConfig;
-const readDirectory = @import("../utils/directory.zig").readDirectory;
-pub const sysvars = @import("sysvars.zig");
-const ZstdReader = @import("zstd").Reader;
-const parallelUntarToFileSystem = @import("../utils/tar.zig").parallelUntarToFileSystem;
+const ZstdReader = zstd.Reader;
+
+const Account = sig.core.account.Account;
+const Hash = sig.core.hash.Hash;
+const Slot = sig.core.time.Slot;
+const Epoch = sig.core.time.Epoch;
+const Pubkey = sig.core.pubkey.Pubkey;
+const UnixTimestamp = sig.accounts_db.genesis_config.UnixTimestamp;
+const FeeRateGovernor = sig.accounts_db.genesis_config.FeeRateGovernor;
+const EpochSchedule = sig.accounts_db.genesis_config.EpochSchedule;
+const Rent = sig.accounts_db.genesis_config.Rent;
+const Inflation = sig.accounts_db.genesis_config.Inflation;
+const SlotHistory = sig.accounts_db.sysvars.SlotHistory;
+
+const defaultArrayListOnEOFConfig = bincode.arraylist.defaultArrayListOnEOFConfig;
+const readDirectory = sig.utils.directory.readDirectory;
+const parallelUntarToFileSystem = sig.utils.tar.parallelUntarToFileSystem;
 
 pub const MAXIMUM_ACCOUNT_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024; // 16 GiB
 pub const MAX_RECENT_BLOCKHASHES: usize = 300;
@@ -739,7 +742,7 @@ pub const StatusCache = struct {
         self: *const StatusCache,
         allocator: std.mem.Allocator,
         bank_slot: Slot,
-        slot_history: *const sysvars.SlotHistory,
+        slot_history: *const SlotHistory,
     ) !void {
         // status cache validation
         const len = self.bank_slot_deltas.items.len;

src/utils/arraylist.zig → src/bincode/arraylist.zig

@@ -1,5 +1,7 @@
 const std = @import("std");
-const bincode = @import("../bincode/bincode.zig");
+const sig = @import("../lib.zig");
+
+const bincode = sig.bincode;
 
 pub fn ArrayListConfig(comptime Child: type) bincode.FieldConfig(std.ArrayList(Child)) {
     const S = struct {

src/bincode/bincode.zig

@@ -1,16 +1,13 @@
+pub const arraylist = @import("arraylist.zig");
+pub const shortvec = @import("shortvec.zig");
+pub const varint = @import("varint.zig");
+
 const std = @import("std");
 
 const testing = std.testing;
 
 const bincode = @This();
 
-pub const config = struct {
-    // TODO move these files to the bincode folder
-    pub const arraylist = @import("../utils/arraylist.zig");
-    pub const shortvec = @import("../utils/shortvec.zig");
-    pub const varint = @import("../utils/varint.zig");
-};
-
 pub const Params = struct {
     pub const legacy: Params = .{
         .endian = .little,
@@ -706,7 +703,7 @@ test "bincode: custom enum" {
 }
 
 test "bincode: default on eof" {
-    const defaultArrayListOnEOFConfig = @import("../utils/arraylist.zig").defaultArrayListOnEOFConfig;
+    const defaultArrayListOnEOFConfig = @import("../lib.zig").bincode.arraylist.defaultArrayListOnEOFConfig;
     const Foo = struct {
         value: u8 = 0,
         accounts: std.ArrayList(u64),

src/utils/shortvec.zig → src/bincode/shortvec.zig

@@ -1,7 +1,10 @@
 const std = @import("std");
-const bincode = @import("../bincode/bincode.zig");
-const serialize_short_u16 = @import("varint.zig").serialize_short_u16;
-const deserialize_short_u16 = @import("varint.zig").deserialize_short_u16;
+const sig = @import("../lib.zig");
+
+const bincode = sig.bincode;
+
+const serialize_short_u16 = sig.bincode.varint.serialize_short_u16;
+const deserialize_short_u16 = sig.bincode.varint.deserialize_short_u16;
 
 pub fn ShortVecConfig(comptime Child: type) bincode.FieldConfig([]Child) {
     const S = struct {

src/utils/varint.zig → src/bincode/varint.zig

@@ -1,5 +1,7 @@
 const std = @import("std");
-const bincode = @import("../bincode/bincode.zig");
+const sig = @import("../lib.zig");
+
+const bincode = sig.bincode;
 
 pub fn VarIntConfig(comptime VarInt: type) bincode.FieldConfig(VarInt) {
     const S = struct {

src/bloom/bloom.zig

@@ -1,16 +1,17 @@
 const std = @import("std");
-const ArrayList = std.ArrayList;
-const DynamicArrayBitSet = @import("bit_set.zig").DynamicArrayBitSet;
-const BitVec = @import("bit_vec.zig").BitVec;
-const ArrayListConfig = @import("../utils/arraylist.zig").ArrayListConfig;
-
-const bincode = @import("../bincode/bincode.zig");
-const BitVecConfig = @import("bit_vec.zig").BitVecConfig;
+const sig = @import("../lib.zig");
 
-const FnvHasher = @import("../crypto/fnv.zig").FnvHasher;
 const testing = std.testing;
+const bincode = sig.bincode;
 
 const RndGen = std.rand.DefaultPrng;
+const ArrayList = std.ArrayList;
+
+const DynamicArrayBitSet = sig.bloom.bit_set.DynamicArrayBitSet;
+const BitVec = sig.bloom.bit_vec.BitVec;
+const BitVecConfig = sig.bloom.bit_vec.BitVecConfig;
+const ArrayListConfig = bincode.arraylist.ArrayListConfig;
+const FnvHasher = sig.crypto.FnvHasher;
 
 /// A bloom filter whose bitset is made up of u64 blocks
 pub const Bloom = struct {

src/cmd/cmd.zig

@@ -211,7 +211,7 @@ var force_new_snapshot_download_option = cli.Option{
 
 var snapshot_dir_option = cli.Option{
     .long_name = "snapshot-dir",
-    .help = "path to snapshot directory (where snapshots are downloaded and/or unpacked to/from) - default: test_data/",
+    .help = "path to snapshot directory (where snapshots are downloaded and/or unpacked to/from) - default: ledger/accounts_db",
     .short_alias = 's',
     .value_ref = cli.mkRef(&config.current.accounts_db.snapshot_dir),
     .required = false,
@@ -418,6 +418,9 @@ fn validator() !void {
 
     const repair_port: u16 = config.current.shred_collector.repair_port;
     const turbine_recv_port: u16 = config.current.shred_collector.repair_port;
+    const snapshot_dir_str = config.current.accounts_db.snapshot_dir;
+
+    try std.fs.cwd().makePath(snapshot_dir_str);
 
     var gossip_service, var gossip_manager = try startGossip(allocator, &app_base, &.{
         .{ .tag = .repair, .port = repair_port },
@@ -750,11 +753,11 @@ fn loadFromSnapshot(
 
     // cli parsing
     const snapshot_dir_str = config.current.accounts_db.snapshot_dir;
-    const n_cpus = @as(u32, @truncate(try std.Thread.getCpuCount()));
-    var n_threads_snapshot_load: u32 = @intCast(config.current.accounts_db.num_threads_snapshot_load);
-    if (n_threads_snapshot_load == 0) {
-        n_threads_snapshot_load = n_cpus;
-    }
+    const n_threads_snapshot_load: u32 = blk: {
+        const n_threads_snapshot_load: u32 = config.current.accounts_db.num_threads_snapshot_load;
+        if (n_threads_snapshot_load == 0) break :blk @truncate(try std.Thread.getCpuCount());
+        break :blk n_threads_snapshot_load;
+    };
 
     output.accounts_db = try AccountsDB.init(
         allocator,

src/cmd/config.zig

@@ -36,7 +36,7 @@ pub const AccountsDBConfig = struct {
     // where to load/save snapshots from - also where disk indexes and account files are stored
     snapshot_dir: []const u8 = "ledger/accounts_db",
     // number of threads to load snapshot
-    num_threads_snapshot_load: u16 = 0,
+    num_threads_snapshot_load: u32 = 0,
     // number of threads to unpack snapshot from .tar.zstd
     num_threads_snapshot_unpack: u16 = 0,
     // number of shards to use across the index

src/core/transaction.zig

@@ -1,8 +1,10 @@
 const std = @import("std");
-const ShortVecConfig = @import("../utils/shortvec.zig").ShortVecConfig;
-const Signature = @import("signature.zig").Signature;
-const Pubkey = @import("pubkey.zig").Pubkey;
-const Hash = @import("hash.zig").Hash;
+const sig = @import("../lib.zig");
+
+const Signature = sig.core.Signature;
+const Pubkey = sig.core.Pubkey;
+const Hash = sig.core.Hash;
+const ShortVecConfig = sig.bincode.shortvec.ShortVecConfig;
 
 pub const Transaction = struct {
     signatures: []Signature,

src/gossip/active_set.zig

@@ -1,17 +1,19 @@
 const std = @import("std");
-const KeyPair = std.crypto.sign.Ed25519.KeyPair;
 const network = @import("zig-network");
+const sig = @import("../lib.zig");
+
+const KeyPair = std.crypto.sign.Ed25519.KeyPair;
 const EndPoint = network.EndPoint;
-const _gossip_data = @import("../gossip/data.zig");
-const SignedGossipData = _gossip_data.SignedGossipData;
-const getWallclockMs = _gossip_data.getWallclockMs;
-const ContactInfo = _gossip_data.ContactInfo;
-const LegacyContactInfo = _gossip_data.LegacyContactInfo;
 
-const Pubkey = @import("../core/pubkey.zig").Pubkey;
-const GossipTable = @import("../gossip/table.zig").GossipTable;
-const shuffleFirstN = @import("../gossip/pull_request.zig").shuffleFirstN;
-const Bloom = @import("../bloom/bloom.zig").Bloom;
+const Pubkey = sig.core.Pubkey;
+const Bloom = sig.bloom.Bloom;
+const ContactInfo = sig.gossip.data.ContactInfo;
+const SignedGossipData = sig.gossip.data.SignedGossipData;
+const LegacyContactInfo = sig.gossip.data.LegacyContactInfo;
+const GossipTable = sig.gossip.table.GossipTable;
+
+const getWallclockMs = sig.gossip.getWallclockMs;
+const shuffleFirstN = sig.gossip.pull_request.shuffleFirstN;
 
 const NUM_ACTIVE_SET_ENTRIES: usize = 25;
 pub const GOSSIP_PUSH_FANOUT: usize = 6;
@@ -22,28 +24,29 @@ const BLOOM_MAX_BITS: usize = 1024 * 8 * 4;
 
 pub const ActiveSet = struct {
     // store pubkeys as keys in gossip table bc the data can change
-    pruned_peers: std.AutoHashMap(Pubkey, Bloom),
+    // For each peer, a bloom filter is used to store pruned origins
+    peers: std.AutoHashMap(Pubkey, Bloom),
     allocator: std.mem.Allocator,
 
     const Self = @This();
 
     pub fn init(allocator: std.mem.Allocator) Self {
         return Self{
-            .pruned_peers = std.AutoHashMap(Pubkey, Bloom).init(allocator),
+            .peers = std.AutoHashMap(Pubkey, Bloom).init(allocator),
             .allocator = allocator,
         };
     }
 
     pub fn deinit(self: *Self) void {
-        var iter = self.pruned_peers.iterator();
+        var iter = self.peers.iterator();
         while (iter.next()) |entry| {
             entry.value_ptr.deinit();
         }
-        self.pruned_peers.deinit();
+        self.peers.deinit();
     }
 
     pub fn len(self: *const Self) u32 {
-        return self.pruned_peers.count();
+        return self.peers.count();
     }
 
     pub fn rotate(
@@ -52,11 +55,11 @@ pub const ActiveSet = struct {
         peers: []ContactInfo,
     ) error{OutOfMemory}!void {
         // clear the existing
-        var iter = self.pruned_peers.iterator();
+        var iter = self.peers.iterator();
         while (iter.next()) |entry| {
             entry.value_ptr.deinit();
         }
-        self.pruned_peers.clearRetainingCapacity();
+        self.peers.clearRetainingCapacity();
 
         if (peers.len == 0) {
             return;
@@ -66,7 +69,7 @@ pub const ActiveSet = struct {
 
         const bloom_num_items = @max(peers.len, MIN_NUM_BLOOM_ITEMS);
         for (0..size) |i| {
-            const entry = try self.pruned_peers.getOrPut(peers[i].pubkey);
+            const entry = try self.peers.getOrPut(peers[i].pubkey);
             if (entry.found_existing == false) {
                 // *full* hard restart on blooms -- labs doesnt do this - bug?
                 const bloom = try Bloom.random(
@@ -83,7 +86,7 @@ pub const ActiveSet = struct {
 
     pub fn prune(self: *Self, from: Pubkey, origin: Pubkey) void {
         // we only prune peers which we are sending push messages to
-        if (self.pruned_peers.getEntry(from)) |entry| {
+        if (self.peers.getEntry(from)) |entry| {
             const origin_bytes = origin.data;
             entry.value_ptr.add(&origin_bytes);
         }
@@ -101,8 +104,7 @@ pub const ActiveSet = struct {
         var active_set_endpoints = try std.ArrayList(EndPoint).initCapacity(allocator, GOSSIP_PUSH_FANOUT);
         errdefer active_set_endpoints.deinit();
 
-        // change to while loop
-        var iter = self.pruned_peers.iterator();
+        var iter = self.peers.iterator();
         while (iter.next()) |entry| {
             // lookup peer contact info
             const peer_info = table.getContactInfo(entry.key_ptr.*) orelse continue;
@@ -167,7 +169,7 @@ test "gossip.active_set: init/deinit" {
     const no_prune_fanout_len = fanout.items.len;
     try std.testing.expect(no_prune_fanout_len > 0);
 
-    var iter = active_set.pruned_peers.keyIterator();
+    var iter = active_set.peers.keyIterator();
     const peer_pubkey = iter.next().?.*;
     active_set.prune(peer_pubkey, origin);
 

src/gossip/data.zig

@@ -1,34 +1,32 @@
 const std = @import("std");
-const SocketAddr = @import("../net/net.zig").SocketAddr;
-const Hash = @import("../core/hash.zig").Hash;
-const Signature = @import("../core/signature.zig").Signature;
-const Transaction = @import("../core/transaction.zig").Transaction;
-const Slot = @import("../core/time.zig").Slot;
-const bincode = @import("../bincode/bincode.zig");
-const ArrayList = std.ArrayList;
-const KeyPair = std.crypto.sign.Ed25519.KeyPair;
-const Pubkey = @import("../core/pubkey.zig").Pubkey;
-const sanitizeWallclock = @import("./message.zig").sanitizeWallclock;
-const PACKET_DATA_SIZE = @import("../net/packet.zig").PACKET_DATA_SIZE;
-
 const network = @import("zig-network");
-const var_int = @import("../utils/varint.zig");
-const var_int_config_u16 = var_int.var_int_config_u16;
-const var_int_config_u64 = var_int.var_int_config_u64;
+const sig = @import("../lib.zig");
 
-const ShortVecArrayListConfig = @import("../utils/shortvec.zig").ShortVecArrayListConfig;
-const IpAddr = @import("../net/net.zig").IpAddr;
-const gossip = @import("sig").gossip;
 const testing = std.testing;
+const bincode = sig.bincode;
 
-const ClientVersion = @import("../version/version.zig").ClientVersion;
-
+const ArrayList = std.ArrayList;
+const KeyPair = std.crypto.sign.Ed25519.KeyPair;
 const UdpSocket = network.Socket;
 const TcpListener = network.Socket;
-const net = std.net;
 
-const DynamicArrayBitSet = @import("../bloom/bit_set.zig").DynamicArrayBitSet;
-const BitVecConfig = @import("../bloom/bit_vec.zig").BitVecConfig;
+const SocketAddr = sig.net.SocketAddr;
+const Hash = sig.core.Hash;
+const Signature = sig.core.Signature;
+const Transaction = sig.core.Transaction;
+const Slot = sig.core.Slot;
+const Pubkey = sig.core.Pubkey;
+const IpAddr = sig.net.IpAddr;
+const ClientVersion = sig.version.ClientVersion;
+const DynamicArrayBitSet = sig.bloom.bit_set.DynamicArrayBitSet;
+const BitVecConfig = sig.bloom.bit_vec.BitVecConfig;
+const ShortVecArrayListConfig = sig.bincode.shortvec.ShortVecArrayListConfig;
+
+const sanitizeWallclock = sig.gossip.message.sanitizeWallclock;
+
+const PACKET_DATA_SIZE = sig.net.packet.PACKET_DATA_SIZE;
+const var_int_config_u16 = sig.bincode.varint.var_int_config_u16;
+const var_int_config_u64 = sig.bincode.varint.var_int_config_u64;
 
 /// returns current timestamp in milliseconds
 pub fn getWallclockMs() u64 {
@@ -103,8 +101,8 @@ pub const SignedGossipData = struct {
         // should always be enough space or is invalid msg
         var buf: [PACKET_DATA_SIZE]u8 = undefined;
         const bytes = try bincode.writeToSlice(&buf, self.data, bincode.Params.standard);
-        var sig = try keypair.sign(bytes, null);
-        self.signature.data = sig.toBytes();
+        var signature = try keypair.sign(bytes, null);
+        self.signature.data = signature.toBytes();
     }
 
     pub fn verify(self: *Self, pubkey: Pubkey) !bool {