Syndica · 0xNineteen · Oct 11, 2023 · Oct 11, 2023 · Oct 11, 2023 · Oct 12, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,8 @@
 zig-cache/
 zig-out/
-data/
+data/
+
+# unpacking test snapshots
+test_data/version
+test_data/accounts/
+test_data/snapshots/
diff --git a/build.zig b/build.zig
@@ -123,43 +123,57 @@ pub fn build(b: *std.Build) void {
     const run_step = b.step("run", "Run the app");
     run_step.dependOn(&run_cmd.step);
 
-    // gossip fuzz testing
-    // find ./zig-cache/o/* | grep fuzz
-    // lldb $(above path)
-    const fuzz_exe = b.addExecutable(.{
-        .name = "fuzz",
-        .root_source_file = .{ .path = "src/gossip/fuzz.zig" },
-        .target = target,
-        .optimize = optimize,
-        .main_pkg_path = .{ .path = "src" },
-    });
-    fuzz_exe.addModule("base58-zig", base58_module);
-    fuzz_exe.addModule("zig-network", zig_network_module);
-    fuzz_exe.addModule("zig-cli", zig_cli_module);
-    fuzz_exe.addModule("getty", getty_mod);
-    b.installArtifact(fuzz_exe);
-    const fuzz_cmd = b.addRunArtifact(fuzz_exe);
-    b.step("fuzz_gossip", "fuzz gossip").dependOn(&fuzz_cmd.step);
-
-    // benchmarking
-    const benchmark_exe = b.addExecutable(.{
-        .name = "benchmark",
-        .root_source_file = .{ .path = "src/benchmarks.zig" },
-        .target = target,
-        // TODO: make it work
-        // .optimize = std.builtin.Mode.ReleaseSafe, // to get decent results - but things get optimized away
-        .optimize = optimize,
-        .main_pkg_path = .{ .path = "src" },
-    });
-    benchmark_exe.addModule("base58-zig", base58_module);
-    benchmark_exe.addModule("zig-network", zig_network_module);
-    benchmark_exe.addModule("zig-cli", zig_cli_module);
-    benchmark_exe.addModule("getty", getty_mod);
-    b.installArtifact(benchmark_exe);
-    const benchmark_cmd = b.addRunArtifact(benchmark_exe);
-    if (b.args) |args| {
-        benchmark_cmd.addArgs(args);
+    const ExecCommand = struct {
+        name: []const u8,
+        path: []const u8,
+        description: []const u8 = "",
+    };
+
+    const exec_commands = [_]ExecCommand{
+        ExecCommand{
+            .name = "fuzz",
+            .path = "src/gossip/fuzz.zig",
+            .description = "gossip fuzz testing",
+        },
+        ExecCommand{
+            .name = "benchmark",
+            .path = "src/benchmarks.zig",
+            .description = "benchmark client",
+        },
+        ExecCommand{
+            .name = "snapshot_utils",
+            .path = "src/cmd/snapshot_utils.zig",
+            .description = "snapshot utils",
+        },
+        ExecCommand{
+            .name = "db",
+            .path = "src/core/accounts_db.zig",
+            .description = "run accounts-db code",
+        },
+    };
+
+    for (exec_commands) |command_info| {
+        const exec = b.addExecutable(.{
+            .name = command_info.name,
+            .root_source_file = .{ .path = command_info.path },
+            .target = target,
+            .optimize = optimize,
+            .main_pkg_path = .{ .path = "src" },
+        });
+
+        // TODO: maybe we dont need all these for all bins
+        exec.addModule("base58-zig", base58_module);
+        exec.addModule("zig-network", zig_network_module);
+        exec.addModule("zig-cli", zig_cli_module);
+        exec.addModule("getty", getty_mod);
+
+        // this lets us run it as an exec
+        b.installArtifact(exec);
+
+        const cmd = b.addRunArtifact(exec);
+        if (b.args) |args| cmd.addArgs(args);
+        b
+            .step(command_info.name, command_info.description)
+            .dependOn(&cmd.step);
     }
-
-    b.step("benchmark", "benchmark gossip").dependOn(&benchmark_cmd.step);
 }
diff --git a/src/bincode/bincode.zig b/src/bincode/bincode.zig
@@ -32,7 +32,7 @@ pub fn Deserializer(comptime Reader: type) type {
 
         const Self = @This();
         const Error = getty.de.Error || error{
-            IO,
+            EOF,
         };
 
         const De = Self.@"getty.Deserializer";
@@ -65,16 +65,14 @@ pub fn Deserializer(comptime Reader: type) type {
                 .Little => self.reader.readIntLittle(T),
                 .Big => self.reader.readIntBig(T),
             } catch {
-                return Error.IO;
+                return Error.EOF;
             };
 
             var ma = MapAccess{ .d = self, .len = len };
             return visitor.visitMap(allocator, De, ma.mapAccess());
         }
 
         fn deserializeSeq(self: *Self, allocator: ?std.mem.Allocator, visitor: anytype) Error!@TypeOf(visitor).Value {
-            // var len = if (self.params.include_fixed_array_length) try self.deserializeInt(allocator, getty.de.blocks.Int.Visitor(u64)) else null;
-
             const len = blk: {
                 const visitor_value = @TypeOf(visitor).Value;
                 const tmp: visitor_value = undefined; // TODO: fix without stack alloc?
@@ -88,15 +86,14 @@ pub fn Deserializer(comptime Reader: type) type {
                         .Little => self.reader.readIntLittle(T),
                         .Big => self.reader.readIntBig(T),
                     } catch {
-                        return Error.IO;
+                        return Error.EOF;
                     };
                     break :blk len;
                 }
             };
 
             var s = SeqAccess{ .d = self, .len = len };
             const result = try visitor.visitSeq(allocator.?, De, s.seqAccess());
-            errdefer getty.de.free(allocator.?, De, result);
 
             return result;
         }
@@ -108,7 +105,7 @@ pub fn Deserializer(comptime Reader: type) type {
 
         pub fn deserializeOptional(self: *Self, ally: ?std.mem.Allocator, visitor: anytype) Error!@TypeOf(visitor).Value {
             const byte = self.reader.readByte() catch {
-                return Error.IO;
+                return Error.EOF;
             };
             return switch (byte) {
                 0 => try visitor.visitNull(ally, De),
@@ -123,7 +120,7 @@ pub fn Deserializer(comptime Reader: type) type {
                 .Little => self.reader.readIntLittle(T),
                 .Big => self.reader.readIntBig(T),
             } catch {
-                return Error.IO;
+                return Error.EOF;
             };
             return try visitor.visitInt(ally, De, tag);
         }
@@ -134,7 +131,7 @@ pub fn Deserializer(comptime Reader: type) type {
             if (info.bits != 32 and info.bits != 64) {
                 @compileError("Only f{32, 64} floating-point integers may be serialized, but attempted to serialize " ++ @typeName(T) ++ ".");
             }
-            const bytes = self.reader.readBytesNoEof((info.bits + 7) / 8) catch return Error.IO;
+            const bytes = self.reader.readBytesNoEof((info.bits + 7) / 8) catch return Error.EOF;
             const f = @as(T, @bitCast(bytes));
             return visitor.visitFloat(ally, De, f);
         }
@@ -146,19 +143,19 @@ pub fn Deserializer(comptime Reader: type) type {
                 .Little => self.reader.readIntLittle(T),
                 .Big => self.reader.readIntBig(T),
             } catch {
-                return Error.IO;
+                return Error.EOF;
             };
             return try visitor.visitInt(ally, De, value);
         }
 
         pub fn deserializeBool(self: *Self, ally: ?std.mem.Allocator, visitor: anytype) Error!@TypeOf(visitor).Value {
             const byte = self.reader.readByte() catch {
-                return Error.IO;
+                return Error.EOF;
             };
             const value = switch (byte) {
                 0 => false,
                 1 => true,
-                else => return getty.de.Error.InvalidValue,
+                else => return Error.InvalidValue,
             };
 
             return try visitor.visitBool(ally, De, value);
@@ -215,9 +212,8 @@ pub fn Deserializer(comptime Reader: type) type {
                     },
                     .Struct => |*info| {
                         inline for (info.fields) |field| {
-                            if (get_field_config(T, field)) |config| {
+                            if (getFieldConfig(T, field)) |config| {
                                 if (config.free) |free_fcn| {
-                                    // std.debug.print("found free fcn...\n", .{});
                                     var field_value = @field(value, field.name);
                                     switch (@typeInfo(field.type)) {
                                         .Pointer => |*field_info| {
@@ -278,18 +274,38 @@ pub fn Deserializer(comptime Reader: type) type {
 
                         inline for (info.fields) |field| {
                             if (!field.is_comptime) {
-                                if (get_field_config(T, field)) |config| {
-                                    if (shouldUseDefaultValue(field, config)) |default_val| {
-                                        @field(data, field.name) = @as(*const field.type, @ptrCast(@alignCast(default_val))).*;
+                                if (getFieldConfig(T, field)) |config| {
+                                    if (shouldUseDefaultValue(field, config)) |default_value| {
+                                        @field(data, field.name) = @as(*const field.type, @ptrCast(@alignCast(default_value))).*;
                                         continue;
                                     }
 
                                     if (config.deserializer) |deser_fcn| {
                                         @field(data, field.name) = deser_fcn(alloc, reader, params) catch return getty.de.Error.InvalidValue;
                                         continue;
                                     }
-                                }
 
+                                    if (config.default_on_eof) {
+                                        const field_type = field.type;
+
+                                        @field(data, field.name) = getty.deserialize(alloc, field_type, dd) catch |err| blk: {
+                                            if (err == Error.EOF) {
+                                                if (config.default_fn) |default_fn| {
+                                                    break :blk default_fn(alloc.?);
+                                                } else {
+                                                    if (field.default_value) |default_value| {
+                                                        break :blk @as(*const field.type, @ptrCast(@alignCast(default_value))).*;
+                                                    } else {
+                                                        @compileError("field " ++ field.name ++ " has no default value");
+                                                    }
+                                                }
+                                            } else {
+                                                return err;
+                                            }
+                                        };
+                                        continue;
+                                    }
+                                }
                                 @field(data, field.name) = try getty.deserialize(alloc, field.type, dd);
                             }
                         }
@@ -557,7 +573,7 @@ pub fn Serializer(
 
                         inline for (info.fields) |field| {
                             if (!field.is_comptime) {
-                                if (get_field_config(T, field)) |config| {
+                                if (getFieldConfig(T, field)) |config| {
                                     if (config.skip) {
                                         continue;
                                     }
@@ -652,10 +668,12 @@ pub fn FieldConfig(comptime T: type) type {
         serializer: ?SerializeFunction = null,
         free: ?FreeFunction = null,
         skip: bool = false,
+        default_on_eof: bool = false,
+        default_fn: ?fn (alloc: std.mem.Allocator) T = null,
     };
 }
 
-pub fn get_field_config(comptime struct_type: type, comptime field: std.builtin.Type.StructField) ?FieldConfig(field.type) {
+pub fn getFieldConfig(comptime struct_type: type, comptime field: std.builtin.Type.StructField) ?FieldConfig(field.type) {
     const bincode_field = "!bincode-config:" ++ field.name;
     if (@hasDecl(struct_type, bincode_field)) {
         const config = @field(struct_type, bincode_field);
@@ -715,7 +733,6 @@ pub fn getSerializedSizeWithSlice(slice: []u8, data: anytype, params: Params) !u
 pub fn getSerializedSize(alloc: std.mem.Allocator, data: anytype, params: Params) !usize {
     var list = try writeToArray(alloc, data, params);
     defer list.deinit();
-
     return list.items.len;
 }
 
@@ -769,17 +786,44 @@ fn TestSliceConfig(comptime Child: type) FieldConfig([]Child) {
     };
 }
 
+test "bincode: default on eof" {
+    const defaultArrayListOnEOFConfig = @import("../utils/arraylist.zig").defaultArrayListOnEOFConfig;
+    const Foo = struct {
+        value: u8 = 0,
+        accounts: std.ArrayList(u64),
+        pub const @"!bincode-config:accounts" = defaultArrayListOnEOFConfig(u64);
+        pub const @"!bincode-config:value" = .{ .default_on_eof = true };
+    };
+
+    var buf: [1]u8 = .{1};
+    var r = try readFromSlice(std.testing.allocator, Foo, &buf, .{});
+    try std.testing.expect(r.value == 1);
+
+    var buf2: [1024]u8 = undefined;
+    const slice = try writeToSlice(&buf2, Foo{
+        .value = 10,
+        .accounts = std.ArrayList(u64).init(std.testing.allocator),
+    }, .{});
+
+    var r2 = try readFromSlice(std.testing.allocator, Foo, slice, .{});
+    try std.testing.expect(r2.value == 10);
+}
+
 test "bincode: custom field serialization" {
     const Foo = struct {
         accounts: []u8,
         txs: []u32,
         skip_me: u8 = 20,
+        skip_me_null: ?u8 = null,
 
         pub const @"!bincode-config:accounts" = TestSliceConfig(u8);
         pub const @"!bincode-config:txs" = TestSliceConfig(u32);
         pub const @"!bincode-config:skip_me" = FieldConfig(u8){
             .skip = true,
         };
+        pub const @"!bincode-config:skip_me_null" = FieldConfig(?u8){
+            .skip = true,
+        };
     };
 
     var accounts = [_]u8{ 1, 2, 3 };

diff --git a/src/bloom/bitvec.zig b/src/bloom/bitvec.zig
@@ -43,6 +43,36 @@ pub const BitVec = struct {
     }
 };
 
+pub fn BitVecConfig() bincode.FieldConfig(DynamicBitSet) {
+    const S = struct {
+        pub fn serialize(writer: anytype, data: anytype, params: bincode.Params) !void {
+            var bitset: DynamicBitSet = data;
+            var bitvec = BitVec.initFromBitSet(bitset);
+            try bincode.write(null, writer, bitvec, params);
+        }
+
+        pub fn deserialize(allocator: ?std.mem.Allocator, reader: anytype, params: bincode.Params) !DynamicBitSet {
+            var ally = allocator.?;
+            var bitvec = try bincode.read(ally, BitVec, reader, params);
+            defer bincode.free(ally, bitvec);
+
+            var dynamic_bitset = try bitvec.toBitSet(ally);
+            return dynamic_bitset;
+        }
+
+        pub fn free(allocator: std.mem.Allocator, data: anytype) void {
+            _ = allocator;
+            data.deinit();
+        }
+    };
+
+    return bincode.FieldConfig(DynamicBitSet){
+        .serializer = S.serialize,
+        .deserializer = S.deserialize,
+        .free = S.free,
+    };
+}
+
 test "bloom.bitvec: serializes/deserializes and matches Rust's BitVec" {
     var rust_bit_vec_serialized = [_]u8{
         1,   2,   0,   0,   0,   0,   0, 0, 0, 255, 255, 239, 191, 255, 255, 255, 255, 255, 255, 255,