From 425fdcbf2db07fe263b5658507660b71023cd520 Mon Sep 17 00:00:00 2001
From: Joshua Hoskins <jrh4dy@gmail.com>
Date: Tue, 19 Nov 2024 16:08:31 -0500
Subject: [PATCH] Adding documentation to the header, experiment with reading
 all the data bucket. Still hazy on structure/layout of the column data.

---
 .../templates/standard_storage_manager.ksy    | 39 +++++++++++++++++--
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/src/tablestream/templates/standard_storage_manager.ksy b/src/tablestream/templates/standard_storage_manager.ksy
index 7e2a52c..ea8e8ca 100644
--- a/src/tablestream/templates/standard_storage_manager.ksy
+++ b/src/tablestream/templates/standard_storage_manager.ksy
@@ -13,8 +13,11 @@ seq:
     type: header
   - id: header_remains
     size: 512 - _io.pos
+    doc: |
+      This is just the remainder of the 512 bytes allocated to the header.
+      Nothing to see here really.
   - id: bucket
-    type: bucket(header.bucket_size)
+    type: data_bucket(header.bucket_size, header.n_buckets)
 
 
 types:
@@ -34,6 +37,19 @@ types:
           pos: offset
           type: one_col
 
+  data_bucket:
+    params:
+      - id: bucket_size
+        type: u4
+      - id: n_buckets
+        type: u4
+
+    seq:
+      - id: data
+        size: bucket_size
+        repeat: expr
+        repeat-expr: n_buckets
+
   string_bucket_header:
     seq:
       - id: free_bucket_list
@@ -61,13 +77,14 @@ types:
       - id: rows_populated
         type: u4
         repeat: expr
-        repeat-expr: 2 #rows_stored
+        repeat-expr: 1 #rows_stored
       - id: some_int
         type: u4
         repeat: expr
         repeat-expr: 1 #50
 
 
+
   header:
     seq:
       - id: magic
@@ -86,27 +103,41 @@ types:
         if: version > 2
       - id: bucket_size
         type: u4
+        doc: |
+          Bucket size in bytes
       - id: n_buckets
         type: u4
+        doc: |
+          Number of buckets
       - id: cache_size
         type: u4
         doc: |
           Cache size in buckets
-      - id: n_index_buckets
+      - id: n_free_buckets
         type: u4
+        doc: |
+          Number of free buckets
       - id: first_free_bucket
         type: s4
-      - id: index_of_bucket
+        doc: |
+          First free bucket (None=-1)
+      - id: n_index_buckets
         type: u4
       - id: first_index_bucket
         type: s4
       - id: offset_index
         type: u4
         if: version > 1
+        doc: |
+          Offset of index in bucket (No string heap => -1)
       - id: last_string_heap_bucket
         type: s4
+        doc: |
+          Last string heap bucket (None = -1)
       - id: index_length
         type: u4
+        doc: |
+          Index length in bytes
       - id: n_indicies
         type: u4