Merge branch 'tursodatabase:main' into main

.env

@@ -0,0 +1,5 @@
+export LIBSQL_BOTTOMLESS_ENDPOINT="http://localhost:9000"
+export LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY="minioadmin"
+export LIBSQL_BOTTOMLESS_AWS_ACCESS_KEY_ID="minioadmin"
+export LIBSQL_BOTTOMLESS_AWS_DEFAULT_REGION="us-east-1"
+export LIBSQL_BOTTOMLESS_BUCKET="my-bucket"

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   "bottomless-cli",
   "libsql-sys-tmp",
   "libsql-replication",
+  "libsql-ffi",
 
   "vendored/rusqlite",
   "vendored/sqlite3-parser",

README.md

@@ -38,7 +38,7 @@ To get started with the libSQL API:
 * [Rust](libsql) 
 * [Python](https://github.com/libsql/libsql-experimental-python) (experimental)
 * [Go](bindings/go) (experimental)
-* [C](bindings/c) (experimantal)
+* [C](bindings/c) (experimental)
 
 To build the SQLite-compatible C library and tools, run:
 

bottomless-cli/src/main.rs

@@ -128,14 +128,17 @@ async fn run() -> Result<()> {
             Ok(Some((page_size, checksum))) => (page_size, checksum),
             Ok(None) => {
                 println!("No local metadata found, continuing anyway");
-                (4096, 0)
+                (4096, (0, 0))
             }
             Err(e) => {
                 println!("Failed to get local metadata: {e}, continuing anyway");
-                (4096, 0)
+                (4096, (0, 0))
             }
         };
-        println!("Local metadata: page_size={page_size}, checksum={checksum:x}");
+        println!(
+            "Local metadata: page_size={page_size}, checksum={:X}-{:X}",
+            checksum.0, checksum.1
+        );
         Replicator::restore_from_local_snapshot(&from_dir, &mut db_file).await?;
         println!("Restored local snapshot to {}", database);
         let applied_frames = Replicator::apply_wal_from_local_generation(

bottomless-cli/src/replicator_extras.rs

@@ -145,7 +145,7 @@ impl Replicator {
                         println!("\tconsistent WAL frame: {consistent_frame}");
                         if let Some((page_size, crc)) = m {
                             println!("\tpage size:            {}", page_size);
-                            println!("\tWAL frame checksum:   {:x}", crc);
+                            println!("\tWAL frame checksum:   {:X}-{:X}", crc.0, crc.1);
                         }
                         if let Some(prev_gen) = parent {
                             println!("\tprevious generation:  {}", prev_gen);
@@ -256,7 +256,7 @@ impl Replicator {
         println!("\tconsistent WAL frame: {consistent_frame}");
         if let Some((page_size, crc)) = meta {
             println!("\tpage size:            {}", page_size);
-            println!("\tWAL frame checksum:   {:x}", crc);
+            println!("\tWAL frame checksum:   {:X}-{:X}", crc.0, crc.1);
         }
         if let Some(prev_gen) = dep {
             println!("\tprevious generation:  {}", prev_gen);
@@ -316,7 +316,7 @@ impl Replicator {
         from_dir: impl AsRef<std::path::Path>,
         db: &mut tokio::fs::File,
         page_size: u32,
-        checksum: u64,
+        checksum: (u32, u32),
     ) -> Result<u32> {
         use bottomless::transaction_cache::TransactionPageCache;
         use tokio::io::AsyncWriteExt;
@@ -356,7 +356,7 @@ impl Replicator {
         let mut pending_pages =
             TransactionPageCache::new(SWAP_AFTER, page_size, TMP_RESTORE_DIR.into());
 
-        let mut checksum: Option<u64> = Some(checksum);
+        let mut checksum: Option<(u32, u32)> = Some(checksum);
         for obj in objs {
             let key = obj.file_name().unwrap().to_str().unwrap();
             tracing::debug!("Loading {}", key);
@@ -418,13 +418,13 @@ impl Replicator {
 
     pub async fn get_local_metadata(
         from_dir: impl AsRef<std::path::Path>,
-    ) -> Result<Option<(u32, u64)>> {
+    ) -> Result<Option<(u32, (u32, u32))>> {
         use bytes::Buf;
 
         if let Ok(data) = tokio::fs::read(from_dir.as_ref().join(".meta")).await {
             let mut data = bytes::Bytes::from(data);
             let page_size = data.get_u32();
-            let crc = data.get_u64();
+            let crc = (data.get_u32(), data.get_u32());
             Ok(Some((page_size, crc)))
         } else {
             Ok(None)

bottomless/src/backup.rs

@@ -11,7 +11,6 @@ use uuid::Uuid;
 
 #[derive(Debug)]
 pub(crate) struct WalCopier {
-    wal: Option<WalFileReader>,
     outbox: Sender<String>,
     use_compression: CompressionKind,
     max_frames_per_batch: usize,
@@ -32,7 +31,6 @@ impl WalCopier {
         outbox: Sender<String>,
     ) -> Self {
         WalCopier {
-            wal: None,
             bucket,
             db_name,
             generation,
@@ -49,15 +47,9 @@ impl WalCopier {
             tracing::trace!("Trying to flush empty frame range");
             return Ok(frames.start - 1);
         }
-        let wal = {
-            if self.wal.is_none() {
-                self.wal = WalFileReader::open(&self.wal_path).await?;
-            }
-            if let Some(wal) = self.wal.as_mut() {
-                wal
-            } else {
-                return Err(anyhow!("WAL file not found: `{}`", self.wal_path));
-            }
+        let mut wal = match WalFileReader::open(&self.wal_path).await? {
+            Some(wal) => wal,
+            None => return Err(anyhow!("WAL file not found: `{}`", self.wal_path)),
         };
         let generation = if let Some(generation) = self.generation.load_full() {
             generation
@@ -74,10 +66,11 @@ impl WalCopier {
             let mut meta_file = tokio::fs::File::create(&meta_path).await?;
             let buf = {
                 let page_size = wal.page_size();
-                let crc = wal.checksum();
+                let (checksum_1, checksum_2) = wal.checksum();
                 let mut buf = [0u8; 12];
                 buf[0..4].copy_from_slice(page_size.to_be_bytes().as_slice());
-                buf[4..].copy_from_slice(crc.to_be_bytes().as_slice());
+                buf[4..8].copy_from_slice(checksum_1.to_be_bytes().as_slice());
+                buf[8..12].copy_from_slice(checksum_2.to_be_bytes().as_slice());
                 buf
             };
             meta_file.write_all(buf.as_ref()).await?;

bottomless/src/replicator.rs

@@ -1251,7 +1251,7 @@ impl Replicator {
         generation: &Uuid,
         page_size: usize,
         last_consistent_frame: Option<u32>,
-        mut checksum: u64,
+        mut checksum: (u32, u32),
         utc_time: Option<NaiveDateTime>,
         db: &mut File,
     ) -> Result<bool> {
@@ -1499,18 +1499,20 @@ impl Replicator {
         None
     }
 
-    async fn store_metadata(&self, page_size: u32, crc: u64) -> Result<()> {
+    async fn store_metadata(&self, page_size: u32, checksum: (u32, u32)) -> Result<()> {
         let generation = self.generation()?;
         let key = format!("{}-{}/.meta", self.db_name, generation);
         tracing::debug!(
-            "Storing metadata at '{}': page size - {}, crc - {}",
+            "Storing metadata at '{}': page size - {}, crc - {},{}",
             key,
             page_size,
-            crc
+            checksum.0,
+            checksum.1,
         );
         let mut body = Vec::with_capacity(12);
         body.extend_from_slice(page_size.to_be_bytes().as_slice());
-        body.extend_from_slice(crc.to_be_bytes().as_slice());
+        body.extend_from_slice(checksum.0.to_be_bytes().as_slice());
+        body.extend_from_slice(checksum.1.to_be_bytes().as_slice());
         let _ = self
             .client
             .put_object()
@@ -1522,7 +1524,7 @@ impl Replicator {
         Ok(())
     }
 
-    pub async fn get_metadata(&self, generation: &Uuid) -> Result<Option<(u32, u64)>> {
+    pub async fn get_metadata(&self, generation: &Uuid) -> Result<Option<(u32, (u32, u32))>> {
         let key = format!("{}-{}/.meta", self.db_name, generation);
         if let Ok(obj) = self
             .client
@@ -1534,8 +1536,8 @@ impl Replicator {
         {
             let mut data = obj.body.collect().await?;
             let page_size = data.get_u32();
-            let crc = data.get_u64();
-            Ok(Some((page_size, crc)))
+            let checksum = (data.get_u32(), data.get_u32());
+            Ok(Some((page_size, checksum)))
         } else {
             Ok(None)
         }

bottomless/src/wal.rs

@@ -37,26 +37,28 @@ impl WalFrameHeader {
         ])
     }
 
-    pub fn crc(&self) -> u64 {
-        u64::from_be_bytes([
-            self.0[16], self.0[17], self.0[18], self.0[19], self.0[20], self.0[21], self.0[22],
-            self.0[23],
-        ])
+    pub fn crc(&self) -> (u32, u32) {
+        (
+            u32::from_be_bytes([self.0[16], self.0[17], self.0[18], self.0[19]]),
+            u32::from_be_bytes([self.0[20], self.0[21], self.0[22], self.0[23]]),
+        )
     }
 
-    pub fn verify(&self, init_crc: u64, page_data: &[u8]) -> Result<u64> {
+    pub fn verify(&self, init_crc: (u32, u32), page_data: &[u8]) -> Result<(u32, u32)> {
         let mut crc = init_crc;
-        crc = checksum_be(crc, &self.0[0..8]);
-        crc = checksum_be(crc, page_data);
+        crc = checksum_step(crc, &self.0[0..8]);
+        crc = checksum_step(crc, page_data);
         let frame_crc = self.crc();
         if crc == frame_crc {
             Ok(crc)
         } else {
             Err(anyhow!(
-                "Frame checksum verification failed for page no. {}. Expected: {:X}. Got: {:X}",
+                "Frame checksum verification failed for page no. {}. Expected: {:X}-{:X}. Got: {:X}-{:X}",
                 self.pgno(),
-                frame_crc,
-                crc
+                frame_crc.0,
+                frame_crc.1,
+                crc.0,
+                crc.1,
             ))
         }
     }
@@ -96,7 +98,8 @@ pub(crate) struct WalHeader {
     /// A different random integer changing with each checkpoint
     pub salt_2: u32,
     /// Checksum for first 24 bytes of header
-    pub crc: u64,
+    pub checksum_1: u32,
+    pub checksum_2: u32,
 }
 
 impl WalHeader {
@@ -112,7 +115,8 @@ impl From<[u8; WalHeader::SIZE as usize]> for WalHeader {
             checkpoint_seq_no: u32::from_be_bytes([v[12], v[13], v[14], v[15]]),
             salt_1: u32::from_be_bytes([v[16], v[17], v[18], v[19]]),
             salt_2: u32::from_be_bytes([v[20], v[21], v[22], v[23]]),
-            crc: u64::from_be_bytes([v[24], v[25], v[26], v[27], v[28], v[29], v[30], v[31]]),
+            checksum_1: u32::from_be_bytes([v[24], v[25], v[26], v[27]]),
+            checksum_2: u32::from_be_bytes([v[28], v[29], v[30], v[31]]),
         }
     }
 }
@@ -143,8 +147,8 @@ impl WalFileReader {
         self.header.page_size
     }
 
-    pub fn checksum(&self) -> u64 {
-        self.header.crc
+    pub fn checksum(&self) -> (u32, u32) {
+        (self.header.checksum_1, self.header.checksum_2)
     }
 
     pub fn frame_size(&self) -> u64 {
@@ -241,18 +245,21 @@ impl AsMut<File> for WalFileReader {
 
 /// Generate or extend an 8 byte checksum based on the data in
 ///the `page` and the `init` value. `page` size must be multiple of 8.
-pub fn checksum_be(init: u64, page: &[u8]) -> u64 {
+/// FIXME: these computations are performed with host endianness,
+/// which is softly assumed to be little endian for majority of devices.
+/// However, the only proper way to do this is to get the endianness
+/// from the WAL header, as per https://www.sqlite.org/fileformat.html#checksum_algorithm
+pub fn checksum_step(init: (u32, u32), page: &[u8]) -> (u32, u32) {
     debug_assert_eq!(page.len() % 8, 0);
-    let mut s1 = (init >> 32) as u32;
-    let mut s2 = (init & u32::MAX as u64) as u32;
+    let (mut s0, mut s1) = init;
     let page = unsafe { std::slice::from_raw_parts(page.as_ptr() as *const u32, page.len() / 4) };
     let mut i = 0;
     while i < page.len() {
-        s1 = s1.wrapping_add(page[i]).wrapping_add(s2);
-        s2 = s2.wrapping_add(page[i + 1]).wrapping_add(s1);
+        s0 = s0.wrapping_add(page[i]).wrapping_add(s1);
+        s1 = s1.wrapping_add(page[i + 1]).wrapping_add(s0);
         i += 2;
     }
-    ((s1 as u64) << 32) | (s2 as u64)
+    (s0, s1)
 }
 
 #[cfg(test)]
@@ -273,7 +280,8 @@ mod test {
             checkpoint_seq_no: 0,
             salt_1: 3188076412,
             salt_2: 666853980,
-            crc: 5842868361513443485,
+            checksum_1: 1360398801,
+            checksum_2: 1700831389,
         };
         let actual = WalHeader::from(source);
         assert_eq!(actual, expected);

docs/client_version_metrics.md

@@ -0,0 +1,15 @@
+# Client verison metrics
+
+Currently, `sqld` supports clients passing their client version via a
+`x-libsql-client-version` header. The value of this header should follow this
+pattern:
+
+- Hrana/Remote clients should be `libsql-remote-<language>-<version>`
+- Embedded replica clients should be `libsql-rpc-<language>-<version>`
+
+
+`<language>` should be a reference to the language, for example,
+`rust`/`go`/`js`/`python`.
+
+`<version>` should be a reference to either a semver version or a commit sha
+(first 6 chars of the sha).