bundle SQLean extensions

A common complain with libSQL is how to run extensions. The main
mechanism, with a .so, has a lot of issues around how those .so are
distributed.

The most common extensions are the ones in the sqlean package. We can
improve this experience by bundling them in our sqlite build.

Not all SQLean extensions are kosher: some of them, like fileio, use
the vfs. Others, are deemed too complex.

The extensions included here are a subset that we deem important enough,
and low risk enough, to just be a part of the main bundle.

The file sqlean.c has the registration function so that the extensions
are enabled in every connection.

The file sqlite3-sqlean-stripped.c is the stripped down version of
sqlite3-sqlean.c from the bundle, with the extensions we don't want
to enable removed.

libsql-ffi/Cargo.toml

@@ -16,6 +16,7 @@ libsql-wasmtime-bindings = { version = "0.2.1", optional = true }
 [build-dependencies]
 bindgen = "0.66.1"
 cc = "1.0"
+glob = "0.3"
 
 [features]
 session = []
@@ -28,3 +29,4 @@ wasm32-wasi-vfs = []
 unlock_notify = []
 preupdate_hook = []
 sqlcipher = []
+sqlean-extensions = []

libsql-ffi/build.rs

@@ -1,3 +1,4 @@
+use glob::glob;
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, OpenOptions};
@@ -146,8 +147,7 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
     std::fs::copy(format!("{dir}/{bindgen_rs_path}"), out_path).unwrap();
 
     let mut cfg = cc::Build::new();
-    cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"))
-        .flag("-std=c11")
+    cfg.flag("-std=c11")
         .flag("-DSQLITE_CORE")
         .flag("-DSQLITE_DEFAULT_FOREIGN_KEYS=1")
         .flag("-DSQLITE_ENABLE_API_ARMOR")
@@ -169,6 +169,36 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
         .flag("-D_POSIX_THREAD_SAFE_FUNCTIONS") // cross compile with MinGW
         .warnings(false);
 
+    if cfg!(feature = "sqlean-extensions") {
+        cfg.flag("-DSQLITE_EXTRA_INIT=core_init");
+        let sqlean_patterns = [
+            "crypto/*.c",
+            "fuzzy/*.c",
+            "math/*.c",
+            "stats/*.c",
+            "text/*.c",
+            "text/*/*.c",
+            "uuid/*.c",
+        ];
+
+        let mut sqlean_sources = Vec::new();
+        for pattern in sqlean_patterns {
+            let full_pattern = format!("{BUNDLED_DIR}/sqlean/{}", pattern);
+            sqlean_sources.extend(glob(&full_pattern).unwrap().filter_map(Result::ok));
+        }
+
+        cfg.files(sqlean_sources);
+
+        let sqlean_entry = format!("{BUNDLED_DIR}/src/sqlite3-sqlean-stripped.c");
+        let sqlean = format!("{BUNDLED_DIR}/src/sqlean.c");
+        cfg.file(sqlean_entry);
+        cfg.file(sqlean);
+
+        cfg.include(format!("{BUNDLED_DIR}/sqlean/"));
+    } else {
+        cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"));
+    }
+
     if cfg!(feature = "wasmtime-bindings") {
         cfg.flag("-DLIBSQL_ENABLE_WASM_RUNTIME=1");
     }

libsql-ffi/bundled/sqlean/crypto/base32.c

@@ -0,0 +1,82 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base32_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    *out_len = ((len + 4) / 5) * 8;
+    uint8_t* encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len;) {
+        uint32_t octet0 = i < len ? src[i++] : 0;
+        uint32_t octet1 = i < len ? src[i++] : 0;
+        uint32_t octet2 = i < len ? src[i++] : 0;
+        uint32_t octet3 = i < len ? src[i++] : 0;
+        uint32_t octet4 = i < len ? src[i++] : 0;
+
+        encoded[j++] = base32_chars[octet0 >> 3];
+        encoded[j++] = base32_chars[((octet0 & 0x07) << 2) | (octet1 >> 6)];
+        encoded[j++] = base32_chars[(octet1 >> 1) & 0x1F];
+        encoded[j++] = base32_chars[((octet1 & 0x01) << 4) | (octet2 >> 4)];
+        encoded[j++] = base32_chars[((octet2 & 0x0F) << 1) | (octet3 >> 7)];
+        encoded[j++] = base32_chars[(octet3 >> 2) & 0x1F];
+        encoded[j++] = base32_chars[((octet3 & 0x03) << 3) | (octet4 >> 5)];
+        encoded[j++] = base32_chars[octet4 & 0x1F];
+    }
+
+    if (len % 5 != 0) {
+        size_t padding = 7 - (len % 5) * 8 / 5;
+        for (size_t i = 0; i < padding; i++) {
+            encoded[*out_len - padding + i] = '=';
+        }
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    while (len > 0 && src[len - 1] == '=') {
+        len--;
+    }
+    *out_len = len * 5 / 8;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    size_t bits = 0, value = 0, count = 0;
+    for (size_t i = 0; i < len; i++) {
+        uint8_t c = src[i];
+        if (c >= 'A' && c <= 'Z') {
+            c -= 'A';
+        } else if (c >= '2' && c <= '7') {
+            c -= '2' - 26;
+        } else {
+            continue;
+        }
+        value = (value << 5) | c;
+        bits += 5;
+        if (bits >= 8) {
+            decoded[count++] = (uint8_t)(value >> (bits - 8));
+            bits -= 8;
+        }
+    }
+    if (bits >= 5 || (value & ((1 << bits) - 1)) != 0) {
+        free(decoded);
+        return NULL;
+    }
+    *out_len = count;
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base32.h

@@ -0,0 +1,14 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#ifndef _BASE32_H_
+#define _BASE32_H_
+
+#include <stdint.h>
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* _BASE32_H_ */

libsql-ffi/bundled/sqlean/crypto/base64.c

@@ -0,0 +1,103 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base64_chars[] =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    uint8_t* encoded = NULL;
+    size_t i, j;
+    uint32_t octets;
+
+    *out_len = ((len + 2) / 3) * 4;
+    encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (i = 0, j = 0; i < len; i += 3, j += 4) {
+        octets =
+            (src[i] << 16) | ((i + 1 < len ? src[i + 1] : 0) << 8) | (i + 2 < len ? src[i + 2] : 0);
+        encoded[j] = base64_chars[(octets >> 18) & 0x3f];
+        encoded[j + 1] = base64_chars[(octets >> 12) & 0x3f];
+        encoded[j + 2] = base64_chars[(octets >> 6) & 0x3f];
+        encoded[j + 3] = base64_chars[octets & 0x3f];
+    }
+
+    if (len % 3 == 1) {
+        encoded[*out_len - 1] = '=';
+        encoded[*out_len - 2] = '=';
+    } else if (len % 3 == 2) {
+        encoded[*out_len - 1] = '=';
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+static const uint8_t base64_table[] = {
+    // Map base64 characters to their corresponding values
+    ['A'] = 0,  ['B'] = 1,  ['C'] = 2,  ['D'] = 3,  ['E'] = 4,  ['F'] = 5,  ['G'] = 6,  ['H'] = 7,
+    ['I'] = 8,  ['J'] = 9,  ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
+    ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
+    ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
+    ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
+    ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
+    ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
+    ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
+};
+
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    if (len % 4 != 0) {
+        return NULL;
+    }
+
+    size_t padding = 0;
+    if (src[len - 1] == '=') {
+        padding++;
+    }
+    if (src[len - 2] == '=') {
+        padding++;
+    }
+
+    *out_len = (len / 4) * 3 - padding;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len; i += 4, j += 3) {
+        uint32_t block = 0;
+        for (size_t k = 0; k < 4; k++) {
+            block <<= 6;
+            if (src[i + k] == '=') {
+                padding--;
+            } else {
+                uint8_t index = base64_table[src[i + k]];
+                if (index == 0 && src[i + k] != 'A') {
+                    free(decoded);
+                    return NULL;
+                }
+                block |= index;
+            }
+        }
+
+        decoded[j] = (block >> 16) & 0xFF;
+        if (j + 1 < *out_len) {
+            decoded[j + 1] = (block >> 8) & 0xFF;
+        }
+        if (j + 2 < *out_len) {
+            decoded[j + 2] = block & 0xFF;
+        }
+    }
+
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base64.h

@@ -0,0 +1,15 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#ifndef BASE64_H
+#define BASE64_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* BASE64_H */