Skip to content

Commit

Permalink
feat: overwrite, benchmark, setMany, index
Browse files Browse the repository at this point in the history
  • Loading branch information
@dan-online
dan-online committed Apr 7, 2024
1 parent ee75221 commit abc0b42
Show file tree
Hide file tree
Showing 14 changed files with 394 additions and 482 deletions.
2 changes: 1 addition & 1 deletion benchmark/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
data
*.sqlite*
65 changes: 65 additions & 0 deletions benchmark/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# BrineDB Benchmark

To run the benchmark, you need to follow these steps:

1. Run ``docker compose up -d``
2. Run ``cd .. && yarn release-native && cd benchmark``
3. Run ``yarn bench``

The benchmark will run and output the results to the console.

## Benchmark Results

Valid as of 2024-04-07

```bash
😃 Results for: SQLite (Memory)
┌─────────┬───────────┬──────────┬───────────────────┬──────────┬─────────┬───────────────────┐
│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ Average Time (ms) │
├─────────┼───────────┼──────────┼───────────────────┼──────────┼─────────┼───────────────────┤
│ 0 │ 'get' │ '19,244' │ 51962.33089113228 │ '±1.74%' │ 19245 │ '0.052' │
│ 1 │ 'set' │ '18,509' │ 54027.41947847479 │ '±2.45%' │ 18523 │ '0.054' │
│ 2 │ 'count' │ '4,056' │ 246511.4823761241 │ '±0.25%' │ 4057 │ '0.247' │
│ 3 │ 'setMany' │ '1,186' │ 842960.5206401688 │ '±4.24%' │ 1187 │ '0.843' │
└─────────┴───────────┴──────────┴───────────────────┴──────────┴─────────┴───────────────────┘

😃 Results for: SQLite (File)
┌─────────┬───────────┬──────────┬────────────────────┬──────────┬─────────┬───────────────────┐
│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ Average Time (ms) │
├─────────┼───────────┼──────────┼────────────────────┼──────────┼─────────┼───────────────────┤
│ 0 │ 'get' │ '17,053' │ 58637.99859266467 │ '±1.57%' │ 17054 │ '0.059' │
│ 1 │ 'set' │ '98' │ 10146073.959595695 │ '±6.38%' │ 99 │ '10.146' │
│ 2 │ 'count' │ '189' │ 5266332.842932184 │ '±1.59%' │ 191 │ '5.266' │
│ 3 │ 'setMany' │ '1,475' │ 677721.3696680956 │ '±4.24%' │ 1477 │ '0.678' │
└─────────┴───────────┴──────────┴────────────────────┴──────────┴─────────┴───────────────────┘

😃 Results for: PostgreSQL
┌─────────┬───────────┬─────────┬────────────────────┬──────────┬─────────┬───────────────────┐
│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ Average Time (ms) │
├─────────┼───────────┼─────────┼────────────────────┼──────────┼─────────┼───────────────────┤
│ 0 │ 'get' │ '4,884' │ 204745.00000005757 │ '±1.48%' │ 4885 │ '0.205' │
│ 1 │ 'set' │ '1,730' │ 577933.152512942 │ '±1.78%' │ 1731 │ '0.578' │
│ 2 │ 'count' │ '12' │ 80366305.00000095 │ '±1.26%' │ 13 │ '80.366' │
│ 3 │ 'setMany' │ '445' │ 2243906.7219727584 │ '±2.40%' │ 446 │ '2.244' │
└─────────┴───────────┴─────────┴────────────────────┴──────────┴─────────┴───────────────────┘

😃 Results for: MySQL
┌─────────┬───────────┬─────────┬────────────────────┬───────────┬─────────┬───────────────────┐
│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ Average Time (ms) │
├─────────┼───────────┼─────────┼────────────────────┼───────────┼─────────┼───────────────────┤
│ 0 │ 'get' │ '3,631' │ 275340.9485132162 │ '±1.28%' │ 3632 │ '0.275' │
│ 1 │ 'set' │ '399' │ 2502239.482499499 │ '±12.75%' │ 400 │ '2.502' │
│ 2 │ 'count' │ '11' │ 84151469.41666414 │ '±19.07%' │ 12 │ '84.151' │
│ 3 │ 'setMany' │ '747' │ 1337851.9331546512 │ '±2.58%' │ 748 │ '1.338' │
└─────────┴───────────┴─────────┴────────────────────┴───────────┴─────────┴───────────────────┘

😃 Results for: MariaDB
┌─────────┬───────────┬─────────┬────────────────────┬──────────┬─────────┬───────────────────┐
│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ Average Time (ms) │
├─────────┼───────────┼─────────┼────────────────────┼──────────┼─────────┼───────────────────┤
│ 0 │ 'get' │ '3,572' │ 279935.25776648754 │ '±2.08%' │ 3573 │ '0.280' │
│ 1 │ 'set' │ '1,215' │ 822408.2055921347 │ '±6.10%' │ 1216 │ '0.822' │
│ 2 │ 'count' │ '1' │ 977628002.3000028 │ '±2.41%' │ 10 │ '977.628' │
│ 3 │ 'setMany' │ '630' │ 1584830.0000005558 │ '±2.99%' │ 631 │ '1.585' │
└─────────┴───────────┴─────────┴────────────────────┴──────────┴─────────┴───────────────────┘
```
25 changes: 25 additions & 0 deletions benchmark/docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
services:
mysql:
image: mysql
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_DATABASE: brinedb
ports:
- "3306:3306"

postgres:
image: postgres
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: root
POSTGRES_DB: brinedb
ports:
- "5432:5432"

mariadb:
image: mariadb
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_DATABASE: brinedb
ports:
- "3307:3306"
7 changes: 3 additions & 4 deletions benchmark/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@
"name": "brine-benchmark",
"scripts": {
"build": "tsup",
"start": "rm data/*; node dist/index.js"
"start": "node dist/index.js",
"bench": "yarn build && yarn start"
},
"dependencies": {
"@favware/colorette-spinner": "^1.0.1",
"@joshdb/core": "npm:^1.2.7",
"@joshdb/sqlite": "npm:^1.1.9",
"better-sqlite3": "latest"
"tinybench": "^2.6.0"
},
"devDependencies": {
"tsup": "^8.0.2"
Expand Down
222 changes: 102 additions & 120 deletions benchmark/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,150 +1,132 @@
import { Brine } from "../../dist";
import Josh from "@joshdb/core";
// @ts-expect-error 7016
import JoshSqlite from "@joshdb/sqlite";
import { Brine } from "../../";
import { Spinner } from "@favware/colorette-spinner";
import { BrineDatabases } from "../../src";
import { Bench } from "tinybench";

const runs = 100;
const databaseSize = 5;
const randomData = (length: number) => {
const chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let result = "";

console.log(
`\nBrine Benchmark commencing, runs set at ${runs} and ${databaseSize} database size.\n`,
);

interface kv {
set: (key: string, value: string) => Promise<void>;
get: (key: string) => Promise<void>;
clear: () => Promise<void>;
}

const test = async (store: kv) => {
const start = performance.now();
for (let i = 0; i < length; i++) {
result += chars.charAt(Math.floor(Math.random() * chars.length));
}

const randomKey = Math.random().toString(36).substring(7);
const randomValue = Math.random().toString(36).substring(7);
return result;
};

await store.set(randomKey, randomValue);
console.log(await store.get(randomKey));
const benchme = async (name: string, db: Brine) => {
const spinner = new Spinner("Initializing database");
const bench = new Bench({ time: 1000, warmupTime: 500 });

const end = performance.now();
spinner.start();

return end - start;
};
await db.init();

const spinner = new Spinner("Benchmark");
const benchmark = async (name: string, runner: kv) => {
spinner.start({
text: `Benchmarking (${name}): 0/${runs}`,
spinner.update({
text: "Setting up database",
});

let total = 0;
let lastUpdate = Date.now();
await db.clear();

await runner.clear();
const setInitialManyData: [string, string][] = [];

for (let i = 0; i < 1_000_000; i++) {
setInitialManyData.push([`key-${i}`, randomData(100)]);

for (let i = 0; i < runs; i++) {
total += await test(runner);
spinner.update({
text: `Benchmarking (${name}): ${i + 1}/${runs}`,
text: `Setting up database (${i + 1}/1000000) (${(
(i / 1_000_000) *
100
).toFixed(2)}%)`,
});

if (lastUpdate + 50 < Date.now()) {
lastUpdate = Date.now();
spinner.spin();
}
}

return {
[name]: {
"Average (ms)": (total / runs).toFixed(3),
"Operations (op/s)": Math.round(runs / (total / 1000)).toLocaleString(),
"Total (s)": (total / 1000).toFixed(2),
},
};
};

(async () => {
const josh = new Josh({
name: "josh",
provider: JoshSqlite,
spinner.update({
text: "Setting up database",
});

const joshResults = await benchmark("Josh (SQLite)", {
set: async (key, value) => {
await josh.set(key, value);
},
get: async (key) => {
await josh.get(key);
},
clear: async () => {
await josh.delete(josh.all);
},
});
await db.setMany(setInitialManyData);

const brine_postgres = new Brine<string>(
BrineDatabases.postgres.build({
host: "localhost",
port: 5432,
user: "root",
password: "root",
database: "brine",
}),
const setManyData: [string, string][] = Array.from(
{ length: 100 },
(_, i) => [`key-many-${i}`, randomData(100)],
);

await brine_postgres.init();

const brineResults = await benchmark("Brine (Postgres)", {
set: async (key, value) => {
await brine_postgres.set(key, value);
},
get: async (key) => {
await brine_postgres.get(key);
},
clear: async () => {
await brine_postgres.clear();
},
bench
.add("get", async () => {
await db.get(`key-${Math.floor(Math.random() * 1000)}`);
})
.add("set", async () => {
await db.set(`key-${Math.floor(Math.random() * 1000)}`, randomData(100));
})
.add("count", async () => {
await db.count();
})
.add("setMany", async () => {
await db.setMany(setManyData);
});

spinner.update({
text: "Running warmup",
});

const brine_sqlite = new Brine<string>(
BrineDatabases.sqlite.file("./data/brine.sqlite"),
);
await bench.warmup();

await brine_sqlite.init();

const brineResultsSqlite = await benchmark("Brine (SQLite)", {
set: async (key, value) => {
await brine_sqlite.set(key, value);
},
get: async (key) => {
await brine_sqlite.get(key);
},
clear: async () => {
await brine_sqlite.clear();
},
spinner.update({
text: "Running benchmarks",
});

const brine_memory = new Brine<string>(BrineDatabases.sqlite.memory);
await bench.run();
await db.close();

await brine_memory.init();
spinner.stop();

const brineResultsMemory = await benchmark("Brine (Memory)", {
set: async (key, value) => {
await brine_memory.set(key, value);
},
get: async (key) => {
await brine_memory.get(key);
},
clear: async () => {
await brine_memory.clear();
},
});
// clear line
process.stdout.moveCursor(0, -1);
process.stdout.clearLine(1);

spinner.success({ text: "Benchmarking complete!" });
console.table({
...joshResults,
...brineResults,
...brineResultsSqlite,
...brineResultsMemory,
});
})();
const table = bench.table();
const finalTable: Record<string, string | number>[] = [];

console.log(`😃 Results for: ${name}\n`); // Add Average Time (ms) column based on "Average Time (ns)" column

for (const row of table) {
if (!row) continue;
if (typeof row["Average Time (ns)"] !== "number") continue;

finalTable.push({
...row,
"Average Time (ms)": (row["Average Time (ns)"] / 1000000).toFixed(3),
});
}

console.table(finalTable);
};

(async () => {
const login = {
user: "root",
password: "root",
database: "brinedb",
};

const sqlite_memory = new Brine(BrineDatabases.sqlite.memory);
const sqlite_file = new Brine(BrineDatabases.sqlite.file("benchmark.sqlite"));
const postgres = new Brine(BrineDatabases.postgres.build(login));
const mysql = new Brine(BrineDatabases.mysql.build(login));
const mariadb = new Brine(
BrineDatabases.mysql.build({
...login,
port: 3307,
}),
);

await benchme("SQLite (Memory)", sqlite_memory);
await benchme("SQLite (File)", sqlite_file);
await benchme("PostgreSQL", postgres);
await benchme("MySQL", mysql);
await benchme("MariaDB", mariadb);

console.log("✅ All benchmarks complete");
})().catch(console.error);
Loading

0 comments on commit abc0b42

Please sign in to comment.