Skip to content

Commit

Permalink
Adds transactional benchmarks.
Browse files Browse the repository at this point in the history
  • Loading branch information
koculu committed Jul 24, 2022
1 parent edf7c23 commit 7444ef8
Show file tree
Hide file tree
Showing 8 changed files with 261 additions and 54 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ LSM Tree (Log-structured merge-tree) is the most popular data structure and it i
| int-int tree lazy WAL | 1198 ms | 2379 ms | 3831 ms | 15338 ms |
| string-string tree immediate WAL | 7872 ms | 16065 ms | 24220 ms | 90901 ms |
| string-string tree lazy WAL | 2556 ms | 5240 ms | 7934 ms | 29815 ms |
| RocksDb string-string | 8215 ms | 16146 ms | 23760 ms | 61547 ms |
| RocksDb string-string | 8215 ms | 16146 ms | 23760 ms | 72491 ms |

### Environment:
```
Expand Down Expand Up @@ -201,7 +201,7 @@ ZoneTree supports 3 way of doing transactions.
The following sample shows how to do the transactions with ZoneTree Fluent Transaction API.

```c#
using var zoneTree = new ZoneTreeFactory<int, int>()
using var zoneTree = new ZoneTreeFactory<int, int>()
// Additional stuff goes here
.OpenOrCreateTransactional();
using var transaction =
Expand Down
85 changes: 68 additions & 17 deletions src/Playground/Benchmark/BenchmarkGroups.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,75 @@ namespace Playground.Benchmark;

public static class BenchmarkGroups
{
public static void InsertBenchmark()
public static void InsertBenchmark1()
{
ZoneTree1.TestIntTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree1.TestIntTree(WriteAheadLogMode.Lazy, 10_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 10_000_000);
}

public static void InsertBenchmark2()
{
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 10_000_000);
}

public static void InsertBenchmark3()
{
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree3.TestInsertTransactionIntTree(WriteAheadLogMode.Lazy, 10_000_000);
}

ZoneTree2.TestStringTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree2.TestStringTree(WriteAheadLogMode.Lazy, 10_000_000);
public static void LoadAndIterateBenchmark1()
{
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree1.TestInsertIntTree(WriteAheadLogMode.Lazy, 10_000_000);
}

public static void LoadAndIterateBenchmark2()
{
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree2.TestInsertStringTree(WriteAheadLogMode.Lazy, 10_000_000);
}

public static void LoadAndIterateBenchmark3()
{
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Immediate, 1_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Immediate, 2_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Immediate, 3_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Immediate, 10_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Lazy, 1_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Lazy, 2_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Lazy, 3_000_000);
ZoneTree3.TestIterateIntTree(WriteAheadLogMode.Lazy, 10_000_000);
}
}
67 changes: 51 additions & 16 deletions src/Playground/Benchmark/ZoneTree1.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,19 @@ namespace Playground.Benchmark;

public class ZoneTree1
{
public static void TestIntTree(WriteAheadLogMode mode, int count)
public static void TestInsertIntTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestIntTree\r\n");
Console.WriteLine("\r\nTestIntTree\r\n");
Console.WriteLine("Record count = " + count);
Console.WriteLine("WriteAheadLogMode: = " + mode);

var dataPath = "../../data/TestIntTree" + mode + count;
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = new ZoneTreeFactory<int, int>()
.SetComparer(new Int32ComparerAscending())
.SetMutableSegmentMaxItemCount(1_000_000)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = mode;
x.EnableIncrementalBackup = false;
})
.SetKeySerializer(new Int32Serializer())
.SetValueSerializer(new Int32Serializer())
.OpenOrCreate();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<int, int>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;

Expand All @@ -47,4 +35,51 @@ public static void TestIntTree(WriteAheadLogMode mode, int count)
Console.WriteLine("Completed in: " + stopWatch.ElapsedMilliseconds);
basicMaintainer.CompleteRunningTasks().Wait();
}

public static void TestIterateIntTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestIterationIntTree\r\n");
Console.WriteLine("Record count = " + count);
Console.WriteLine("WriteAheadLogMode: = " + mode);

var dataPath = "../../data/TestIntTree" + mode + count;
var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<int, int>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;

Console.WriteLine("Loaded in: " + stopWatch.ElapsedMilliseconds);

var off = 0;
using var iterator = zoneTree.CreateIterator();
while (iterator.Next())
{
if (iterator.CurrentKey * 2 != iterator.CurrentValue)
throw new Exception("invalid key or value");
++off;
}
if (off != count)
throw new Exception($"missing records. {off} != {count}");

Console.WriteLine("Completed in: " + stopWatch.ElapsedMilliseconds);
basicMaintainer.CompleteRunningTasks().Wait();
}

private static IZoneTree<int, int> OpenOrCreateZoneTree(WriteAheadLogMode mode, string dataPath)
{
return new ZoneTreeFactory<int, int>()
.SetComparer(new Int32ComparerAscending())
.SetMutableSegmentMaxItemCount(1_000_000)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = mode;
x.EnableIncrementalBackup = false;
})
.SetKeySerializer(new Int32Serializer())
.SetValueSerializer(new Int32Serializer())
.OpenOrCreate();
}
}
64 changes: 50 additions & 14 deletions src/Playground/Benchmark/ZoneTree2.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace Playground.Benchmark;

public class ZoneTree2
{
public static void TestStringTree(WriteAheadLogMode mode, int count)
public static void TestInsertStringTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestStringTree\r\n");
Console.WriteLine("Record count = " + count);
Expand All @@ -21,19 +21,7 @@ public static void TestStringTree(WriteAheadLogMode mode, int count)

var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = new ZoneTreeFactory<string, string>()
.SetComparer(new StringOrdinalComparerAscending())
.SetMutableSegmentMaxItemCount(1_000_000)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = mode;
x.EnableIncrementalBackup = false;
})
.SetKeySerializer(new Utf8StringSerializer())
.SetValueSerializer(new Utf8StringSerializer())
.OpenOrCreate();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<string, string>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;
Console.WriteLine("Loaded in: " + stopWatch.ElapsedMilliseconds);
Expand All @@ -48,4 +36,52 @@ public static void TestStringTree(WriteAheadLogMode mode, int count)
basicMaintainer.CompleteRunningTasks().Wait();
Console.WriteLine("\r\n-------------------------\r\n");
}

public static void TestIterateStringTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestStringTree\r\n");
Console.WriteLine("Record count = " + count);
Console.WriteLine("WriteAheadLogMode: = " + mode);

var dataPath = "../../data/TestStringTree" + mode + count;

var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<string, string>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;
Console.WriteLine("Loaded in: " + stopWatch.ElapsedMilliseconds);

var off = 0;
using var iterator = zoneTree.CreateIterator();
while (iterator.Next())
{
if (iterator.CurrentKey != iterator.CurrentValue)
throw new Exception("invalid key or value");
++off;
}
if (off != count)
throw new Exception($"missing records. {off} != {count}");

Console.WriteLine("Completed in: " + stopWatch.ElapsedMilliseconds);
basicMaintainer.CompleteRunningTasks().Wait();
Console.WriteLine("\r\n-------------------------\r\n");
}

private static IZoneTree<string, string> OpenOrCreateZoneTree(WriteAheadLogMode mode, string dataPath)
{
return new ZoneTreeFactory<string, string>()
.SetComparer(new StringOrdinalComparerAscending())
.SetMutableSegmentMaxItemCount(1_000_000)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = mode;
x.EnableIncrementalBackup = false;
})
.SetKeySerializer(new Utf8StringSerializer())
.SetValueSerializer(new Utf8StringSerializer())
.OpenOrCreate();
}
}
85 changes: 85 additions & 0 deletions src/Playground/Benchmark/ZoneTree3.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
using System.Diagnostics;
using Tenray.ZoneTree;
using Tenray.ZoneTree.Comparers;
using Tenray.ZoneTree.Maintainers;
using Tenray.ZoneTree.Serializers;
using Tenray.ZoneTree.WAL;

namespace Playground.Benchmark;

public class ZoneTree3
{
public static void TestInsertTransactionIntTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestIntTree\r\n");
Console.WriteLine("Record count = " + count);
Console.WriteLine("WriteAheadLogMode: = " + mode);

var dataPath = "../../data/TestInsertTransactionIntTree" + mode + count;
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);

var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<int, int>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;

Console.WriteLine("Loaded in: " + stopWatch.ElapsedMilliseconds);

Parallel.For(0, count, (x) =>
{
zoneTree.UpsertAutoCommit(x, x + x);
});

Console.WriteLine("Completed in: " + stopWatch.ElapsedMilliseconds);
basicMaintainer.CompleteRunningTasks().Wait();
}

public static void TestIterateIntTree(WriteAheadLogMode mode, int count)
{
Console.WriteLine("\r\nTestIterationIntTree\r\n");
Console.WriteLine("Record count = " + count);
Console.WriteLine("WriteAheadLogMode: = " + mode);

var dataPath = "../../data/TestInsertTransactionIntTree" + mode + count;
var stopWatch = new Stopwatch();
stopWatch.Start();
using var zoneTree = OpenOrCreateZoneTree(mode, dataPath);
using var basicMaintainer = new BasicZoneTreeMaintainer<int, int>(zoneTree);
basicMaintainer.ThresholdForMergeOperationStart = 2_000_000;

Console.WriteLine("Loaded in: " + stopWatch.ElapsedMilliseconds);

var off = 0;
using var iterator = zoneTree.Maintenance.ZoneTree.CreateIterator();
while (iterator.Next())
{
if (iterator.CurrentKey * 2 != iterator.CurrentValue)
throw new Exception("invalid key or value");
++off;
}
if (off != count)
throw new Exception($"missing records. {off} != {count}");

Console.WriteLine("Completed in: " + stopWatch.ElapsedMilliseconds);
basicMaintainer.CompleteRunningTasks().Wait();
}

private static ITransactionalZoneTree<int, int> OpenOrCreateZoneTree(WriteAheadLogMode mode, string dataPath)
{
return new ZoneTreeFactory<int, int>()
.SetComparer(new Int32ComparerAscending())
.SetMutableSegmentMaxItemCount(1_000_000)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = mode;
x.EnableIncrementalBackup = false;
})
.SetKeySerializer(new Int32Serializer())
.SetValueSerializer(new Int32Serializer())
.OpenOrCreateTransactional();
}
}
2 changes: 1 addition & 1 deletion src/Playground/Program.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
using Playground.Benchmark;

BenchmarkGroups.InsertBenchmark();
BenchmarkGroups.InsertBenchmark1();
4 changes: 2 additions & 2 deletions src/Playground/Test1.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace Playground;

public class Test1
{
public void Run()
public static void Run()
{
var dataPath = "../../data/SeveralParallelTransactions";
var stopWatch = new Stopwatch();
Expand All @@ -21,7 +21,7 @@ public void Run()
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogProvider(x =>
{
x.WriteAheadLogMode = WriteAheadLogMode.Lazy;
x.WriteAheadLogMode = WriteAheadLogMode.Immediate;
x.EnableIncrementalBackup = true;
})
.SetKeySerializer(new Int32Serializer())
Expand Down
Loading

0 comments on commit 7444ef8

Please sign in to comment.