Add Bpe.cs (#20)

nietras · Jul 16, 2024 · 2e9a686 · 2e9a686
1 parent 25b1bb2
commit 2e9a686
Show file tree

Hide file tree

Showing 14 changed files with 51,455 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,9 @@
 *.userprefs
 
 # Build results
-artifacts/
+artifacts/*
+# Include gpt2 tokenizer vocabulary
+!artifacts/gpt2.tiktoken
 Build_*/
 [Dd]ebug/
 [Dd]ebugPublic/