Skip to content

Commit

Permalink
Merge pull request #1645 from tursodatabase/fix-fts5-crash
Browse files Browse the repository at this point in the history 
Fix fts5 crash
  • Loading branch information
@sivukhin
sivukhin authored Aug 8, 2024
2 parents 9120ce6 + 0d41105 commit e7de104
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 85 deletions.
61 changes: 34 additions & 27 deletions libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
** README.md
** configure
** configure.ac
** ext/fts5/fts5_tokenize.c
** ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java
** ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java
** ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java
Expand Down Expand Up @@ -259750,40 +259751,46 @@ static int fts5TriCreate(
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
TrigramTokenizer *pNew = 0;

if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
rc = SQLITE_ERROR;
}
}

if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}

if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
}
*ppOut = (Fts5Tokenizer*)pNew;
Expand Down
16 changes: 12 additions & 4 deletions libsql-ffi/bundled/bindings/bindgen.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ extern "C" {
extern "C" {
pub fn sqlite3_vmprintf(
arg1: *const ::std::os::raw::c_char,
arg2: va_list,
arg2: *mut __va_list_tag,
) -> *mut ::std::os::raw::c_char;
}
extern "C" {
Expand All @@ -956,7 +956,7 @@ extern "C" {
arg1: ::std::os::raw::c_int,
arg2: *mut ::std::os::raw::c_char,
arg3: *const ::std::os::raw::c_char,
arg4: va_list,
arg4: *mut __va_list_tag,
) -> *mut ::std::os::raw::c_char;
}
extern "C" {
Expand Down Expand Up @@ -2503,7 +2503,7 @@ extern "C" {
pub fn sqlite3_str_vappendf(
arg1: *mut sqlite3_str,
zFormat: *const ::std::os::raw::c_char,
arg2: va_list,
arg2: *mut __va_list_tag,
);
}
extern "C" {
Expand Down Expand Up @@ -3524,4 +3524,12 @@ extern "C" {
extern "C" {
pub static sqlite3_wal_manager: libsql_wal_manager;
}
pub type __builtin_va_list = *mut ::std::os::raw::c_char;
pub type __builtin_va_list = [__va_list_tag; 1usize];
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct __va_list_tag {
pub gp_offset: ::std::os::raw::c_uint,
pub fp_offset: ::std::os::raw::c_uint,
pub overflow_arg_area: *mut ::std::os::raw::c_void,
pub reg_save_area: *mut ::std::os::raw::c_void,
}
61 changes: 34 additions & 27 deletions libsql-ffi/bundled/src/sqlite3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
** README.md
** configure
** configure.ac
** ext/fts5/fts5_tokenize.c
** ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java
** ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java
** ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java
Expand Down Expand Up @@ -259750,40 +259751,46 @@ static int fts5TriCreate(
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
TrigramTokenizer *pNew = 0;

if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
rc = SQLITE_ERROR;
}
}

if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}

if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
}
*ppOut = (Fts5Tokenizer*)pNew;
Expand Down
60 changes: 33 additions & 27 deletions libsql-sqlite3/ext/fts5/fts5_tokenize.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1290,40 +1290,46 @@ static int fts5TriCreate(
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
TrigramTokenizer *pNew = 0;

if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
rc = SQLITE_ERROR;
}
}

if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}

if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
}
*ppOut = (Fts5Tokenizer*)pNew;
Expand Down
20 changes: 20 additions & 0 deletions libsql/tests/integration_tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,26 @@ async fn debug_print_row() {
);
}

#[tokio::test]
async fn fts5_invalid_tokenizer() {
let db = Database::open(":memory:").unwrap();
let conn = db.connect().unwrap();
assert!(conn
.execute(
"CREATE VIRTUAL TABLE t USING fts5(s, tokenize='trigram case_sensitive ')",
(),
)
.await
.is_err());
assert!(conn
.execute(
"CREATE VIRTUAL TABLE t USING fts5(s, tokenize='trigram remove_diacritics ')",
(),
)
.await
.is_err());
}

#[cfg(feature = "serde")]
#[tokio::test]
async fn deserialize_row() {
Expand Down

0 comments on commit e7de104

Please sign in to comment.