From 4abe093e0b839550047a1c6e2596539c47b774c5 Mon Sep 17 00:00:00 2001 From: flower-of-the-bridges Date: Mon, 30 Dec 2024 10:53:47 +0100 Subject: [PATCH 1/2] fix: partial index are updated during startup --- lib/createIndexes.js | 24 ++++--- tests/createIndexes.test.js | 132 ++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 8 deletions(-) diff --git a/lib/createIndexes.js b/lib/createIndexes.js index 03aa2ee5..075998a8 100644 --- a/lib/createIndexes.js +++ b/lib/createIndexes.js @@ -26,6 +26,7 @@ const { GEO_INDEX, TEXT_INDEX, } = require('./consts') +const { isEqual } = require('lodash') const addKeyToSpec = (specObject, { name, order }) => ({ ...specObject, [name]: order }) const addTextKeyToSpec = (specObject, { name }) => ({ ...specObject, [name]: TEXT_FIELD }) @@ -115,24 +116,31 @@ function getIndexOptions(index) { options = { ...options, language_override: index.languageOverride } } if (index.usePartialFilter) { - let partialFilterExpression - try { - partialFilterExpression = index.partialFilterExpression ? JSON.parse(index.partialFilterExpression) : {} - } catch (error) { - throw new Error(`Impossible to parse the Partial Index expression of index ${index.name}`, { catch: error }) - } - + const partialFilterExpression = parsePartialIndex(index) options = { ...options, partialFilterExpression } } return options } +function parsePartialIndex(index) { + try { + return index.partialFilterExpression ? JSON.parse(index.partialFilterExpression) : {} + } catch (error) { + throw new Error(`Impossible to parse the Partial Index expression of index ${index.name}`, { catch: error }) + } +} + function checkIndexEquality(foundIndex, index) { - const isOptionUniqueEqual = index.unique ? foundIndex.unique === true : foundIndex.unique === undefined + const isOptionUniqueEqual = Boolean(index.unique) === Boolean(foundIndex.unique) const isOptionExpireAfterSecondsEqual = foundIndex.expireAfterSeconds === index.expireAfterSeconds + const isOptionPartialFilterEqual = isEqual( + foundIndex.partialFilterExpression || {}, + parsePartialIndex(index) + ) return ( isOptionUniqueEqual && isOptionExpireAfterSecondsEqual + && isOptionPartialFilterEqual && checkIndexEqualityByType(foundIndex, index) ) } diff --git a/tests/createIndexes.test.js b/tests/createIndexes.test.js index 83585bc0..9d2e76d7 100644 --- a/tests/createIndexes.test.js +++ b/tests/createIndexes.test.js @@ -1050,6 +1050,138 @@ tap.test('createIndexes', async t => { ], expectedIndexCreatedCount: 1, }, + { + name: 'partial index with already present one (but not partial)', + alreadyPresentIndexes: [ + { + spec: { + name: 1, + }, + options: { + name: nameIndex1, + unique: true, + }, + }, + ], + indexes: [ + { + name: nameIndex1, + type: NORMAL_INDEX, + unique: true, + fields: [ + { name: 'name', + order: 1, + }, + ], + usePartialFilter: true, + partialFilterExpression: '{ "name": { "$eq": "test" } }', + }, + ], + expectedIndexes: [ + { + v: 2, + key: { + name: 1, + }, + name: nameIndex1, + background: true, + unique: true, + partialFilterExpression: { name: { $eq: 'test' } }, + }, + ], + expectedIndexCreatedCount: 1, + }, + { + name: 'complex partial index not replaced', + alreadyPresentIndexes: [ + { + spec: { b: 1, d: 1 }, + options: { + name: nameIndex1, + unique: true, + partialFilterExpression: { $or: [{ b: 'c' }, { d: 'e' }] }, + }, + }, + ], + indexes: [ + { + name: nameIndex1, + type: NORMAL_INDEX, + unique: true, + fields: [ + { + name: 'b', + order: 1, + }, + { + name: 'd', + order: 1, + }, + ], + usePartialFilter: true, + partialFilterExpression: '{"$or":[{"b":"c"},{"d":"e"}]}', + }, + ], + expectedIndexes: [ + { + v: 2, + key: { + b: 1, + d: 1, + }, + name: nameIndex1, + unique: true, + partialFilterExpression: { $or: [{ b: 'c' }, { d: 'e' }] }, + }, + ], + expectedIndexCreatedCount: 0, + }, + { + name: 'complex partial replaced', + alreadyPresentIndexes: [ + { + spec: { b: 1, d: 1 }, + options: { + name: nameIndex1, + unique: true, + partialFilterExpression: { $or: [{ b: 'c' }, { d: 'e' }] }, + }, + }, + ], + indexes: [ + { + name: nameIndex1, + type: NORMAL_INDEX, + unique: true, + fields: [ + { + name: 'b', + order: 1, + }, + { + name: 'd', + order: 1, + }, + ], + usePartialFilter: true, + partialFilterExpression: '{"$or":[{"b":"c"},{"d":"g"}]}', + }, + ], + expectedIndexes: [ + { + v: 2, + key: { + b: 1, + d: 1, + }, + background: true, + name: nameIndex1, + unique: true, + partialFilterExpression: { $or: [{ b: 'c' }, { d: 'g' }] }, + }, + ], + expectedIndexCreatedCount: 1, + }, ] t.plan(testConfigs.length) From e5f65c02fec0ba304697b7ee5fb76039ed286fd3 Mon Sep 17 00:00:00 2001 From: flower-of-the-bridges Date: Mon, 30 Dec 2024 19:48:08 +0100 Subject: [PATCH 2/2] chore: add mongodb version to index tests --- tests/createIndexes.test.js | 62 +++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/tests/createIndexes.test.js b/tests/createIndexes.test.js index 9d2e76d7..e1b20921 100644 --- a/tests/createIndexes.test.js +++ b/tests/createIndexes.test.js @@ -1093,6 +1093,8 @@ tap.test('createIndexes', async t => { }, { name: 'complex partial index not replaced', + // nested operators in partial indexes are not supported on mongodb:5.0 + minMongoDbVersion: '6.0', alreadyPresentIndexes: [ { spec: { b: 1, d: 1 }, @@ -1138,6 +1140,8 @@ tap.test('createIndexes', async t => { }, { name: 'complex partial replaced', + // nested operators in partial indexes are not supported on mongodb:5.0 + minMongoDbVersion: '6.0', alreadyPresentIndexes: [ { spec: { b: 1, d: 1 }, @@ -1184,38 +1188,50 @@ tap.test('createIndexes', async t => { }, ] - t.plan(testConfigs.length) + const MIN_MONGODB_VERSION = '5.0' + const MONGODB_VERSION = process.env.MONGO_VERSION || MIN_MONGODB_VERSION + // RUN TESTS ACCORDING TO MONGO_VERSION env var, since + // nested operators in partial indexes are not supported on mongodb:5.0 + t.plan( + testConfigs + .filter(({ minMongoDbVersion = MIN_MONGODB_VERSION }) => MONGODB_VERSION >= minMongoDbVersion) + .length + ) testConfigs.forEach(({ name, + minMongoDbVersion = MIN_MONGODB_VERSION, alreadyPresentIndexes, indexes, expectedIndexes, expectedIndexCreatedCount, }) => { - t.test(name, async t => { - t.plan(2 + expectedIndexes.length) - await collection.drop() - await database.createCollection(BOOKS_COLLECTION_NAME) - await Promise.all(alreadyPresentIndexes - .map(({ spec, options }) => collection.createIndex(spec, options))) - const createdIndexNames = await createIndexes(collection, indexes, 'preserve_') - const retIndexes = await collection.indexes() + if (MONGODB_VERSION >= minMongoDbVersion) { + t.test(name, async t => { + t.plan(2 + expectedIndexes.length) + await collection.drop() + await database.createCollection(BOOKS_COLLECTION_NAME) - /* Starting in MongoDB 4.4 the `ns` field is no more included in index objects. - * Since we are not using `ns` field in this service we remove it in the below line. - * In this way we guarantee compatibility with MongoDB 4.4 and the previous MongoDB versions. - * https://docs.mongodb.com/manual/reference/method/db.collection.getIndexes/ - */ - retIndexes.forEach(index => delete index.ns) - // sort by name because the return order is not deterministic with background indexes - // skip the first returned index as it is always the _id one - t.strictSame(retIndexes.slice(1).sort(by('name')), expectedIndexes.sort(by('name'))) - retIndexes.slice(1).forEach(retIndex => { - const index = expectedIndexes.find(expectedIndex => expectedIndex.name === retIndex.name) - t.ok(JSON.stringify(index.key) === JSON.stringify(retIndex.key)) + await Promise.all(alreadyPresentIndexes + .map(({ spec, options }) => collection.createIndex(spec, options))) + const createdIndexNames = await createIndexes(collection, indexes, 'preserve_') + const retIndexes = await collection.indexes() + + /* Starting in MongoDB 4.4 the `ns` field is no more included in index objects. + * Since we are not using `ns` field in this service we remove it in the below line. + * In this way we guarantee compatibility with MongoDB 4.4 and the previous MongoDB versions. + * https://docs.mongodb.com/manual/reference/method/db.collection.getIndexes/ + */ + retIndexes.forEach(index => delete index.ns) + // sort by name because the return order is not deterministic with background indexes + // skip the first returned index as it is always the _id one + t.strictSame(retIndexes.slice(1).sort(by('name')), expectedIndexes.sort(by('name'))) + retIndexes.slice(1).forEach(retIndex => { + const index = expectedIndexes.find(expectedIndex => expectedIndex.name === retIndex.name) + t.ok(JSON.stringify(index.key) === JSON.stringify(retIndex.key)) + }) + t.equal(createdIndexNames.length, expectedIndexCreatedCount) }) - t.equal(createdIndexNames.length, expectedIndexCreatedCount) - }) + } }) })