From a09b68207928dee4f87d444f7b1b876fb3dead37 Mon Sep 17 00:00:00 2001 From: Thorsten Marx Date: Fri, 15 Sep 2023 12:30:37 +0200 Subject: [PATCH] #5 #10 #11 updated implementations --- .../ElasticsearchIndexAdapter.java | 70 +++--- .../ElasticsearchIndexConfiguration.java | 25 +-- .../index/LuceneFieldConfiguration.java | 2 +- .../index/LuceneIndexConfiguration.java | 16 +- .../opensearch/OpensearchIndexAdapter.java | 83 +++---- .../OpensearchIndexConfiguration.java | 25 +-- adapters/solr/pom.xml | 2 +- .../adapters/solr/SolrIndexAdapter.java | 202 ++++++++++++++++++ .../adapters/solr/SolrIndexConfiguration.java | 39 ++++ .../adapters/solr/AbstractContainerTest.java | 32 ++- .../adapters/solr/SolrMongoSearcherTest.java | 126 ++++++++--- .../configuration/FieldConfiguration.java | 2 +- .../configuration/IndexConfiguration.java | 22 +- 13 files changed, 476 insertions(+), 170 deletions(-) create mode 100644 adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexAdapter.java create mode 100644 adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexConfiguration.java diff --git a/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexAdapter.java b/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexAdapter.java index ba64339..fb539ed 100644 --- a/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexAdapter.java +++ b/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexAdapter.java @@ -65,9 +65,14 @@ public ElasticsearchIndexAdapter(final ElasticsearchIndexConfiguration configura @Override public void indexDocument(String database, String collection, Document document) throws IOException { + InsertMessage message = new InsertMessage(document.getObjectId("_id").toString(), database, collection, document); + index(message); } + @Override public void clearCollection(String database, String collection) throws IOException { + DropCollectionMessage message = new DropCollectionMessage(database, collection); + dropCollection(message); } public void commit() { @@ -105,7 +110,7 @@ public void update() { } else if (message instanceof DeleteMessage deleteCommand) { delete(deleteCommand); } else if (message instanceof UpdateMessage updateCommand) { - ElasticsearchIndexAdapter.this.update(updateCommand); + updateDocument(updateCommand); } else if (message instanceof DropCollectionMessage dropCollectionCommand) { dropCollection(dropCollectionCommand); } @@ -118,20 +123,30 @@ public void update() { queueWorkerThread.start(); } + private Map createDocument(final String database, final String collection, final Document document) { + Map indexDocument = new HashMap<>(); + indexDocument.put("_database", database); + indexDocument.put("_collection", collection); + if (configuration.hasFieldConfigurations(collection)) { + var fieldConfigs = configuration.getFieldConfigurations(collection); + fieldConfigs.forEach((fc) -> { + var value = fc.getMapper().getFieldValue(fc.getFieldName(), document); + indexDocument.put(fc.getIndexFieldName(), value); + }); + } + + if (configuration.getDocumentExtender() != null) { + configuration.getDocumentExtender().accept(document, indexDocument); + } + + return indexDocument; + } + private void index(InsertMessage command) { try { - Map document = new HashMap<>(); - document.put("_database", command.database()); - document.put("_collection", command.collection()); - if (configuration.hasFieldConfigurations(command.collection())) { - var fieldConfigs = configuration.getFieldConfigurations(command.collection()); - fieldConfigs.forEach((fc) -> { - var value = fc.getMapper().getFieldValue(fc.getFieldName(), command.document()); - document.put(fc.getIndexFieldName(), value); - }); - } - + Map document = createDocument(command.database(), command.collection(), command.document()); + IndexResponse response = esClient.index(i -> i .index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) .id(command.uid()) @@ -141,19 +156,10 @@ private void index(InsertMessage command) { } } - private void update(UpdateMessage command) { + private void updateDocument(UpdateMessage command) { try { - Map document = new HashMap<>(); - document.put("_database", command.database()); - document.put("_collection", command.collection()); - if (configuration.hasFieldConfigurations(command.collection())) { - var fieldConfigs = configuration.getFieldConfigurations(command.collection()); - fieldConfigs.forEach((fc) -> { - var value = fc.getMapper().getFieldValue(fc.getFieldName(), command.document()); - document.put(fc.getIndexFieldName(), value); - }); - } + Map document = createDocument(command.database(), command.collection(), command.document()); esClient.update( u -> u.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) @@ -178,23 +184,23 @@ private void delete(DeleteMessage command) { /** * In elasticsearch the index is not dropped, we just delete all documents - * + * * @param command */ private void dropCollection(DropCollectionMessage command) { try { boolean exists = esClient.indices().exists(fn -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection()))).value(); - + if (exists) { //esClient.indices().delete(fn -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection()))); - esClient.deleteByQuery(fn -> - fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) - .query(qb -> - qb.match(t -> - t.field("_collection").query(command.collection()) - ) - ) + esClient.deleteByQuery(fn + -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) + .query(qb + -> qb.match(t + -> t.field("_collection").query(command.collection()) + ) + ) ); } } catch (IOException | ElasticsearchException ex) { diff --git a/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexConfiguration.java b/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexConfiguration.java index 9b351df..40114e5 100644 --- a/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexConfiguration.java +++ b/adapters/elasticsearch/src/main/java/com/github/thmarx/mongo/search/adapters/elasticsearch/ElasticsearchIndexConfiguration.java @@ -26,32 +26,13 @@ import com.github.thmarx.mongo.search.index.configuration.FieldConfiguration; import com.github.thmarx.mongo.search.index.configuration.IndexConfiguration; -import com.github.thmarx.mongo.search.index.utils.MultiMap; -import java.util.Collection; -import java.util.function.BiFunction; -import lombok.Getter; -import lombok.Setter; +import java.util.Map; +import org.bson.Document; /** * * @author t.marx */ -public class ElasticsearchIndexConfiguration extends IndexConfiguration { - final MultiMap fieldConfigurations = new MultiMap<>(); +public class ElasticsearchIndexConfiguration extends IndexConfiguration, FieldConfiguration> { - @Getter - @Setter - private BiFunction indexNameMapper = (database, collection) -> collection; - - public ElasticsearchIndexConfiguration addFieldConfiguration (final String collection, final FieldConfiguration fieldConfig) { - fieldConfigurations.put(collection, fieldConfig); - return this; - } - - public Collection getFieldConfigurations (final String collection) { - return fieldConfigurations.get(collection); - } - public boolean hasFieldConfigurations (final String collection) { - return fieldConfigurations.containsKey(collection); - } } diff --git a/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneFieldConfiguration.java b/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneFieldConfiguration.java index 1651ac9..2572878 100644 --- a/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneFieldConfiguration.java +++ b/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneFieldConfiguration.java @@ -34,7 +34,7 @@ @SuperBuilder @Getter -public class LuceneFieldConfiguration extends FieldConfiguration { +public class LuceneFieldConfiguration extends FieldConfiguration { /** * Field value is stored. diff --git a/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneIndexConfiguration.java b/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneIndexConfiguration.java index da9736b..c6226bf 100644 --- a/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneIndexConfiguration.java +++ b/adapters/lucene/src/main/java/com/github/thmarx/mongo/search/adapters/lucene/index/LuceneIndexConfiguration.java @@ -38,7 +38,7 @@ * * @author t.marx */ -public class LuceneIndexConfiguration extends IndexConfiguration { +public class LuceneIndexConfiguration extends IndexConfiguration { Storage storage; Analyzer analyzer; @@ -47,24 +47,12 @@ public class LuceneIndexConfiguration extends IndexConfiguration fieldConfigurations = new MultiMap<>(); + @Getter @Setter DateTimeFormatter defaultDateFormatter = DateTimeFormatter.ISO_LOCAL_DATE; - public LuceneIndexConfiguration addFieldConfiguration (final String collection, final LuceneFieldConfiguration fieldConfig) { - fieldConfigurations.put(collection, fieldConfig); - return this; - } - - public Collection getFieldConfigurations (final String collection) { - return fieldConfigurations.get(collection); - } - public boolean hasFieldConfigurations (final String collection) { - return fieldConfigurations.containsKey(collection); - } - public LuceneIndexConfiguration setStorage (final Storage storage) { this.storage = storage; return this; diff --git a/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexAdapter.java b/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexAdapter.java index 75835a8..f012391 100644 --- a/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexAdapter.java +++ b/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexAdapter.java @@ -62,12 +62,14 @@ public OpensearchIndexAdapter(final OpensearchIndexConfiguration configuration) @Override public void indexDocument(String database, String collection, Document document) throws IOException { - + InsertMessage message = new InsertMessage(document.getObjectId("_id").toString(), database, collection, document); + index(message); } @Override public void clearCollection(String database, String collection) throws IOException { - + DropCollectionMessage message = new DropCollectionMessage(database, collection); + dropCollection(message); } public void commit() { @@ -105,7 +107,7 @@ public void update() { } else if (command instanceof DeleteMessage deleteCommand) { delete(deleteCommand); } else if (command instanceof UpdateMessage updateCommand) { - OpensearchIndexAdapter.this.update(updateCommand); + updateDocument(updateCommand); } else if (command instanceof DropCollectionMessage dropCollectionCommand) { dropCollection(dropCollectionCommand); } @@ -118,19 +120,29 @@ public void update() { queueWorkerThread.start(); } + private Map createDocument(final String database, final String collection, final Document document) { + Map indexDocument = new HashMap<>(); + indexDocument.put("_database", database); + indexDocument.put("_collection", collection); + if (configuration.hasFieldConfigurations(collection)) { + var fieldConfigs = configuration.getFieldConfigurations(collection); + fieldConfigs.forEach((fc) -> { + var value = fc.getMapper().getFieldValue(fc.getFieldName(), document); + indexDocument.put(fc.getIndexFieldName(), value); + }); + } + + if (configuration.getDocumentExtender() != null) { + configuration.getDocumentExtender().accept(document, indexDocument); + } + + return indexDocument; + } + private void index(InsertMessage command) { try { - Map document = new HashMap<>(); - document.put("_database", command.database()); - document.put("_collection", command.collection()); - if (configuration.hasFieldConfigurations(command.collection())) { - var fieldConfigs = configuration.getFieldConfigurations(command.collection()); - fieldConfigs.forEach((fc) -> { - var value = fc.getMapper().getFieldValue(fc.getFieldName(), command.document()); - document.put(fc.getIndexFieldName(), value); - }); - } + Map document = createDocument(command.database(), command.collection(), command.document()); IndexResponse response = osClient.index(i -> i .index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) @@ -141,26 +153,16 @@ private void index(InsertMessage command) { Logger.getLogger(OpensearchIndexAdapter.class.getName()).log(Level.SEVERE, null, ex); } } - - private void update(UpdateMessage command) { - try { - Map document = new HashMap<>(); - document.put("_database", command.database()); - document.put("_collection", command.collection()); - if (configuration.hasFieldConfigurations(command.collection())) { - var fieldConfigs = configuration.getFieldConfigurations(command.collection()); - fieldConfigs.forEach((fc) -> { - var value = fc.getMapper().getFieldValue(fc.getFieldName(), command.document()); - document.put(fc.getIndexFieldName(), value); - }); - } + private void updateDocument(UpdateMessage command) { + try { + Map document = createDocument(command.database(), command.collection(), command.document()); - osClient.update(u -> - u.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) - .id(command.uid()) - .doc(document) - , Map.class + osClient.update(u + -> u.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) + .id(command.uid()) + .doc(document), + Map.class ); } catch (IOException ex) { Logger.getLogger(OpensearchIndexAdapter.class.getName()).log(Level.SEVERE, null, ex); @@ -178,25 +180,26 @@ private void delete(DeleteMessage command) { Logger.getLogger(OpensearchIndexAdapter.class.getName()).log(Level.SEVERE, null, ex); } } - + /** * For OpenSearch the collection will not be dropped! + * * @param command */ private void dropCollection(DropCollectionMessage command) { try { boolean exists = osClient.indices().exists(fn -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection()))).value(); - + if (exists) { //osClient.indices().delete(fn -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection()))); - osClient.deleteByQuery(fn -> - fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) - .query(qb -> - qb.match(t -> - t.field("_collection").query(FieldValue.of(command.collection())) - ) - ) + osClient.deleteByQuery(fn + -> fn.index(configuration.getIndexNameMapper().apply(command.database(), command.collection())) + .query(qb + -> qb.match(t + -> t.field("_collection").query(FieldValue.of(command.collection())) + ) + ) ); } } catch (IOException ex) { diff --git a/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexConfiguration.java b/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexConfiguration.java index 97307cc..76a63a1 100644 --- a/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexConfiguration.java +++ b/adapters/opensearch/src/main/java/com/github/thmarx/mongo/search/adapters/opensearch/OpensearchIndexConfiguration.java @@ -26,32 +26,13 @@ import com.github.thmarx.mongo.search.index.configuration.FieldConfiguration; import com.github.thmarx.mongo.search.index.configuration.IndexConfiguration; -import com.github.thmarx.mongo.search.index.utils.MultiMap; -import java.util.Collection; -import java.util.function.BiFunction; -import lombok.Getter; -import lombok.Setter; +import java.util.Map; +import org.bson.Document; /** * * @author t.marx */ -public class OpensearchIndexConfiguration extends IndexConfiguration { - final MultiMap fieldConfigurations = new MultiMap<>(); +public class OpensearchIndexConfiguration extends IndexConfiguration, FieldConfiguration> { - @Getter - @Setter - private BiFunction indexNameMapper = (database, collection) -> collection; - - public OpensearchIndexConfiguration addFieldConfiguration (final String collection, final FieldConfiguration fieldConfig) { - fieldConfigurations.put(collection, fieldConfig); - return this; - } - - public Collection getFieldConfigurations (final String collection) { - return fieldConfigurations.get(collection); - } - public boolean hasFieldConfigurations (final String collection) { - return fieldConfigurations.containsKey(collection); - } } diff --git a/adapters/solr/pom.xml b/adapters/solr/pom.xml index 85fe78f..8e6186b 100644 --- a/adapters/solr/pom.xml +++ b/adapters/solr/pom.xml @@ -8,7 +8,7 @@ mongo-search-adapters 0.1 - monog-search-adapters-solr + mongo-search-adapters-solr jar 9.3.0 diff --git a/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexAdapter.java b/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexAdapter.java new file mode 100644 index 0000000..df385ab --- /dev/null +++ b/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexAdapter.java @@ -0,0 +1,202 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package com.github.thmarx.mongo.search.adapters.solr; + +/*- + * #%L + * monog-search-adapters-solr + * %% + * Copyright (C) 2023 Marx-Software + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +import com.github.thmarx.mongo.search.adapter.AbstractIndexAdapter; +import com.github.thmarx.mongo.search.index.messages.Message; +import com.github.thmarx.mongo.search.index.messages.DeleteMessage; +import com.github.thmarx.mongo.search.index.messages.DropCollectionMessage; +import com.github.thmarx.mongo.search.index.messages.InsertMessage; +import com.github.thmarx.mongo.search.index.messages.UpdateMessage; +import com.github.thmarx.mongo.search.index.utils.PausableThread; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; +import lombok.extern.slf4j.Slf4j; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.bson.Document; + +/** + * + * @author t.marx + */ +@Slf4j +public class SolrIndexAdapter extends AbstractIndexAdapter { + + private Thread queueWorkerThread; + + private PausableThread queueWorker; + + SolrClient indexClient; + + public SolrIndexAdapter(final SolrIndexConfiguration configuration) { + super(configuration); + + } + + @Override + public void indexDocument(String database, String collection, Document document) throws IOException { + } + + @Override + public void clearCollection(String database, String collection) throws IOException { + } + + public void commit() { + try { + indexClient.commit(); + } catch (SolrServerException | IOException ex) { + log.error("", ex); + } + } + + @Override + public void close() throws Exception { + queueWorker.stop(); + queueWorkerThread.interrupt(); + } + + @Override + public void startQueueWorker() { + this.queueWorker.unpause(); + } + + @Override + public void pauseQueueWorker() { + this.queueWorker.pause(); + } + + public void open(SolrClient indexClient) throws IOException { + + this.indexClient = indexClient; + + queueWorker = new PausableThread(true) { + @Override + public void update() { + try { + Message message = getMessageQueue().take(); + + if (message instanceof InsertMessage indexCommand) { + index(indexCommand); + } else if (message instanceof DeleteMessage deleteCommand) { + delete(deleteCommand); + } else if (message instanceof UpdateMessage updateCommand) { + updateDocument(updateCommand); + } else if (message instanceof DropCollectionMessage dropCollectionCommand) { + dropCollection(dropCollectionCommand); + } + + } catch (Exception ex) { + log.error("", ex); + } + } + }; + queueWorkerThread = new Thread(queueWorker); + queueWorkerThread.start(); + } + + private SolrInputDocument createDocument(final String uuid, final String database, final String collection, final Document document) { + SolrInputDocument indexDocument = new SolrInputDocument(); + + indexDocument.addField("id", uuid); + indexDocument.addField("_database", database); + indexDocument.addField("_collection", collection); + if (configuration.hasFieldConfigurations(collection)) { + var fieldConfigs = configuration.getFieldConfigurations(collection); + fieldConfigs.forEach((fc) -> { + var value = fc.getMapper().getFieldValue(fc.getFieldName(), document); + indexDocument.addField(fc.getIndexFieldName(), value); + }); + } + + if (configuration.getDocumentExtender() != null) { + configuration.getDocumentExtender().accept(document, indexDocument); + } + + return indexDocument; + } + + private void index(InsertMessage command) { + try { + + SolrInputDocument document = createDocument(command.uid(), command.database(), command.collection(), command.document()); + + indexClient.add( + configuration.getIndexNameMapper().apply(command.database(), command.collection()) + , document, 1000); + } catch (IOException | SolrServerException ex) { + log.error("error " ,ex); + } + } + + private void updateDocument(UpdateMessage command) { + try { + + SolrInputDocument document = createDocument(command.uid(), command.database(), command.collection(), command.document()); + + indexClient.add( + configuration.getIndexNameMapper().apply(command.database(), command.collection()) + , document, 1000); + } catch (IOException | SolrServerException ex) { + log.error("error " ,ex); + } + } + + private void delete(DeleteMessage command) { + try { + + indexClient.deleteById( + configuration.getIndexNameMapper().apply(command.database(), command.collection()), + command.uid()); + } catch (IOException | SolrServerException ex) { + log.error("error " ,ex); + } + } + + /** + * In elasticsearch the index is not dropped, we just delete all documents + * + * @param command + */ + private void dropCollection(DropCollectionMessage command) { + try { + + String query = "_database : '%s' AND _collection: '%s' ".formatted(command.database(), command.collection()); + + indexClient.deleteByQuery( + configuration.getIndexNameMapper().apply(command.database(), command.collection()), + query); + } catch (IOException | SolrServerException ex) { + log.error("error " ,ex); + } + } +} diff --git a/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexConfiguration.java b/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexConfiguration.java new file mode 100644 index 0000000..760fc97 --- /dev/null +++ b/adapters/solr/src/main/java/com/github/thmarx/mongo/search/adapters/solr/SolrIndexConfiguration.java @@ -0,0 +1,39 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package com.github.thmarx.mongo.search.adapters.solr; + +/*- + * #%L + * monog-search-adapters-solr + * %% + * Copyright (C) 2023 Marx-Software + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.github.thmarx.mongo.search.index.configuration.FieldConfiguration; +import com.github.thmarx.mongo.search.index.configuration.IndexConfiguration; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.bson.Document; + +/** + * + * @author t.marx + */ +public class SolrIndexConfiguration extends IndexConfiguration { + +} diff --git a/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/AbstractContainerTest.java b/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/AbstractContainerTest.java index 3a9ce68..823d41b 100644 --- a/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/AbstractContainerTest.java +++ b/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/AbstractContainerTest.java @@ -24,10 +24,18 @@ * #L% */ +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import java.io.IOException; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.testcontainers.containers.MongoDBContainer; import org.testcontainers.containers.SolrContainer; import org.testcontainers.utility.DockerImageName; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeTest; /** @@ -39,7 +47,13 @@ public class AbstractContainerTest { protected SolrContainer solrContainer; protected MongoDBContainer mongdbContainer; - @BeforeTest + protected MongoClient mongoClient; + protected SolrClient solrClient; + protected MongoDatabase database; + + + + @BeforeClass public void up() { solrContainer = new SolrContainer(DockerImageName.parse( "solr:9.3" @@ -50,12 +64,22 @@ public void up() { "mongo:6.0.9" )); mongdbContainer.start(); + + mongoClient = MongoClients.create(mongdbContainer.getConnectionString()); + database = mongoClient.getDatabase("search"); + + // Do whatever you want with the client ... + solrClient = new Http2SolrClient.Builder( + "http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr" + ).build(); } - @AfterTest - public void down() { - solrContainer.stop(); + @AfterClass + public void down() throws IOException { + mongoClient.close(); + solrClient.close(); + solrContainer.stop(); mongdbContainer.stop(); } } diff --git a/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/SolrMongoSearcherTest.java b/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/SolrMongoSearcherTest.java index f7b0301..8dc56dc 100644 --- a/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/SolrMongoSearcherTest.java +++ b/adapters/solr/src/test/java/com/github/thmarx/mongo/search/adapters/solr/SolrMongoSearcherTest.java @@ -23,18 +23,31 @@ import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import com.github.thmarx.mongo.search.index.MongoSearch; +import com.github.thmarx.mongo.search.index.configuration.FieldConfiguration; +import com.github.thmarx.mongo.search.mapper.FieldMappers; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; import java.io.IOException; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.NamedList; +import org.assertj.core.api.Assertions; +import org.awaitility.Awaitility; +import org.bson.Document; +import org.bson.types.ObjectId; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -47,60 +60,109 @@ */ public class SolrMongoSearcherTest extends AbstractContainerTest { - MongoClient client; - MongoSearch mongoSearch; - private MongoDatabase database; + SolrIndexAdapter adapter; private final static String COLLECTION_DOKUMENTE = "dokumente"; - SolrClient solrClient; - - @BeforeClass - public void setup() throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - - client = MongoClients.create(mongdbContainer.getConnectionString()); - database = client.getDatabase("search"); - - // Do whatever you want with the client ... - solrClient = new Http2SolrClient.Builder( - "http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr" - ).build(); - } - - @AfterClass - public void shutdown() throws Exception { - client.close(); - } - @BeforeMethod - public void cleanup() throws IOException { + public void beforeMethod() throws IOException, SolrServerException { + if (database.getCollection(COLLECTION_DOKUMENTE) != null) { + database.getCollection(COLLECTION_DOKUMENTE).drop(); + } + database.createCollection(COLLECTION_DOKUMENTE); + + SolrIndexConfiguration indexConfiguration = new SolrIndexConfiguration(); + indexConfiguration.addFieldConfiguration(COLLECTION_DOKUMENTE, + FieldConfiguration.builder() + .fieldName("name") + .indexFieldName("name") + .mapper(FieldMappers::toString) + .build()); + + adapter = new SolrIndexAdapter(indexConfiguration); + adapter.open(solrClient); + + mongoSearch = new MongoSearch(); + mongoSearch.open(adapter, database, List.of(COLLECTION_DOKUMENTE)); + + CollectionAdminRequest col = CollectionAdminRequest.createCollection("dokumente", 1, 1); + NamedList request = solrClient.request(col); } @AfterMethod - public void afterMethod() throws Exception { + public void afterMethod() throws Exception, SolrServerException { + adapter.close(); + mongoSearch.close(); + + CollectionAdminRequest col = CollectionAdminRequest.deleteCollection(COLLECTION_DOKUMENTE); + NamedList request = solrClient.request(col); } @Test - public void test_insert() throws IOException, InterruptedException, SolrServerException { - - Thread.sleep(2000); + public void test_solr() throws IOException, InterruptedException, SolrServerException { System.out.println("TEST"); - + SolrPingResponse response = solrClient.ping("dummy"); System.out.println("response: " + response.jsonStr()); + + CollectionAdminRequest col = CollectionAdminRequest.createCollection(COLLECTION_DOKUMENTE, 1, 1); - CollectionAdminRequest col = CollectionAdminRequest.createCollection("dokumente", 1, 1); - + NamedList request = solrClient.request(col); System.out.println(request); - + SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", UUID.randomUUID().toString()); doc.addField("name", "thorsten"); - solrClient.add("dokumente", doc); + solrClient.add(COLLECTION_DOKUMENTE, doc); + } + + @Test + public void test_insert() throws IOException, InterruptedException, SolrServerException { + + Thread.sleep(2000); + + assertCollectionSize(COLLECTION_DOKUMENTE, 0); + + insertDocument(COLLECTION_DOKUMENTE, Map.of("name", "thorsten")); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> getSize(COLLECTION_DOKUMENTE) > 0); + + assertCollectionSize(COLLECTION_DOKUMENTE, 1); + + insertDocument(COLLECTION_DOKUMENTE, Map.of("name", "thorsten")); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> getSize(COLLECTION_DOKUMENTE) > 1); + + assertCollectionSize(COLLECTION_DOKUMENTE, 2); + } + + private void insertDocument(final String collectionName, final Map attributes) { + MongoCollection collection = database.getCollection(collectionName); + Document document = new Document(attributes); + collection.insertOne(document); + } + + private void deleteDocument(final String collectionName, final String uid) { + MongoCollection collection = database.getCollection(collectionName); + collection.deleteOne(Filters.eq("_id", new ObjectId(uid))); + } + + private long getSize(final String collection) throws IOException, SolrServerException { + SolrQuery query = new SolrQuery("*:*"); + final QueryResponse response = solrClient.query(collection, query); + + System.out.println(response.getResults().getNumFound()); + + return response.getResults().getNumFound(); + } + + private void assertCollectionSize(final String collection, int size) throws IOException, SolrServerException { + long count = getSize(collection); + Assertions.assertThat(count).isEqualTo(size); } } diff --git a/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/FieldConfiguration.java b/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/FieldConfiguration.java index c7dd9bc..fe2bc3c 100644 --- a/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/FieldConfiguration.java +++ b/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/FieldConfiguration.java @@ -35,7 +35,7 @@ */ @SuperBuilder @Getter -public class FieldConfiguration { +public class FieldConfiguration { private final String fieldName; private final String indexFieldName; diff --git a/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/IndexConfiguration.java b/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/IndexConfiguration.java index 9394e81..00c5875 100644 --- a/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/IndexConfiguration.java +++ b/core/src/main/java/com/github/thmarx/mongo/search/index/configuration/IndexConfiguration.java @@ -4,7 +4,10 @@ */ package com.github.thmarx.mongo.search.index.configuration; +import com.github.thmarx.mongo.search.index.utils.MultiMap; +import java.util.Collection; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import lombok.Builder; import lombok.Getter; @@ -34,8 +37,25 @@ * * @author t.marx */ -public class IndexConfiguration { +public class IndexConfiguration { @Getter @Setter private BiConsumer documentExtender = (source, target) -> {}; + + final MultiMap fieldConfigurations = new MultiMap<>(); + + @Getter + @Setter + private BiFunction indexNameMapper = (database, collection) -> collection; + + public void addFieldConfiguration (final String collection, final FCT fieldConfig) { + fieldConfigurations.put(collection, fieldConfig); + } + + public Collection getFieldConfigurations (final String collection) { + return fieldConfigurations.get(collection); + } + public boolean hasFieldConfigurations (final String collection) { + return fieldConfigurations.containsKey(collection); + } }