Skip to content

Commit

Permalink
Delete ElasticSearch support from Kitodo - Core
Browse files Browse the repository at this point in the history
  • Loading branch information
@matthias-ronge
matthias-ronge committed Aug 28, 2024
1 parent 80c5f81 commit b7f6236
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 269 deletions.
4 changes: 0 additions & 4 deletions Kitodo/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,6 @@
<groupId>org.apache.avalon.framework</groupId>
<artifactId>avalon-framework-impl</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
</dependency>
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom2</artifactId>
Expand Down
200 changes: 0 additions & 200 deletions Kitodo/src/main/java/org/kitodo/production/services/data/base/SearchService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,34 +11,21 @@

package org.kitodo.production.services.data.base;

import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.kitodo.data.database.beans.BaseBean;
import org.kitodo.data.database.beans.BaseIndexedBean;
import org.kitodo.data.database.enums.IndexAction;
import org.kitodo.data.database.exceptions.DAOException;
import org.kitodo.data.database.persistence.BaseDAO;
import org.kitodo.data.elasticsearch.exceptions.CustomResponseException;
import org.kitodo.data.exceptions.DataException;
import org.kitodo.data.interfaces.DataInterface;
import org.primefaces.model.SortOrder;

/**
* Class for implementing methods used by all service classes which search in
Expand Down Expand Up @@ -266,16 +253,6 @@ protected List<S> convertJSONObjectsToInterfaces(List<Map<String, Object>> jsonO
return results;
}

/**
* Builds a ElasticSearch query for list of Ids.
*
* @param ids as a List of Integer
* @return query as QueryBuilder
*/
public QueryBuilder createSetQueryForIds(List<Integer> ids) {
return termsQuery("_id", ids);
}

/**
* Get id from JSON object returned form ElasticSearch.
*
Expand All @@ -293,183 +270,6 @@ public Integer getIdFromJSONObject(Map<String, Object> jsonObject) {
return 0;
}

/**
* Create query for set of data.
*
* @param key
* JSON key for searched object
* @param values
* set of values for searched objects or some objects related to
* searched object
* @param contains
* determine if results should contain given value or should not
* contain given value
* @return query
*/
protected QueryBuilder createSetQuery(String key, Set<?> values, boolean contains) {
if (contains && !values.isEmpty()) {
return termsQuery(key, values);
} else if (!contains && Objects.nonNull(values)) {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
return boolQuery.mustNot(termsQuery(key, values));
} else {
return matchQuery(key, 0);
}
}

protected QueryBuilder createSetQuery(String key, List<Map<String, Object>> values, boolean contains) {
Set<Integer> valuesIds = new HashSet<>();
for (Map<String, Object> value : values) {
valuesIds.add(getIdFromJSONObject(value));
}

return createSetQuery(key, valuesIds, contains);
}

protected QueryBuilder createSetQueryForBeans(String key, List<? extends BaseBean> values, boolean contains) {
Set<Integer> valuesIds = new HashSet<>();
for (BaseBean value : values) {
valuesIds.add(value.getId());
}

return createSetQuery(key, valuesIds, contains);
}

/**
* Used for cases where operator is not necessary to create query - checking
* only for one parameter.
*
* @param key
* JSON key for searched object
* @param id
* id value for searched object or some object related to searched
* object
* @param contains
* determine if results should contain given value or should not
* contain given value
* @return query
*/
protected QueryBuilder createSimpleQuery(String key, Integer id, boolean contains) {
if (contains && Objects.nonNull(id)) {
return matchQuery(key, id);
} else if (!contains && Objects.nonNull(id)) {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
return boolQuery.mustNot(matchQuery(key, id));
} else {
return matchQuery(key, 0);
}
}

/**
* Used for cases where operator is not necessary to create query - checking
* only for one parameter.
*
* @param key
* JSON key for searched object
* @param condition
* id value for searched object or some object related to searched
* object
* @param contains
* determine if results should contain given value or should not
* contain given value
* @return query
*/
protected QueryBuilder createSimpleQuery(String key, Boolean condition, boolean contains) {
if (contains && Objects.nonNull(condition)) {
return matchQuery(key, condition);
} else if (!contains && Objects.nonNull(condition)) {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
return boolQuery.mustNot(matchQuery(key, condition));
} else {
return matchQuery(key, false);
}
}

/**
* Used for cases where operator is not necessary to create query - checking
* only for one parameter.
*
* @param key
* JSON key for searched object
* @param value
* JSON value for searched object
* @param contains
* determine if results should contain given value or should not
* contain given value
* @return query
*/
protected QueryBuilder createSimpleQuery(String key, String value, boolean contains) {
if (contains) {
return matchQuery(key, value);
} else {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
return boolQuery.mustNot(matchQuery(key, value));
}
}

/**
* Used for cases where operator is necessary to create query - checking for
* more than one parameter.
*
* @param key
* JSON key for searched object
* @param value
* JSON value for searched object
* @param contains
* determine if results should contain given value or should not
* contain given value
* @param operator
* as Operator AND or OR - useful when value contains more than one
* word
* @return query
*/
protected QueryBuilder createSimpleQuery(String key, String value, boolean contains, Operator operator) {
if (Objects.isNull(operator)) {
operator = Operator.OR;
}

if (contains) {
return matchQuery(key, value).operator(operator);
} else {
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
return boolQuery.mustNot(matchQuery(key, value).operator(operator));
}
}

protected QueryBuilder createSimpleWildcardQuery(String key, String value) {
return queryStringQuery(key + ".keyword: *" + value + "*");
}

/**
* Converts properties' values returned from ElasticSearch index.
*
* @param object
* JSONObject
* @return display properties as list of Integers
*/
@SuppressWarnings("unchecked")
private List<Integer> getRelatedPropertyFor(Map<String, Object> object, String key) {
if (Objects.nonNull(object)) {
List<Map<String, Object>> jsonArray = (List<Map<String, Object>>) object.get(key);
List<Integer> ids = new ArrayList<>();
for (Map<String, Object> singleObject : jsonArray) {
ids.add((Integer) singleObject.get("id"));
}
return ids;
}
return new ArrayList<>();
}

protected SortBuilder getSortBuilder(String sortField, SortOrder sortOrder) {
if (!Objects.equals(sortField, null) && Objects.equals(sortOrder, SortOrder.ASCENDING)) {
return SortBuilders.fieldSort(sortField).order(org.elasticsearch.search.sort.SortOrder.ASC);
} else if (!Objects.equals(sortField, null) && Objects.equals(sortOrder, SortOrder.DESCENDING)) {
return SortBuilders.fieldSort(sortField).order(org.elasticsearch.search.sort.SortOrder.DESC);
} else {
return null;
}
}

/**
* Removes all objects from index, which are no longer in Database.
* @param baseIndexedBeansId the list of beans to check for missing db eintries.
Expand Down
72 changes: 7 additions & 65 deletions Kitodo/src/test/java/org/kitodo/MockDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,13 @@

package org.kitodo;

import java.io.File;
import static com.xebialabs.restito.builder.stub.StubHttp.whenHttp;
import static com.xebialabs.restito.semantics.Condition.get;
import static com.xebialabs.restito.semantics.Condition.parameter;

import com.xebialabs.restito.semantics.Action;
import com.xebialabs.restito.server.StubServer;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
Expand All @@ -24,7 +30,6 @@
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -33,29 +38,19 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;

import com.xebialabs.restito.semantics.Action;
import com.xebialabs.restito.server.StubServer;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.awaitility.Awaitility;
import org.awaitility.Durations;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import org.h2.tools.Server;
import org.hibernate.Session;
import org.hibernate.Transaction;
import org.kitodo.api.externaldatamanagement.ImportConfigurationType;
import org.kitodo.api.externaldatamanagement.SearchInterfaceType;
import org.kitodo.api.schemaconverter.FileFormat;
import org.kitodo.api.schemaconverter.MetadataFormat;
import org.kitodo.config.ConfigMain;
import org.kitodo.data.database.beans.Authority;
import org.kitodo.data.database.beans.Batch;
import org.kitodo.data.database.beans.Client;
Expand Down Expand Up @@ -91,7 +86,6 @@
import org.kitodo.exceptions.WorkflowException;
import org.kitodo.production.enums.ObjectType;
import org.kitodo.production.enums.ProcessState;
import org.kitodo.production.helper.Helper;
import org.kitodo.production.process.ProcessGenerator;
import org.kitodo.production.security.password.SecurityPasswordEncoder;
import org.kitodo.production.services.ServiceManager;
Expand All @@ -100,21 +94,13 @@
import org.kitodo.test.utils.ProcessTestUtils;
import org.kitodo.test.utils.TestConstants;

import static com.xebialabs.restito.builder.stub.StubHttp.whenHttp;
import static com.xebialabs.restito.semantics.Condition.get;
import static com.xebialabs.restito.semantics.Condition.parameter;

/**
* Insert data to test database.
*/
public class MockDatabase {

private static Node node;
private static String testIndexName;
private static final String GLOBAL_ASSIGNABLE = "_globalAssignable";
private static final String CLIENT_ASSIGNABLE = "_clientAssignable";
private static final String HTTP_TRANSPORT_PORT = "9305";
private static final String TARGET = "target";
private static final String CHOICE = "choice";
private static final String TEST = "test";
private static final String FIRST_VALUE = "first value";
Expand Down Expand Up @@ -146,26 +132,9 @@ public static void startNode() throws Exception {
}

public static void startNodeWithoutMapping() throws Exception {
String nodeName = Helper.generateRandomString(6);
final String port = ConfigMain.getParameter("elasticsearch.port", "9205");

testIndexName = ConfigMain.getParameter("elasticsearch.index", "testindex");

Settings settings = prepareNodeSettings(port, nodeName);

removeOldDataDirectories("target/" + nodeName);

if (node != null) {
stopNode();
}
Supplier<String> nodeNameSupplier = () -> nodeName;
node = new ExtendedNode(settings, Collections.singleton(Netty4Plugin.class), nodeNameSupplier);
node.start();
}

public static void stopNode() throws Exception {
node.close();
node = null;
}

public static void setUpAwaitility() {
Expand Down Expand Up @@ -241,33 +210,6 @@ public static void insertForDataEditorTesting() throws Exception {
insertDataEditorSettings();
}

private static class ExtendedNode extends Node {
ExtendedNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins,
Supplier<String> nodeNameSupplier) {
super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(),
Paths.get("target"), nodeNameSupplier), classpathPlugins, false);
}
}

private static void removeOldDataDirectories(String dataDirectory) throws Exception {
File dataDir = new File(dataDirectory);
if (dataDir.exists()) {
FileSystemUtils.deleteSubDirectories(dataDir.toPath());
}
}

private static Settings prepareNodeSettings(String httpPort, String nodeName) {
return Settings.builder().put("node.name", nodeName)
.put("path.data", TARGET)
.put("path.logs", TARGET)
.put("path.home", TARGET)
.put("http.type", "netty4")
.put("http.port", httpPort)
.put("transport.tcp.port", HTTP_TRANSPORT_PORT)
.put("transport.type", "netty4")
.put("action.auto_create_index", "false").build();
}

public static void insertAuthorities() throws DAOException {
List<Authority> authorities = new ArrayList<>();

Expand Down

0 comments on commit b7f6236

Please sign in to comment.