diff --git a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/RunningElasticsearchInstanceES7.java b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/RunningElasticsearchInstanceES7.java index 4b6f1897d749..bb550f912cdb 100644 --- a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/RunningElasticsearchInstanceES7.java +++ b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/RunningElasticsearchInstanceES7.java @@ -38,6 +38,7 @@ import java.net.URL; import java.nio.file.Paths; import java.util.List; +import java.util.Map; public class RunningElasticsearchInstanceES7 implements SearchServerInstance { private final RestHighLevelClient restHighLevelClient; @@ -91,7 +92,7 @@ public FixtureImporter fixtureImporter() { } @Override - public GenericContainer createContainer(SearchVersion version, Network network, String heapSize) { + public GenericContainer createContainer(SearchVersion version, Network network, String heapSize, Map env) { return null; } diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/MessagesBatchOS2IT.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/MessagesBatchOS2IT.java index f0e892107343..7f66d508a4d4 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/MessagesBatchOS2IT.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/MessagesBatchOS2IT.java @@ -22,11 +22,16 @@ import org.graylog2.indexer.messages.MessagesBatchIT; import org.junit.Rule; -import java.util.Collections; - public class MessagesBatchOS2IT extends MessagesBatchIT { @Rule - public final OpenSearchInstance openSearchInstance = OpenSearchInstanceBuilder.builder().heapSize("256m").build(); + public final OpenSearchInstance openSearchInstance = OpenSearchInstanceBuilder.builder() + .heapSize("256m") + // Disable the real memory circuit breaker because it has issues with JDK 21. Turning it off relaxes + // the limits in the circuit breaker and makes our test work correctly. + // See: https://github.com/opensearch-project/OpenSearch/issues/12694 + // https://opensearch.org/docs/latest/install-and-configure/configuring-opensearch/circuit-breaker/#parent-circuit-breaker-settings + .env("indices.breaker.total.use_real_memory", "false") + .build(); @Override protected SearchServerInstance searchServer() { diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstance.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstance.java index a17b9d944430..d5b131115917 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstance.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstance.java @@ -77,7 +77,11 @@ public class OpenSearchInstance extends TestableSearchServerInstance { private List featureFlags; public OpenSearchInstance(final SearchVersion version, final String hostname, final Network network, final String heapSize, final List featureFlags) { - super(version, hostname, network, heapSize); + this(version, hostname, network, heapSize, featureFlags, Map.of()); + } + + public OpenSearchInstance(final SearchVersion version, final String hostname, final Network network, final String heapSize, final List featureFlags, Map env) { + super(version, hostname, network, heapSize, env); this.featureFlags = featureFlags; } @@ -226,6 +230,8 @@ public GenericContainer buildContainer(String image, Network network) { .withNetwork(network) .withNetworkAliases(hostname); + getContainerEnv().forEach(container::withEnv); + // disabling the performance plugin in 2.0.1 consistently created errors during CI runs, but keeping it running // in later versions sometimes created errors on CI, too. if (version().satisfies(SearchVersion.Distribution.OPENSEARCH, "^2.7.0")) { diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstanceBuilder.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstanceBuilder.java index 80ff470a6831..512c24dd2692 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstanceBuilder.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/OpenSearchInstanceBuilder.java @@ -32,6 +32,6 @@ public static OpenSearchInstanceBuilder builder() { @Override protected OpenSearchInstance instantiate() { - return new OpenSearchInstance(getVersion(), getHostname(), getNetwork(), getHeapSize(), getFeatureFlags()).init(); + return new OpenSearchInstance(getVersion(), getHostname(), getNetwork(), getHeapSize(), getFeatureFlags(), getEnv()).init(); } } diff --git a/graylog2-server/src/test/java/org/graylog/testing/completebackend/SearchServerBuilder.java b/graylog2-server/src/test/java/org/graylog/testing/completebackend/SearchServerBuilder.java index 2a41988ff9d3..9e2c02ddd580 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/completebackend/SearchServerBuilder.java +++ b/graylog2-server/src/test/java/org/graylog/testing/completebackend/SearchServerBuilder.java @@ -21,7 +21,9 @@ import org.testcontainers.containers.Network; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class SearchServerBuilder { public static final String DEFAULT_HEAP_SIZE = "2g"; @@ -29,6 +31,7 @@ public abstract class SearchServerBuilder { private String hostname = "indexer"; private Network network; private String heapSize = DEFAULT_HEAP_SIZE; + private final Map env = new HashMap<>(); private List featureFlags = List.of(); private String mongoDbUri; private String passwordSecret; @@ -60,6 +63,15 @@ public String getHeapSize() { return heapSize; } + public SearchServerBuilder env(final String key, final String value) { + this.env.put(key, value); + return this; + } + + public Map getEnv() { + return env; + } + public SearchServerBuilder featureFlags(final List featureFlags) { this.featureFlags = new ArrayList<>(featureFlags); return this; diff --git a/graylog2-server/src/test/java/org/graylog/testing/containermatrix/SearchServer.java b/graylog2-server/src/test/java/org/graylog/testing/containermatrix/SearchServer.java index 48927ba717d9..fd9970a47980 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/containermatrix/SearchServer.java +++ b/graylog2-server/src/test/java/org/graylog/testing/containermatrix/SearchServer.java @@ -27,7 +27,7 @@ public enum SearchServer { OS1(OPENSEARCH, "1.3.12"), OS2(OPENSEARCH, "2.0.1"), OS2_4(OPENSEARCH, "2.4.1"), - OS2_LATEST(OPENSEARCH, "2.11.0"), + OS2_LATEST(OPENSEARCH, "2.12.0"), DATANODE_PRE_52(DATANODE, "5.1.0"), DATANODE_DEV(DATANODE, "5.2.0"); diff --git a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/ContainerCacheKey.java b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/ContainerCacheKey.java index 587cdd717bc2..c615e7e4a8f3 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/ContainerCacheKey.java +++ b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/ContainerCacheKey.java @@ -16,5 +16,9 @@ */ package org.graylog.testing.elasticsearch; -public record ContainerCacheKey(org.graylog2.storage.SearchVersion version, String heapSize) { +import org.graylog2.storage.SearchVersion; + +import java.util.Map; + +public record ContainerCacheKey(SearchVersion version, String heapSize, Map env) { } diff --git a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/SearchServerInstance.java b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/SearchServerInstance.java index 7490e6630274..eb95e31cfe8b 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/SearchServerInstance.java +++ b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/SearchServerInstance.java @@ -22,6 +22,7 @@ import org.testcontainers.containers.Network; import java.io.Closeable; +import java.util.Map; public interface SearchServerInstance extends Closeable { Client client(); @@ -30,7 +31,7 @@ public interface SearchServerInstance extends Closeable { FixtureImporter fixtureImporter(); - GenericContainer createContainer(SearchVersion version, Network network, String heapSize); + GenericContainer createContainer(SearchVersion version, Network network, String heapSize, Map env); GenericContainer buildContainer(String image, Network network); diff --git a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/TestableSearchServerInstance.java b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/TestableSearchServerInstance.java index 28767b72b61b..506dcb89b7e1 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/TestableSearchServerInstance.java +++ b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/TestableSearchServerInstance.java @@ -47,6 +47,7 @@ public abstract class TestableSearchServerInstance extends ExternalResource impl protected final String heapSize; protected final Network network; protected final String hostname; + private final Map env; protected GenericContainer container; protected static volatile boolean isFirstContainerStart = true; @@ -59,22 +60,27 @@ public abstract class TestableSearchServerInstance extends ExternalResource impl public abstract FixtureImporter fixtureImporter(); protected TestableSearchServerInstance(final SearchVersion version, final String hostname, final Network network, final String heapSize) { + this(version, hostname, network, heapSize, Map.of()); + } + + protected TestableSearchServerInstance(final SearchVersion version, final String hostname, final Network network, final String heapSize, Map env) { this.version = version; this.heapSize = heapSize; this.network = network; this.hostname = hostname; + this.env = env; } protected abstract String imageName(); public void createContainer() { - this.container = createContainer(version, network, heapSize); + this.container = createContainer(version, network, heapSize, env); } @Override - public GenericContainer createContainer(SearchVersion version, Network network, String heapSize) { + public GenericContainer createContainer(SearchVersion version, Network network, String heapSize, Map env) { final var image = imageName(); - final ContainerCacheKey cacheKey = new ContainerCacheKey(version, heapSize); + final ContainerCacheKey cacheKey = new ContainerCacheKey(version, heapSize, env); if (!containersByVersion.containsKey(cacheKey)) { LOG.debug("Creating instance {}", image); GenericContainer container = buildContainer(image, network); @@ -145,6 +151,10 @@ protected String getEsJavaOpts() { return StringUtils.f("-Xms%s -Xmx%s -Dlog4j2.formatMsgNoLookups=true", heapSize, heapSize); } + protected Map getContainerEnv() { + return env; + } + @Override public String getHttpHostAddress() { return this.container.getHost() + ":" + this.container.getMappedPort(9200);