Skip to content

Commit

Permalink
Revert OpenSearch Client Migration (#18686)
Browse files Browse the repository at this point in the history 
* Revert "Monkey-patching `NodeRole` class to avoid role parsing bug. (#18648)"

This reverts commit 0bf99aa.

* Revert "Fix parsing of missing watermark settings. (#18624)"

This reverts commit e9783e4.

* Revert "Migrating `IndexToolsAdapter` to new Opensearch client. (#18505)"

This reverts commit 64921ab.

* Revert "Migrating query suggestions to new OpenSearch client. (#17998)"

This reverts commit 27f55ec.

* Revert "Setting up injection/usage of new OpenSearch Java Client. (#17760)"

This reverts commit aacaa0c.

* needed changes to revert part1

* Fix opensearch rest client providers

* fix OpenSearchInstance

* Add license

---------

Co-authored-by: Tomas Dvorak <[email protected]>
  • Loading branch information
@janheise @todvora
janheise and todvora authored Mar 19, 2024
1 parent 090c7c4 commit 38bf3e1
Show file tree
Hide file tree
Showing 57 changed files with 569 additions and 1,919 deletions.
13 changes: 2 additions & 11 deletions data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.graylog.datanode.management;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.oxo42.stateless4j.StateMachine;
import org.apache.commons.collections4.queue.CircularFifoQueue;
import org.apache.commons.exec.ExecuteException;
import org.apache.http.client.utils.URIBuilder;
Expand All @@ -34,9 +35,6 @@
import org.graylog2.cluster.nodes.DataNodeDto;
import org.graylog2.cluster.nodes.NodeService;
import org.graylog2.security.CustomCAX509TrustManager;
import org.opensearch.client.RestClient;
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
import org.opensearch.client.transport.rest_client.RestClientTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -96,10 +94,6 @@ private RestHighLevelClient createRestClient(OpensearchConfiguration configurati
return OpensearchRestClient.build(configuration, datanodeConfiguration, trustManager);
}

private RestClient createNewRestClient(OpensearchConfiguration configuration) {
return OpensearchRestClient.buildNewClient(configuration, datanodeConfiguration, trustManager);
}

@Override
public List<String> stdOutLogs() {
return stdout.stream().toList();
Expand Down Expand Up @@ -206,10 +200,7 @@ public synchronized void start() {
(config -> {
commandLineProcess.start();
restClient = Optional.of(createRestClient(config));
final var newRestClient = createNewRestClient(config);
final var transport = new RestClientTransport(newRestClient, new JacksonJsonpMapper(objectMapper));
final var client = new org.opensearch.client.opensearch.OpenSearchClient(transport);
openSearchClient = restClient.map(c -> new OpenSearchClient(c, client, newRestClient, objectMapper));
openSearchClient = restClient.map(c -> new OpenSearchClient(c, objectMapper));
}),
() -> {throw new IllegalArgumentException("Opensearch configuration required but not supplied!");}
);
Expand Down
26 changes: 0 additions & 26 deletions data-node/src/main/java/org/graylog/datanode/management/OpensearchRestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.graylog.shaded.opensearch2.org.apache.http.HttpRequestInterceptor;
import org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient;
import org.graylog2.security.CustomCAX509TrustManager;
import org.opensearch.client.RestClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -59,29 +58,4 @@ public static RestHighLevelClient build(final OpensearchConfiguration configurat
}
return new RestHighLevelClient(builder);
}

public static RestClient buildNewClient(OpensearchConfiguration configuration, DatanodeConfiguration datanodeConfiguration, CustomCAX509TrustManager tm) {
final HttpHost host = configuration.getRestBaseUrl();

final var builder = org.opensearch.client.RestClient.builder(new org.apache.http.HttpHost(host.getHostName(), host.getPort(), host.getSchemeName()));
if ("https".equals(host.getSchemeName())) {

try {
final var sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, new TrustManager[]{tm}, new SecureRandom());

builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.addInterceptorLast((org.apache.http.HttpRequestInterceptor) (request, context) -> {
final String jwtToken = datanodeConfiguration.indexerJwtAuthTokenProvider().get();
request.addHeader("Authorization", jwtToken);
});
httpClientBuilder.setSSLContext(sslContext);
return httpClientBuilder;
});
} catch (NoSuchAlgorithmException | KeyManagementException ex) {
LOG.error("Could not initialize SSL correctly: {}", ex.getMessage(), ex);
}
}
return builder.build();
}
}
35 changes: 12 additions & 23 deletions full-backend-tests/src/test/java/org/graylog/plugins/views/SuggestionResourceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,14 @@
import org.graylog.testing.containermatrix.SearchServer;
import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.BeforeAll;

import javax.annotation.Nullable;
import java.util.Set;

import static io.restassured.RestAssured.given;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.graylog.testing.completebackend.Lifecycle.CLASS;
import static org.graylog.testing.completebackend.Lifecycle.VM;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
Expand Down Expand Up @@ -130,10 +129,8 @@ void testMinimalRequest() {
.body(SuggestionsRequest.create("facility", ""))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.assertThat()
.log().ifValidationFails()
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("test"))
.body("suggestions.occurrence[0]", greaterThanOrEqualTo(3));
}
Expand All @@ -146,8 +143,7 @@ void testNumericalValueSuggestion() {
.body(SuggestionsRequest.create("http_response_code", "20"))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("200"))
.body("suggestions.occurrence[0]", greaterThanOrEqualTo(2));
Expand All @@ -161,8 +157,7 @@ void testAugmentedSuggestionTitlesForStreams() {
.body(SuggestionsRequest.create("streams", ""))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.title", hasItems("Default Stream", "Stream #1", "Stream #2"));
}
Expand All @@ -175,8 +170,7 @@ void testAugmentedSuggestionTitlesForNodes() {
.body(SuggestionsRequest.create("gl2_source_node", ""))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.title", not(empty()));
}
Expand All @@ -189,8 +183,7 @@ void testAugmentedSuggestionTitlesForInputs() {
.body(SuggestionsRequest.create("gl2_source_input", ""))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.title", hasItems("Integration test GELF input"));
}
Expand All @@ -203,7 +196,7 @@ void testSuggestionsAreLimitedToStream() {
.body(SuggestionsRequest.create("source", "", Set.of(stream1Id)))
.post("/search/suggest")
.then()
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("example.org"))
.body("suggestions.occurrence[0]", equalTo(3));
Expand All @@ -214,8 +207,7 @@ void testSuggestionsAreLimitedToStream() {
.body(SuggestionsRequest.create("source", "", Set.of(stream2Id)))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("foreign.org"))
.body("suggestions.occurrence[0]", equalTo(1));
Expand All @@ -229,8 +221,7 @@ void testInvalidField() {
.body(SuggestionsRequest.create("message", "foo"))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
// error types and messages are different for each ES version, so let's just check that there is an error in the response
.body("error", notNullValue());
Expand All @@ -244,8 +235,7 @@ void testSizeOtherDocsCount() {
.body(SuggestionsRequest.create("facility", "", 1))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("test"))
.body("suggestions.occurrence[0]", greaterThanOrEqualTo(2))
Expand All @@ -260,8 +250,7 @@ void testTypoCorrection() {
.body(SuggestionsRequest.create("facility", "tets"))
.post("/search/suggest")
.then()
.log().ifStatusCodeMatches(Matchers.not(HTTP_OK))
.statusCode(HTTP_OK)
.statusCode(200)
.assertThat().log().ifValidationFails()
.body("suggestions.value[0]", equalTo("test"))
.body("suggestions.occurrence[0]", greaterThanOrEqualTo(1));
Expand Down
42 changes: 0 additions & 42 deletions graylog-storage-opensearch2/assembly.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,53 +14,11 @@
<scope>runtime</scope>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<useStrictFiltering>true</useStrictFiltering>
<includes>
<include>org.opensearch.client:opensearch-java</include>
</includes>
<unpackOptions>
<excludes>
<!-- exclude NodeInfoAction since it was monkey-patched in
https://github.com/Graylog2/graylog2-server/pull/18658 .
Should be removed after the bug has been fixed and the patch removed. -->
<exclude>org/opensearch/client/opensearch/nodes/info/NodeInfoAction.class</exclude>
<exclude>org/opensearch/client/opensearch/nodes/info/NodeInfoBootstrap.class</exclude>
<exclude>org/opensearch/client/opensearch/_types/NodeRole.class</exclude>
</excludes>
</unpackOptions>
</dependencySet>
<dependencySet>
<outputDirectory>/</outputDirectory>
<unpack>true</unpack>
<scope>runtime</scope>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<useStrictFiltering>true</useStrictFiltering>
<includes>
<include>org.graylog:graylog-storage-opensearch2</include>
<include>org.graylog.shaded:opensearch2-rest-high-level-client</include>
<include>org.graylog.shaded:opensearch2-rest-client-sniffer</include>
<include>org.opensearch.client:opensearch-rest-client</include>
<include>org.opensearch.client:opensearch-rest-client-sniffer</include>
<include>org.apache.httpcomponents:httpasyncclient</include>
<include>org.apache.httpcomponents:httpclient</include>
<include>org.apache.httpcomponents:httpcore</include>
<include>org.apache.httpcomponents:httpcore-nio</include>
<include>jakarta.json.bind:jakarta.json.bind-api</include>
<include>jakarta.json:jakarta.json-api</include>
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>com.fasterxml.jackson.core:jackson-databind</include>
<include>org.eclipse.parsson:parsson</include>
<include>org.eclipse:yasson</include>
</includes>
<unpackOptions>
<excludes>
<!-- Including module-info.class files would turn the
artifact into a Java module and we would run into
class loader issues. -->
<exclude>module-info.class</exclude>
</excludes>
</unpackOptions>
</dependencySet>
</dependencySets>
</assembly>
36 changes: 1 addition & 35 deletions graylog-storage-opensearch2/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,39 +72,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-java</artifactId>
<version>2.9.1</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-client</artifactId>
<version>2.12.0</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-client-sniffer</artifactId>
<version>2.12.0</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
Expand Down Expand Up @@ -210,8 +177,7 @@
<descriptors>assembly.xml</descriptors>
<archive>
<manifestEntries>
<Graylog-Plugin-Properties-Path>${project.groupId}.${project.artifactId}
</Graylog-Plugin-Properties-Path>
<Graylog-Plugin-Properties-Path>${project.groupId}.${project.artifactId}</Graylog-Plugin-Properties-Path>
</manifestEntries>
</archive>
</configuration>
Expand Down
Loading

0 comments on commit 38bf3e1

Please sign in to comment.