Arrow Flight Server bootstrap logic

* new plugin for StreamManager implementation * integration with server module * support for SslContext in Flight server and client * ClientManager for creating a pool of flight clients for data nodes * custom event loop group and thread pool for server and client channel Signed-off-by: Rishabh Maurya <[email protected]>
opensearch-project · Jan 7, 2025 · 69e9a97 · 69e9a97
1 parent e7e19f7
commit 69e9a97
Show file tree

Hide file tree

Showing 202 changed files with 21,577 additions and 729 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce framework for auxiliary transports and an experimental gRPC transport plugin ([#16534](https://github.com/opensearch-project/OpenSearch/pull/16534))
 - Changes to support IP field in star tree indexing([#16641](https://github.com/opensearch-project/OpenSearch/pull/16641/))
 - Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))
+- Arrow Flight server bootstrap logic ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java
@@ -174,6 +174,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
     private boolean isWorkingDirConfigured = false;
     private String httpPort = "0";
     private String transportPort = "0";
+    private String streamPort = "0";
     private Path confPathData;
     private String keystorePassword = "";
     private boolean preserveDataDir = false;
@@ -1175,6 +1176,8 @@ private void createConfiguration() {
         baseConfig.put("node.portsfile", "true");
         baseConfig.put("http.port", httpPort);
         baseConfig.put("transport.port", transportPort);
+        baseConfig.put("node.attr.transport.stream.port", streamPort);
+
         // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
         baseConfig.put("cluster.routing.allocation.disk.watermark.low", "1b");
         baseConfig.put("cluster.routing.allocation.disk.watermark.high", "1b");
@@ -1447,6 +1450,10 @@ void setTransportPort(String transportPort) {
         this.transportPort = transportPort;
     }
 
+    void setStreamPort(String streamPort) {
+        this.streamPort = streamPort;
+    }
+
     void setDataPath(Path dataPath) {
         this.confPathData = dataPath;
     }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java
@@ -61,6 +61,7 @@ public class RunTask extends DefaultTestClustersTask {
     public static final String CUSTOM_SETTINGS_PREFIX = "tests.opensearch.";
     private static final int DEFAULT_HTTP_PORT = 9200;
     private static final int DEFAULT_TRANSPORT_PORT = 9300;
+    private static final int DEFAULT_STREAM_PORT = 9880;
     private static final int DEFAULT_DEBUG_PORT = 5005;
     public static final String LOCALHOST_ADDRESS_PREFIX = "127.0.0.1:";
 
@@ -140,6 +141,8 @@ public void beforeStart() {
         int debugPort = DEFAULT_DEBUG_PORT;
         int httpPort = DEFAULT_HTTP_PORT;
         int transportPort = DEFAULT_TRANSPORT_PORT;
+        int streamPort = DEFAULT_STREAM_PORT;
+
         Map<String, String> additionalSettings = System.getProperties()
             .entrySet()
             .stream()
@@ -164,15 +167,19 @@ public void beforeStart() {
             firstNode.setHttpPort(String.valueOf(httpPort));
             httpPort++;
             firstNode.setTransportPort(String.valueOf(transportPort));
+            firstNode.setStreamPort(String.valueOf(streamPort));
             transportPort++;
+            streamPort++;
             firstNode.setting("discovery.seed_hosts", LOCALHOST_ADDRESS_PREFIX + DEFAULT_TRANSPORT_PORT);
             cluster.setPreserveDataDir(preserveData);
             for (OpenSearchNode node : cluster.getNodes()) {
                 if (node != firstNode) {
                     node.setHttpPort(String.valueOf(httpPort));
                     httpPort++;
                     node.setTransportPort(String.valueOf(transportPort));
+                    node.setStreamPort(String.valueOf(streamPort));
                     transportPort++;
+                    streamPort++;
                     node.setting("discovery.seed_hosts", LOCALHOST_ADDRESS_PREFIX + DEFAULT_TRANSPORT_PORT);
                 }
                 additionalSettings.forEach(node::setting);

diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
@@ -28,7 +28,7 @@
  * under the License.
  */
 
-import org.opensearch.gradle.JavaPackageType 
+import org.opensearch.gradle.JavaPackageType
 
 apply plugin: 'opensearch.internal-distribution-archive-setup'
 
@@ -190,7 +190,7 @@ distribution_archives {
     }
   }
 
-  
+
   linuxPpc64leTar {
     archiveClassifier = 'linux-ppc64le'
     content {

diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
@@ -89,3 +89,4 @@ ${error.file}
 # See please https://bugs.openjdk.org/browse/JDK-8341127 (openjdk/jdk#21283)
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
+--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
@@ -78,7 +78,5 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.tukaani.xz.UnsupportedOptionsException',
   'org.tukaani.xz.XZ',
   'org.tukaani.xz.XZOutputStream',
-  'org.apache.commons.codec.digest.PureJavaCrc32C',
-  'org.apache.commons.codec.digest.XXHash32',
   'org.apache.commons.lang3.reflect.FieldUtils'
 )
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-LICENSE b/distribution/tools/upgrade-cli/licenses/jackson-LICENSE
diff --git a/distribution/tools/upgrade-cli/licenses/jackson-NOTICE b/distribution/tools/upgrade-cli/licenses/jackson-NOTICE
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -82,7 +82,7 @@ opentelemetry         = "1.41.0"
 opentelemetrysemconv  = "1.27.0-alpha"
 
 # arrow dependencies
-arrow                 = "17.0.0"
+arrow                 = "18.1.0"
 flatbuffers           = "2.0.0"
 
 [libraries]

diff --git a/libs/arrow-spi/build.gradle b/libs/arrow-spi/build.gradle
@@ -72,17 +72,7 @@ tasks.named('thirdPartyAudit').configure {
     "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField",
     "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField",
     "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField",
-    "org.apache.arrow.memory.ArrowBuf",
-    "org.apache.arrow.memory.util.ByteFunctionHelpers",
     "org.apache.arrow.memory.util.MemoryUtil",
-    "org.apache.arrow.memory.util.MemoryUtil\$1",
-    "org.apache.arrow.memory.util.hash.MurmurHasher",
-    "org.apache.arrow.memory.util.hash.SimpleHasher",
-    "org.apache.arrow.vector.BaseFixedWidthVector",
-    "org.apache.arrow.vector.BitVectorHelper",
-    "org.apache.arrow.vector.Decimal256Vector",
-    "org.apache.arrow.vector.DecimalVector",
-    "org.apache.arrow.vector.util.DecimalUtility",
-    "org.apache.arrow.vector.util.VectorAppender"
+    "org.apache.arrow.memory.util.MemoryUtil\$1"
   )
 }
diff --git a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1
diff --git a/libs/arrow-spi/licenses/arrow-format-18.1.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-format-18.1.0.jar.sha1
@@ -0,0 +1 @@
+9d356b6f20620f5619ff85b174f97ae507df4997
diff --git a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1
diff --git a/libs/arrow-spi/licenses/arrow-memory-core-18.1.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-core-18.1.0.jar.sha1
@@ -0,0 +1 @@
+35f4853d512f06759759b40b53bac850867886f8
diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1
diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-18.1.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-18.1.0.jar.sha1
@@ -0,0 +1 @@
+9e9e08d0b548d2c02c632e5daaf176e588810d22
diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1
diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1
@@ -0,0 +1 @@
+86c8fbdb6ab220603ea3a215f48a7f793ac6a08d
diff --git a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1
diff --git a/libs/arrow-spi/licenses/arrow-vector-18.1.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-vector-18.1.0.jar.sha1
@@ -0,0 +1 @@
+b1fb77f4ef36fd52afe480ba12b7da77367eb88c