Merge branch 'master' into feat/Add-telemetry-to-events-actions

changelog/unreleased/pr-21208.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "batching request for index block status if the combined length of the indices exceed the max possible URL length "
+
+pulls = ["21208"]

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java

@@ -109,6 +109,11 @@ public class IndicesAdapterES7 implements IndicesAdapter {
     private final ClusterStateApi clusterStateApi;
     private final IndexTemplateAdapter indexTemplateAdapter;
 
+    // this is the maximum amount of bytes that the index list is supposed to fill in a request,
+    // it assumes that these don't need url encoding. If we exceed the maximum, we request settings for all indices
+    // and filter after wards
+    private final int MAX_INDICES_URL_LENGTH = 3000;
+
     @Inject
     public IndicesAdapterES7(ElasticsearchClient client,
                              StatsApi statsApi,
@@ -435,14 +440,17 @@ public IndicesBlockStatus getIndicesBlocksStatus(final List<String> indices) {
         if (indices == null || indices.isEmpty()) {
             throw new IllegalArgumentException("Expecting list of indices with at least one index present.");
         }
-        final GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices(indices.toArray(new String[]{}))
+
+        final GetSettingsRequest request = new GetSettingsRequest()
                 .indicesOptions(IndicesOptions.fromOptions(false, true, true, true))
-                .names(new String[]{});
+                .names("index.blocks.read", "index.blocks.write", "index.blocks.metadata", "index.blocks.read_only", "index.blocks.read_only_allow_delete");
+
+        final var maxLengthExceeded = String.join(",", indices).length() > MAX_INDICES_URL_LENGTH;
+        final GetSettingsRequest getSettingsRequest = maxLengthExceeded ? request : request.indices(indices.toArray(new String[]{}));
 
         return client.execute((c, requestOptions) -> {
             final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions);
-            return BlockSettingsParser.parseBlockSettings(settingsResponse);
+            return BlockSettingsParser.parseBlockSettings(settingsResponse, maxLengthExceeded ? Optional.of(indices) : Optional.empty());
         });
     }
 

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/blocks/BlockSettingsParser.java

@@ -21,6 +21,9 @@
 import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.settings.Settings;
 import org.graylog2.indexer.indices.blocks.IndicesBlockStatus;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -29,23 +32,31 @@ public class BlockSettingsParser {
     static final String BLOCK_SETTINGS_PREFIX = "index.blocks.";
 
     public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse) {
-        IndicesBlockStatus result = new IndicesBlockStatus();
+        return parseBlockSettings(settingsResponse, Optional.empty());
+    }
+
+    public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse, final Optional<List<String>> indices) {
+        final IndicesBlockStatus result = new IndicesBlockStatus();
         final ImmutableOpenMap<String, Settings> indexToSettingsMap = settingsResponse.getIndexToSettings();
         final String[] indicesInResponse = indexToSettingsMap.keys().toArray(String.class);
-        for (String index : indicesInResponse) {
-            final Settings blockSettings = indexToSettingsMap.get(index).getByPrefix(BLOCK_SETTINGS_PREFIX);
-
-            if (!blockSettings.isEmpty()) {
-                final Set<String> blockSettingsNames = blockSettings.names();
-                final Set<String> blockSettingsSetToTrue = blockSettingsNames.stream()
-                        .filter(s -> blockSettings.getAsBoolean(s, false))
-                        .map(s -> BLOCK_SETTINGS_PREFIX + s)
-                        .collect(Collectors.toSet());
-                if (!blockSettingsSetToTrue.isEmpty()) {
-                    result.addIndexBlocks(index, blockSettingsSetToTrue);
+
+        indices.orElse(Arrays.stream(indicesInResponse).toList()).forEach(index -> {
+            final var settings = indexToSettingsMap.get(index);
+            if(settings != null) {
+                final Settings blockSettings = settings.getByPrefix(BLOCK_SETTINGS_PREFIX);
+
+                if (!blockSettings.isEmpty()) {
+                    final Set<String> blockSettingsNames = blockSettings.names();
+                    final Set<String> blockSettingsSetToTrue = blockSettingsNames.stream()
+                            .filter(s -> blockSettings.getAsBoolean(s, false))
+                            .map(s -> BLOCK_SETTINGS_PREFIX + s)
+                            .collect(Collectors.toSet());
+                    if (!blockSettingsSetToTrue.isEmpty()) {
+                        result.addIndexBlocks(index, blockSettingsSetToTrue);
+                    }
                 }
             }
-        }
+        });
 
         return result;
     }

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java

@@ -84,6 +84,7 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -97,6 +98,7 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 import static org.graylog.storage.opensearch2.OpenSearchClient.withTimeout;
 
@@ -109,6 +111,11 @@ public class IndicesAdapterOS2 implements IndicesAdapter {
     private final ClusterStateApi clusterStateApi;
     private final IndexTemplateAdapter indexTemplateAdapter;
 
+    // this is the maximum amount of bytes that the index list is supposed to fill in a request,
+    // it assumes that these don't need url encoding. If we exceed the maximum, we request settings for all indices
+    // and filter after wards
+    private final int MAX_INDICES_URL_LENGTH = 3000;
+
     @Inject
     public IndicesAdapterOS2(OpenSearchClient client,
                              StatsApi statsApi,
@@ -431,19 +438,23 @@ public List<ShardsInfo> getShardsInfo(String indexName) {
         return catApi.getShardsInfo(indexName);
     }
 
+
     @Override
     public IndicesBlockStatus getIndicesBlocksStatus(final List<String> indices) {
         if (indices == null || indices.isEmpty()) {
             throw new IllegalArgumentException("Expecting list of indices with at least one index present.");
         }
-        final GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices(indices.toArray(new String[]{}))
+
+        final GetSettingsRequest request = new GetSettingsRequest()
                 .indicesOptions(IndicesOptions.fromOptions(false, true, true, true))
-                .names(new String[]{});
+                .names("index.blocks.read", "index.blocks.write", "index.blocks.metadata", "index.blocks.read_only", "index.blocks.read_only_allow_delete");
+
+        final var maxLengthExceeded = String.join(",", indices).length() > MAX_INDICES_URL_LENGTH;
+        final GetSettingsRequest getSettingsRequest = maxLengthExceeded ? request : request.indices(indices.toArray(new String[]{}));
 
         return client.execute((c, requestOptions) -> {
             final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions);
-            return BlockSettingsParser.parseBlockSettings(settingsResponse);
+            return BlockSettingsParser.parseBlockSettings(settingsResponse, maxLengthExceeded ? Optional.of(indices) : Optional.empty());
         });
     }
 

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParser.java

@@ -20,6 +20,8 @@
 import org.graylog.shaded.opensearch2.org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings;
 
+import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -28,23 +30,30 @@ public class BlockSettingsParser {
     static final String BLOCK_SETTINGS_PREFIX = "index.blocks.";
 
     public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse) {
-        IndicesBlockStatus result = new IndicesBlockStatus();
+        return parseBlockSettings(settingsResponse, Optional.empty());
+    }
+
+    public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse, final Optional<List<String>> indices) {
+        final IndicesBlockStatus result = new IndicesBlockStatus();
         final var indexToSettingsMap = settingsResponse.getIndexToSettings();
-        final String[] indicesInResponse = indexToSettingsMap.keySet().toArray(new String[0]);
-        for (String index : indicesInResponse) {
-            final Settings blockSettings = indexToSettingsMap.get(index).getByPrefix(BLOCK_SETTINGS_PREFIX);
-
-            if (!blockSettings.isEmpty()) {
-                final Set<String> blockSettingsNames = blockSettings.names();
-                final Set<String> blockSettingsSetToTrue = blockSettingsNames.stream()
-                        .filter(s -> blockSettings.getAsBoolean(s, false))
-                        .map(s -> BLOCK_SETTINGS_PREFIX + s)
-                        .collect(Collectors.toSet());
-                if (!blockSettingsSetToTrue.isEmpty()) {
-                    result.addIndexBlocks(index, blockSettingsSetToTrue);
+
+        indices.orElse(indexToSettingsMap.keySet().stream().toList()).forEach(index -> {
+            final var settings = indexToSettingsMap.get(index);
+            if(settings != null) {
+                final Settings blockSettings = settings.getByPrefix(BLOCK_SETTINGS_PREFIX);
+
+                if (!blockSettings.isEmpty()) {
+                    final Set<String> blockSettingsNames = blockSettings.names();
+                    final Set<String> blockSettingsSetToTrue = blockSettingsNames.stream()
+                            .filter(s -> blockSettings.getAsBoolean(s, false))
+                            .map(s -> BLOCK_SETTINGS_PREFIX + s)
+                            .collect(Collectors.toSet());
+                    if (!blockSettingsSetToTrue.isEmpty()) {
+                        result.addIndexBlocks(index, blockSettingsSetToTrue);
+                    }
                 }
             }
-        }
+        });
 
         return result;
     }

graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParserTest.java

@@ -23,6 +23,7 @@
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -35,7 +36,7 @@ public class BlockSettingsParserTest {
     @Test
     public void noBlockedIndicesIdentifiedIfEmptyResponseParsed() {
         GetSettingsResponse emptyResponse = new GetSettingsResponse(Map.of(), Map.of());
-        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptyResponse);
+        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptyResponse, Optional.empty());
         assertNotNull(indicesBlockStatus);
         assertEquals(0, indicesBlockStatus.countBlockedIndices());
     }
@@ -44,7 +45,7 @@ public void noBlockedIndicesIdentifiedIfEmptyResponseParsed() {
     public void noBlockedIndicesIdentifiedIfEmptySettingsPresent() {
         var settingsBuilder = Map.of("index_0", Settings.builder().build());
         GetSettingsResponse emptySettingsResponse = new GetSettingsResponse(settingsBuilder, Map.of());
-        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptySettingsResponse);
+        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptySettingsResponse, Optional.empty());
         assertNotNull(indicesBlockStatus);
         assertEquals(0, indicesBlockStatus.countBlockedIndices());
     }
@@ -64,7 +65,7 @@ public void parserProperlyResponseWithMultipleIndicesWithDifferentBlockSettings(
                         .put("index.blocks.read_only_allow_delete", true)
                         .build());
         GetSettingsResponse settingsResponse = new GetSettingsResponse(settingsBuilder, Map.of());
-        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(settingsResponse);
+        final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(settingsResponse, Optional.empty());
         assertNotNull(indicesBlockStatus);
         assertEquals(3, indicesBlockStatus.countBlockedIndices());
         final Set<String> blockedIndices = indicesBlockStatus.getBlockedIndices();

graylog2-web-interface/packages/eslint-config-graylog/package.json

@@ -15,8 +15,8 @@
   "dependencies": {
     "@babel/eslint-parser": "7.16.5",
     "@tanstack/eslint-plugin-query": "4.36.1",
-    "@typescript-eslint/eslint-plugin": "8.19.0",
-    "@typescript-eslint/parser": "8.19.0",
+    "@typescript-eslint/eslint-plugin": "8.19.1",
+    "@typescript-eslint/parser": "8.19.1",
     "eslint": "8.57.0",
     "eslint-config-airbnb": "19.0.4",
     "eslint-import-resolver-webpack": "0.13.10",

pom.xml

@@ -107,7 +107,7 @@
         <caffeine.version>3.1.8</caffeine.version>
         <cef-parser.version>0.0.1.10</cef-parser.version>
         <classgraph.version>4.8.179</classgraph.version>
-        <commons-codec.version>1.17.1</commons-codec.version>
+        <commons-codec.version>1.17.2</commons-codec.version>
         <commons-csv.version>1.12.0</commons-csv.version>
         <commons-email.version>1.6.0</commons-email.version>
         <commons-exec.version>1.4.0</commons-exec.version>
@@ -117,7 +117,7 @@
         <commons-io.version>2.18.0</commons-io.version>
         <disruptor.version>4.0.0</disruptor.version>
         <error-prone.version>2.36.0</error-prone.version>
-        <freemarker.version>2.3.33</freemarker.version>
+        <freemarker.version>2.3.34</freemarker.version>
         <gelfclient.version>1.5.1</gelfclient.version>
         <geoip2.version>4.2.1</geoip2.version>
         <grok.version>0.1.9-graylog-3</grok.version>
@@ -144,7 +144,7 @@
         <jmte.version>7.0.3</jmte.version>
         <joda-time.version>2.13.0</joda-time.version>
         <jool.version>0.9.15</jool.version>
-        <json-org.version>20240303</json-org.version>
+        <json-org.version>20241224</json-org.version>
         <json-path.version>2.9.0</json-path.version>
         <kafka.version>3.9.0</kafka.version>
         <kafka09.version>0.9.0.1-7</kafka09.version>
@@ -179,7 +179,7 @@
         <unboundid-ldap.version>7.0.2</unboundid-ldap.version>
         <uuid.version>3.2.1</uuid.version>
         <validation-api.version>3.1.0</validation-api.version>
-        <zstd.version>1.5.6-8</zstd.version>
+        <zstd.version>1.5.6-9</zstd.version>
         <cron-utils.version>9.2.1</cron-utils.version>
         <asciitable.version>0.3.2</asciitable.version>
         <jjwt.version>0.12.6</jjwt.version>
@@ -188,14 +188,14 @@
 
         <!-- Test dependencies -->
         <apacheds-server.version>2.0.0.AM27</apacheds-server.version>
-        <assertj-core.version>3.27.0</assertj-core.version>
+        <assertj-core.version>3.27.2</assertj-core.version>
         <assertj-joda-time.version>2.2.0</assertj-joda-time.version>
         <awaitility.version>4.2.2</awaitility.version>
-        <equalsverifier.version>3.17.5</equalsverifier.version>
+        <equalsverifier.version>3.18</equalsverifier.version>
         <jukito.version>1.5</jukito.version>
         <junit.version>4.13.2</junit.version>
         <junit-jupiter.version>5.11.4</junit-jupiter.version>
-        <mockito.version>5.14.2</mockito.version>
+        <mockito.version>5.15.2</mockito.version>
         <restassured.version>5.5.0</restassured.version>
         <system-rules.version>1.19.0</system-rules.version>
         <testcontainers.version>1.20.4</testcontainers.version>