diff --git a/changelog/unreleased/pr-21208.toml b/changelog/unreleased/pr-21208.toml new file mode 100644 index 000000000000..eb3b0b8ff1b8 --- /dev/null +++ b/changelog/unreleased/pr-21208.toml @@ -0,0 +1,4 @@ +type = "f" +message = "batching request for index block status if the combined length of the indices exceed the max possible URL length " + +pulls = ["21208"] diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java index 5a82c91e9242..7d604e8461db 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java @@ -109,6 +109,11 @@ public class IndicesAdapterES7 implements IndicesAdapter { private final ClusterStateApi clusterStateApi; private final IndexTemplateAdapter indexTemplateAdapter; + // this is the maximum amount of bytes that the index list is supposed to fill in a request, + // it assumes that these don't need url encoding. If we exceed the maximum, we request settings for all indices + // and filter after wards + private final int MAX_INDICES_URL_LENGTH = 3000; + @Inject public IndicesAdapterES7(ElasticsearchClient client, StatsApi statsApi, @@ -435,14 +440,17 @@ public IndicesBlockStatus getIndicesBlocksStatus(final List indices) { if (indices == null || indices.isEmpty()) { throw new IllegalArgumentException("Expecting list of indices with at least one index present."); } - final GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices(indices.toArray(new String[]{})) + + final GetSettingsRequest request = new GetSettingsRequest() .indicesOptions(IndicesOptions.fromOptions(false, true, true, true)) - .names(new String[]{}); + .names("index.blocks.read", "index.blocks.write", "index.blocks.metadata", "index.blocks.read_only", "index.blocks.read_only_allow_delete"); + + final var maxLengthExceeded = String.join(",", indices).length() > MAX_INDICES_URL_LENGTH; + final GetSettingsRequest getSettingsRequest = maxLengthExceeded ? request : request.indices(indices.toArray(new String[]{})); return client.execute((c, requestOptions) -> { final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions); - return BlockSettingsParser.parseBlockSettings(settingsResponse); + return BlockSettingsParser.parseBlockSettings(settingsResponse, maxLengthExceeded ? Optional.of(indices) : Optional.empty()); }); } diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/blocks/BlockSettingsParser.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/blocks/BlockSettingsParser.java index a735b957784b..f073d6121e46 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/blocks/BlockSettingsParser.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/blocks/BlockSettingsParser.java @@ -21,6 +21,9 @@ import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.settings.Settings; import org.graylog2.indexer.indices.blocks.IndicesBlockStatus; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,23 +32,31 @@ public class BlockSettingsParser { static final String BLOCK_SETTINGS_PREFIX = "index.blocks."; public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse) { - IndicesBlockStatus result = new IndicesBlockStatus(); + return parseBlockSettings(settingsResponse, Optional.empty()); + } + + public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse, final Optional> indices) { + final IndicesBlockStatus result = new IndicesBlockStatus(); final ImmutableOpenMap indexToSettingsMap = settingsResponse.getIndexToSettings(); final String[] indicesInResponse = indexToSettingsMap.keys().toArray(String.class); - for (String index : indicesInResponse) { - final Settings blockSettings = indexToSettingsMap.get(index).getByPrefix(BLOCK_SETTINGS_PREFIX); - - if (!blockSettings.isEmpty()) { - final Set blockSettingsNames = blockSettings.names(); - final Set blockSettingsSetToTrue = blockSettingsNames.stream() - .filter(s -> blockSettings.getAsBoolean(s, false)) - .map(s -> BLOCK_SETTINGS_PREFIX + s) - .collect(Collectors.toSet()); - if (!blockSettingsSetToTrue.isEmpty()) { - result.addIndexBlocks(index, blockSettingsSetToTrue); + + indices.orElse(Arrays.stream(indicesInResponse).toList()).forEach(index -> { + final var settings = indexToSettingsMap.get(index); + if(settings != null) { + final Settings blockSettings = settings.getByPrefix(BLOCK_SETTINGS_PREFIX); + + if (!blockSettings.isEmpty()) { + final Set blockSettingsNames = blockSettings.names(); + final Set blockSettingsSetToTrue = blockSettingsNames.stream() + .filter(s -> blockSettings.getAsBoolean(s, false)) + .map(s -> BLOCK_SETTINGS_PREFIX + s) + .collect(Collectors.toSet()); + if (!blockSettingsSetToTrue.isEmpty()) { + result.addIndexBlocks(index, blockSettingsSetToTrue); + } } } - } + }); return result; } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java index d015335b44e9..d79852836d72 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java @@ -84,6 +84,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -97,6 +98,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static org.graylog.storage.opensearch2.OpenSearchClient.withTimeout; @@ -109,6 +111,11 @@ public class IndicesAdapterOS2 implements IndicesAdapter { private final ClusterStateApi clusterStateApi; private final IndexTemplateAdapter indexTemplateAdapter; + // this is the maximum amount of bytes that the index list is supposed to fill in a request, + // it assumes that these don't need url encoding. If we exceed the maximum, we request settings for all indices + // and filter after wards + private final int MAX_INDICES_URL_LENGTH = 3000; + @Inject public IndicesAdapterOS2(OpenSearchClient client, StatsApi statsApi, @@ -431,19 +438,23 @@ public List getShardsInfo(String indexName) { return catApi.getShardsInfo(indexName); } + @Override public IndicesBlockStatus getIndicesBlocksStatus(final List indices) { if (indices == null || indices.isEmpty()) { throw new IllegalArgumentException("Expecting list of indices with at least one index present."); } - final GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices(indices.toArray(new String[]{})) + + final GetSettingsRequest request = new GetSettingsRequest() .indicesOptions(IndicesOptions.fromOptions(false, true, true, true)) - .names(new String[]{}); + .names("index.blocks.read", "index.blocks.write", "index.blocks.metadata", "index.blocks.read_only", "index.blocks.read_only_allow_delete"); + + final var maxLengthExceeded = String.join(",", indices).length() > MAX_INDICES_URL_LENGTH; + final GetSettingsRequest getSettingsRequest = maxLengthExceeded ? request : request.indices(indices.toArray(new String[]{})); return client.execute((c, requestOptions) -> { final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions); - return BlockSettingsParser.parseBlockSettings(settingsResponse); + return BlockSettingsParser.parseBlockSettings(settingsResponse, maxLengthExceeded ? Optional.of(indices) : Optional.empty()); }); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParser.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParser.java index c7f6da23aa98..84ba9cbfe114 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParser.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParser.java @@ -20,6 +20,8 @@ import org.graylog.shaded.opensearch2.org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings; +import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -28,23 +30,30 @@ public class BlockSettingsParser { static final String BLOCK_SETTINGS_PREFIX = "index.blocks."; public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse) { - IndicesBlockStatus result = new IndicesBlockStatus(); + return parseBlockSettings(settingsResponse, Optional.empty()); + } + + public static IndicesBlockStatus parseBlockSettings(final GetSettingsResponse settingsResponse, final Optional> indices) { + final IndicesBlockStatus result = new IndicesBlockStatus(); final var indexToSettingsMap = settingsResponse.getIndexToSettings(); - final String[] indicesInResponse = indexToSettingsMap.keySet().toArray(new String[0]); - for (String index : indicesInResponse) { - final Settings blockSettings = indexToSettingsMap.get(index).getByPrefix(BLOCK_SETTINGS_PREFIX); - - if (!blockSettings.isEmpty()) { - final Set blockSettingsNames = blockSettings.names(); - final Set blockSettingsSetToTrue = blockSettingsNames.stream() - .filter(s -> blockSettings.getAsBoolean(s, false)) - .map(s -> BLOCK_SETTINGS_PREFIX + s) - .collect(Collectors.toSet()); - if (!blockSettingsSetToTrue.isEmpty()) { - result.addIndexBlocks(index, blockSettingsSetToTrue); + + indices.orElse(indexToSettingsMap.keySet().stream().toList()).forEach(index -> { + final var settings = indexToSettingsMap.get(index); + if(settings != null) { + final Settings blockSettings = settings.getByPrefix(BLOCK_SETTINGS_PREFIX); + + if (!blockSettings.isEmpty()) { + final Set blockSettingsNames = blockSettings.names(); + final Set blockSettingsSetToTrue = blockSettingsNames.stream() + .filter(s -> blockSettings.getAsBoolean(s, false)) + .map(s -> BLOCK_SETTINGS_PREFIX + s) + .collect(Collectors.toSet()); + if (!blockSettingsSetToTrue.isEmpty()) { + result.addIndexBlocks(index, blockSettingsSetToTrue); + } } } - } + }); return result; } diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParserTest.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParserTest.java index b9382514c989..53123e253208 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParserTest.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/blocks/BlockSettingsParserTest.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; @@ -35,7 +36,7 @@ public class BlockSettingsParserTest { @Test public void noBlockedIndicesIdentifiedIfEmptyResponseParsed() { GetSettingsResponse emptyResponse = new GetSettingsResponse(Map.of(), Map.of()); - final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptyResponse); + final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptyResponse, Optional.empty()); assertNotNull(indicesBlockStatus); assertEquals(0, indicesBlockStatus.countBlockedIndices()); } @@ -44,7 +45,7 @@ public void noBlockedIndicesIdentifiedIfEmptyResponseParsed() { public void noBlockedIndicesIdentifiedIfEmptySettingsPresent() { var settingsBuilder = Map.of("index_0", Settings.builder().build()); GetSettingsResponse emptySettingsResponse = new GetSettingsResponse(settingsBuilder, Map.of()); - final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptySettingsResponse); + final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(emptySettingsResponse, Optional.empty()); assertNotNull(indicesBlockStatus); assertEquals(0, indicesBlockStatus.countBlockedIndices()); } @@ -64,7 +65,7 @@ public void parserProperlyResponseWithMultipleIndicesWithDifferentBlockSettings( .put("index.blocks.read_only_allow_delete", true) .build()); GetSettingsResponse settingsResponse = new GetSettingsResponse(settingsBuilder, Map.of()); - final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(settingsResponse); + final IndicesBlockStatus indicesBlockStatus = BlockSettingsParser.parseBlockSettings(settingsResponse, Optional.empty()); assertNotNull(indicesBlockStatus); assertEquals(3, indicesBlockStatus.countBlockedIndices()); final Set blockedIndices = indicesBlockStatus.getBlockedIndices(); diff --git a/graylog2-web-interface/packages/eslint-config-graylog/package.json b/graylog2-web-interface/packages/eslint-config-graylog/package.json index 8779bcccf8dd..37b480067fe8 100644 --- a/graylog2-web-interface/packages/eslint-config-graylog/package.json +++ b/graylog2-web-interface/packages/eslint-config-graylog/package.json @@ -15,8 +15,8 @@ "dependencies": { "@babel/eslint-parser": "7.16.5", "@tanstack/eslint-plugin-query": "4.36.1", - "@typescript-eslint/eslint-plugin": "8.19.0", - "@typescript-eslint/parser": "8.19.0", + "@typescript-eslint/eslint-plugin": "8.19.1", + "@typescript-eslint/parser": "8.19.1", "eslint": "8.57.0", "eslint-config-airbnb": "19.0.4", "eslint-import-resolver-webpack": "0.13.10", diff --git a/pom.xml b/pom.xml index dae4fddcbbe4..676618629f10 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ 3.1.8 0.0.1.10 4.8.179 - 1.17.1 + 1.17.2 1.12.0 1.6.0 1.4.0 @@ -117,7 +117,7 @@ 2.18.0 4.0.0 2.36.0 - 2.3.33 + 2.3.34 1.5.1 4.2.1 0.1.9-graylog-3 @@ -144,7 +144,7 @@ 7.0.3 2.13.0 0.9.15 - 20240303 + 20241224 2.9.0 3.9.0 0.9.0.1-7 @@ -179,7 +179,7 @@ 7.0.2 3.2.1 3.1.0 - 1.5.6-8 + 1.5.6-9 9.2.1 0.3.2 0.12.6 @@ -188,14 +188,14 @@ 2.0.0.AM27 - 3.27.0 + 3.27.2 2.2.0 4.2.2 - 3.17.5 + 3.18 1.5 4.13.2 5.11.4 - 5.14.2 + 5.15.2 5.5.0 1.19.0 1.20.4