From 86359e0e5ccf5683eba2e800acaa09bd51f6867d Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Tue, 18 Jul 2023 10:20:54 +0200 Subject: [PATCH] Centralizing shared logic. --- .../elasticsearch7/IndicesAdapterES7.java | 25 ++++++++++- .../opensearch2/IndicesAdapterOS2.java | 32 +++------------ .../indexer/indices/IndicesAdapter.java | 41 +++++++++++++++++-- 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java index 893b863c0347..12c19d66f1f5 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/IndicesAdapterES7.java @@ -43,15 +43,18 @@ import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.GetAliasesResponse; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.Requests; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CloseIndexRequest; +import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.ComponentTemplatesExistRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.ComposableIndexTemplateExistRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.DeleteAliasRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.DeleteComposableIndexTemplateRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.GetMappingsRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.GetMappingsResponse; +import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.PutComponentTemplateRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.PutComposableIndexTemplateRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.PutMappingRequest; import org.graylog.shaded.elasticsearch7.org.elasticsearch.cluster.metadata.AliasMetadata; +import org.graylog.shaded.elasticsearch7.org.elasticsearch.cluster.metadata.ComponentTemplate; import org.graylog.shaded.elasticsearch7.org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.compress.CompressedXContent; import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.settings.Settings; @@ -215,11 +218,29 @@ public Map getIndexMetaData(@Nonnull String index) { } @Override - public boolean ensureIndexTemplate(String templateName, Template template) { + public boolean componentTemplateExists(String templateName) { + var request = new ComponentTemplatesExistRequest(templateName); + return client.execute((c, requestOptions) -> c.cluster().existsComponentTemplate(request, requestOptions)); + } + + @Override + public boolean createComponentTemplate(String templateName, Template template) { var serializedMapping = serialize(template.mappings()); var settings = Settings.builder().loadFromSource(serializeJson(template.settings()), XContentType.JSON).build(); var esTemplate = new org.graylog.shaded.elasticsearch7.org.elasticsearch.cluster.metadata.Template(settings, serializedMapping, null); - var indexTemplate = new ComposableIndexTemplate(template.indexPatterns(), esTemplate, null, template.order(), null, null); + var componentTemplate = new ComponentTemplate(esTemplate, 1L, Map.of()); + var request = new PutComponentTemplateRequest() + .name(templateName) + .componentTemplate(componentTemplate); + final AcknowledgedResponse result = client.execute((c, requestOptions) -> c.cluster().putComponentTemplate(request, requestOptions), + "Unable to create component template " + templateName); + + return result.isAcknowledged(); + } + + @Override + public boolean createComposableIndexTemplate(String templateName, Template template, List composedOf) { + var indexTemplate = new ComposableIndexTemplate(template.indexPatterns(), null, composedOf, template.order(), null, null); var request = new PutComposableIndexTemplateRequest() .name(templateName) .indexTemplate(indexTemplate); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java index 1d21545b3ad1..01b6a9601525 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndicesAdapterOS2.java @@ -214,12 +214,14 @@ public Map getIndexMetaData(@Nonnull String index) { return Map.of(); } - private boolean componentTemplateExists(String templateName) { + @Override + public boolean componentTemplateExists(String templateName) { var request = new ComponentTemplatesExistRequest(templateName); return client.execute((c, requestOptions) -> c.cluster().existsComponentTemplate(request, requestOptions)); } - private boolean createComponentTemplate(String templateName, Template template) { + @Override + public boolean createComponentTemplate(String templateName, Template template) { var serializedMapping = serialize(template.mappings()); var settings = org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings.builder().loadFromMap(template.settings()).build(); var osTemplate = new org.graylog.shaded.opensearch2.org.opensearch.cluster.metadata.Template(settings, serializedMapping, null); @@ -233,31 +235,9 @@ private boolean createComponentTemplate(String templateName, Template template) return result.isAcknowledged(); } - private boolean ensureComponentTemplate(String templateNameBase, Template template) { - var templateName = templateNameBase + "-base"; - - var baseResult = createComponentTemplate(templateName, template); - if (!baseResult) { - return false; - } - - var overridesComponentName = templateNameBase + "-overrides"; - if (componentTemplateExists(overridesComponentName)) { - return true; - } - - return createComponentTemplate(overridesComponentName, new Template(List.of(), new Template.Mappings(Map.of()), 1L, new Template.Settings(Map.of()))); - } - @Override - public boolean ensureIndexTemplate(String templateName, Template template) { - var componentTemplateBase = templateName + "-base"; - var componentTemplateOverrides = templateName + "-overrides"; - var componentTemplateExists = ensureComponentTemplate(templateName, template); - if (!componentTemplateExists) { - return false; - } - var indexTemplate = new ComposableIndexTemplate(template.indexPatterns(), null, List.of(componentTemplateBase, componentTemplateOverrides), template.order(), null, null); + public boolean createComposableIndexTemplate(String templateName, Template template, List composedOf) { + var indexTemplate = new ComposableIndexTemplate(template.indexPatterns(), null, composedOf, template.order(), null, null); var request = new PutComposableIndexTemplateRequest() .name(templateName) .indexTemplate(indexTemplate); diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/IndicesAdapter.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/IndicesAdapter.java index d673baa47132..eb62d8201d49 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/IndicesAdapter.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/IndicesAdapter.java @@ -33,6 +33,9 @@ import java.util.function.Consumer; public interface IndicesAdapter { + String templateBaseSuffix = "-base"; + String templateOverridesSuffic = "-overrides"; + void move(String source, String target, Consumer resultCallback); void delete(String indexName); @@ -51,14 +54,46 @@ public interface IndicesAdapter { /** * Updates the metadata field (_meta) of an index mapping - * @param indexName existing index name - * @param metaData the new metadata + * + * @param indexName existing index name + * @param metaData the new metadata * @param mergeExisting merge or overwrite existing metadata */ void updateIndexMetaData(@Nonnull String indexName, @Nonnull Map metaData, boolean mergeExisting); + Map getIndexMetaData(@Nonnull String indexName); - boolean ensureIndexTemplate(String templateName, Template template); + boolean componentTemplateExists(String templateName); + + boolean createComponentTemplate(String templateName, Template template); + + default boolean ensureComponentTemplate(String templateNameBase, Template template) { + var templateName = templateNameBase + templateBaseSuffix; + + var baseResult = createComponentTemplate(templateName, template); + if (!baseResult) { + return false; + } + + var overridesComponentName = templateNameBase + templateOverridesSuffic; + if (componentTemplateExists(overridesComponentName)) { + return true; + } + + return createComponentTemplate(overridesComponentName, new Template(List.of(), new Template.Mappings(Map.of()), 1L, new Template.Settings(Map.of()))); + } + + boolean createComposableIndexTemplate(String templateName, Template template, List composedOf); + + default boolean ensureIndexTemplate(String templateName, Template template) { + var componentTemplateBase = templateName + templateBaseSuffix; + var componentTemplateOverrides = templateName + templateOverridesSuffic; + var componentTemplateExists = ensureComponentTemplate(templateName, template); + if (!componentTemplateExists) { + return false; + } + return createComposableIndexTemplate(templateName, template, List.of(componentTemplateBase, componentTemplateOverrides)); + } boolean indexTemplateExists(String templateName);