From c06f95d7d83abc1873eb40da5c2fa303bab3f506 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Tue, 10 Oct 2023 13:05:54 -0400 Subject: [PATCH 1/5] Implemented filtering on the ISM eplain API Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 100 ++++++++++++++++++ .../resthandler/RestExplainAction.kt | 21 ++++ .../action/explain/ExplainRequest.kt | 6 ++ .../action/explain/TransportExplainAction.kt | 28 ++--- 4 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt new file mode 100644 index 000000000..8fb09d7e7 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.model + +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import java.io.IOException + +data class ExplainFilter( + val filterPolicyID: String?, + val filterState: String?, + val filterAction: String? +) : Writeable { + + constructor() : this( + filterPolicyID = null, + filterState = null, + filterAction = null + ) + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + filterPolicyID = sin.readString(), + filterState = sin.readString(), + filterAction = sin.readString() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(filterPolicyID) + out.writeString(filterState) + out.writeString(filterAction) + } + + fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { + var valid = true + + if (filterPolicyID != null && filterPolicyID != managedIndexConfig.policyID) { + valid = false + } + + val policy = managedIndexConfig.policy + + if (filterState != null && policy != null && !policy.states.any { it.name == filterState }) { + valid = false + } + + if (filterAction != null && policy != null && !policy.states.any { state -> state.actions.any { it.type == filterAction } }) { + valid = false + } + + return valid + } + + companion object { + const val FILTER_FIELD = "filter" + const val POLICY_ID_FIELD = "policy_id" + const val STATE_FIELD = "state" + const val ACTION_FIELD = "action_type" + + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser): ExplainFilter { + var policyID: String? = null + var state: String? = null + var action: String? = null + + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + FILTER_FIELD -> { + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val filter = xcp.currentName() + xcp.nextToken() + + when (filter) { + POLICY_ID_FIELD -> policyID = xcp.text() + STATE_FIELD -> state = xcp.text() + ACTION_FIELD -> action = xcp.text() + } + } + } + } + } + + return ExplainFilter(policyID, state, action) + } + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt index c9bde8bbf..68f8c353c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt @@ -9,8 +9,11 @@ import org.apache.logging.log4j.LogManager import org.opensearch.client.node.NodeClient import org.opensearch.core.common.Strings import org.opensearch.common.logging.DeprecationLogger +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainRequest import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_EXPLAIN_VALIDATE_ACTION @@ -28,6 +31,7 @@ import org.opensearch.rest.RestHandler.ReplacedRoute import org.opensearch.rest.RestHandler.Route import org.opensearch.rest.RestRequest import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.POST import org.opensearch.rest.action.RestToXContentListener private val log = LogManager.getLogger(RestExplainAction::class.java) @@ -52,6 +56,14 @@ class RestExplainAction : BaseRestHandler() { ReplacedRoute( GET, "$EXPLAIN_BASE_URI/{index}", GET, "$LEGACY_EXPLAIN_BASE_URI/{index}" + ), + ReplacedRoute( + POST, EXPLAIN_BASE_URI, + POST, LEGACY_EXPLAIN_BASE_URI + ), + ReplacedRoute( + POST, "$EXPLAIN_BASE_URI/{index}", + POST, "$LEGACY_EXPLAIN_BASE_URI/{index}" ) ) } @@ -69,6 +81,14 @@ class RestExplainAction : BaseRestHandler() { val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val explainFilter = if (request.method() == RestRequest.Method.POST) { + val xcp = request.contentParser() + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + ExplainFilter.parse(xcp) + } else { + ExplainFilter() + } + val clusterManagerTimeout = parseClusterManagerTimeout( request, DeprecationLogger.getLogger(RestExplainAction::class.java), name ) @@ -78,6 +98,7 @@ class RestExplainAction : BaseRestHandler() { request.paramAsBoolean("local", false), clusterManagerTimeout, searchParams, + explainFilter, request.paramAsBoolean(SHOW_POLICY_QUERY_PARAM, DEFAULT_EXPLAIN_SHOW_POLICY), request.paramAsBoolean(SHOW_VALIDATE_ACTION, DEFAULT_EXPLAIN_VALIDATE_ACTION), indexType diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt index a34ce2020..ba0090442 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt @@ -12,6 +12,7 @@ import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.common.model.rest.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException @@ -21,6 +22,7 @@ class ExplainRequest : ActionRequest { val local: Boolean val clusterManagerTimeout: TimeValue val searchParams: SearchParams + val explainFilter: ExplainFilter val showPolicy: Boolean val validateAction: Boolean val indexType: String @@ -31,6 +33,7 @@ class ExplainRequest : ActionRequest { local: Boolean, clusterManagerTimeout: TimeValue, searchParams: SearchParams, + explainFilter: ExplainFilter, showPolicy: Boolean, validateAction: Boolean, indexType: String @@ -39,6 +42,7 @@ class ExplainRequest : ActionRequest { this.local = local this.clusterManagerTimeout = clusterManagerTimeout this.searchParams = searchParams + this.explainFilter = explainFilter this.showPolicy = showPolicy this.validateAction = validateAction this.indexType = indexType @@ -50,6 +54,7 @@ class ExplainRequest : ActionRequest { local = sin.readBoolean(), clusterManagerTimeout = sin.readTimeValue(), searchParams = SearchParams(sin), + explainFilter = ExplainFilter(sin), showPolicy = sin.readBoolean(), validateAction = sin.readBoolean(), indexType = sin.readString() @@ -72,6 +77,7 @@ class ExplainRequest : ActionRequest { out.writeBoolean(local) out.writeTimeValue(clusterManagerTimeout) searchParams.writeTo(out) + explainFilter.writeTo(out) out.writeBoolean(showPolicy) out.writeBoolean(validateAction) out.writeString(indexType) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index ec32bcbd9..d8f140b7b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -190,19 +190,21 @@ class TransportExplainAction @Inject constructor( } parseSearchHits(response.hits.hits).forEach { managedIndex -> - managedIndices.add(managedIndex.index) - enabledState[managedIndex.index] = managedIndex.enabled - managedIndicesMetaDataMap[managedIndex.index] = mapOf( - "index" to managedIndex.index, - "index_uuid" to managedIndex.indexUuid, - "policy_id" to managedIndex.policyID, - "enabled" to managedIndex.enabled.toString() - ) - if (showPolicy) { - managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } - } - if (validateAction) { - managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } + if (request.explainFilter.isValid(managedIndex)) { + managedIndices.add(managedIndex.index) + enabledState[managedIndex.index] = managedIndex.enabled + managedIndicesMetaDataMap[managedIndex.index] = mapOf( + "index" to managedIndex.index, + "index_uuid" to managedIndex.indexUuid, + "policy_id" to managedIndex.policyID, + "enabled" to managedIndex.enabled.toString() + ) + if (showPolicy) { + managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } + } + if (validateAction) { + managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } + } } } From c58d8d1d17f98e5af6c772fec92b247bba6fb241 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 11 Oct 2023 14:15:52 -0400 Subject: [PATCH 2/5] Fixed tests for ExplainRequest Signed-off-by: Joshua Au --- .../indexstatemanagement/model/ExplainFilter.kt | 12 ++++++------ .../transport/action/explain/ExplainRequestTests.kt | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 8fb09d7e7..ba562948f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -27,16 +27,16 @@ data class ExplainFilter( @Throws(IOException::class) constructor(sin: StreamInput) : this( - filterPolicyID = sin.readString(), - filterState = sin.readString(), - filterAction = sin.readString() + filterPolicyID = sin.readOptionalString(), + filterState = sin.readOptionalString(), + filterAction = sin.readOptionalString() ) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { - out.writeString(filterPolicyID) - out.writeString(filterState) - out.writeString(filterAction) + out.writeOptionalString(filterPolicyID) + out.writeOptionalString(filterState) + out.writeOptionalString(filterAction) } fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt index 0c8d0e01e..832322a1a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt @@ -9,6 +9,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.common.model.rest.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase @@ -19,9 +20,10 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") + val filter = ExplainFilter() val showPolicy = false val showValidationResult = false - val req = ExplainRequest(indices, local, clusterManagerTimeout, params, showPolicy, showValidationResult, DEFAULT_INDEX_TYPE) + val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -36,9 +38,10 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") + val filter = ExplainFilter() val showPolicy = false val showValidationResult = false - val req = ExplainRequest(indices, local, clusterManagerTimeout, params, showPolicy, showValidationResult, "non-existent-index-type") + val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, "non-existent-index-type") val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() val expectedException: String = ExplainRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR From 2be92534800cd3cc578a3e8344963602864f946b Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Thu, 26 Oct 2023 12:33:29 -0400 Subject: [PATCH 3/5] Added filtering on query and metadata map Signed-off-by: Joshua Au --- .../model/StateMetaData.kt | 3 +- .../model/ExplainFilter.kt | 93 ++++++++++++++----- .../action/explain/TransportExplainAction.kt | 67 +++++++++---- .../util/RestHandlerUtils.kt | 1 + 4 files changed, 120 insertions(+), 44 deletions(-) diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt index 7c6174106..21a94d418 100644 --- a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt @@ -22,7 +22,8 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets -data class StateMetaData( +data class +StateMetaData( val name: String, val startTime: Long ) : Writeable, ToXContentFragment { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index ba562948f..83ab63a58 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,58 +5,96 @@ package org.opensearch.indexmanagement.indexstatemanagement.model +import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable import org.opensearch.core.xcontent.XContentParser import org.opensearch.core.xcontent.XContentParser.Token import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilders +import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_POLICY_ID_FIELD +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException +private val log = LogManager.getLogger(ExplainFilter::class.java) + data class ExplainFilter( - val filterPolicyID: String?, - val filterState: String?, - val filterAction: String? + val policyID: String?, + val state: String?, + val actionType: String?, + val failed: Boolean? ) : Writeable { constructor() : this( - filterPolicyID = null, - filterState = null, - filterAction = null + policyID = null, + state = null, + actionType = null, + failed = null ) @Throws(IOException::class) constructor(sin: StreamInput) : this( - filterPolicyID = sin.readOptionalString(), - filterState = sin.readOptionalString(), - filterAction = sin.readOptionalString() + policyID = sin.readString(), + state = sin.readString(), + actionType = sin.readString(), + failed = sin.readBoolean() ) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { - out.writeOptionalString(filterPolicyID) - out.writeOptionalString(filterState) - out.writeOptionalString(filterAction) + out.writeOptionalString(policyID) + out.writeOptionalString(state) + out.writeOptionalString(actionType) + out.writeOptionalBoolean(failed) + } + + fun convertToBoolQueryBuilder(): BoolQueryBuilder { + val boolQuery = QueryBuilders.boolQuery() + + if (policyID != null) { + boolQuery.must( + QueryBuilders + .queryStringQuery(policyID) + .field(MANAGED_INDEX_POLICY_ID_FIELD) + .defaultOperator(Operator.AND) + ) + } + + return boolQuery } - fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { - var valid = true + fun validMetaData(metaData: ManagedIndexMetaData): Boolean { + log.info("----FILTER---- $metaData") // : ${metaData.stateMetaData!!.name} ${metaData.actionMetaData!!.name} ${metaData.actionMetaData!!.failed}") - if (filterPolicyID != null && filterPolicyID != managedIndexConfig.policyID) { - valid = false + val stateMetaData = metaData.stateMetaData + if (stateMetaData != null) { + log.info("STATE META DATA: ${stateMetaData.name} : $state") } - val policy = managedIndexConfig.policy + if (state != null && (stateMetaData == null || stateMetaData.name != state)) { + log.info("filter error on state") + return false + } - if (filterState != null && policy != null && !policy.states.any { it.name == filterState }) { - valid = false + val actionMetaData = metaData.actionMetaData + if (actionMetaData != null) { + log.info("ACTION META DATA: ${actionMetaData.name} : $actionType, ${actionMetaData.failed} : $failed") } - if (filterAction != null && policy != null && !policy.states.any { state -> state.actions.any { it.type == filterAction } }) { - valid = false + if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { + log.info("filter error on action") + return false } - return valid + if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { + log.info("filter error on failed") + return false + } + + return true } companion object { @@ -64,13 +102,15 @@ data class ExplainFilter( const val POLICY_ID_FIELD = "policy_id" const val STATE_FIELD = "state" const val ACTION_FIELD = "action_type" + const val FAILURE_FIELD = "failure" @JvmStatic @Throws(IOException::class) fun parse(xcp: XContentParser): ExplainFilter { var policyID: String? = null var state: String? = null - var action: String? = null + var actionType: String? = null + var failed: Boolean? = null ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { @@ -87,14 +127,17 @@ data class ExplainFilter( when (filter) { POLICY_ID_FIELD -> policyID = xcp.text() STATE_FIELD -> state = xcp.text() - ACTION_FIELD -> action = xcp.text() + ACTION_FIELD -> actionType = xcp.text() + FAILURE_FIELD -> failed = xcp.booleanValue() } } } } } - return ExplainFilter(policyID, state, action) + log.info("[PARSE] pid: $policyID, s: $state, a: $actionType, f: $failed") + + return ExplainFilter(policyID, state, actionType, failed) } } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index d8f140b7b..f12b8bc91 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -155,11 +155,24 @@ class TransportExplainAction @Inject constructor( private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = params.getSortBuilder() - val queryBuilder = QueryBuilders.boolQuery() + // change query builder based off of the given json body +// val queryBuilder = QueryBuilders.boolQuery() +// .must( +// QueryBuilders +// .queryStringQuery(params.queryString) +// .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) +// .defaultOperator(Operator.AND) +// ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) + + log.info("QUERY STRING: ${params.queryString} :: $params") + + // can I string multiple bool queries together + + val queryBuilder = request.explainFilter.convertToBoolQueryBuilder() .must( QueryBuilders .queryStringQuery(params.queryString) - .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) + .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) .defaultOperator(Operator.AND) ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) @@ -190,21 +203,19 @@ class TransportExplainAction @Inject constructor( } parseSearchHits(response.hits.hits).forEach { managedIndex -> - if (request.explainFilter.isValid(managedIndex)) { - managedIndices.add(managedIndex.index) - enabledState[managedIndex.index] = managedIndex.enabled - managedIndicesMetaDataMap[managedIndex.index] = mapOf( - "index" to managedIndex.index, - "index_uuid" to managedIndex.indexUuid, - "policy_id" to managedIndex.policyID, - "enabled" to managedIndex.enabled.toString() - ) - if (showPolicy) { - managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } - } - if (validateAction) { - managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } - } + managedIndices.add(managedIndex.index) + enabledState[managedIndex.index] = managedIndex.enabled + managedIndicesMetaDataMap[managedIndex.index] = mapOf( + "index" to managedIndex.index, + "index_uuid" to managedIndex.indexUuid, + "policy_id" to managedIndex.policyID, + "enabled" to managedIndex.enabled.toString() + ) + if (showPolicy) { + managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } + } + if (validateAction) { + managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } } } @@ -293,8 +304,28 @@ class TransportExplainAction @Inject constructor( mgetMetadataReq, object : ActionListener { override fun onResponse(response: MultiGetResponse) { - val metadataMap: Map = + var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } + + log.info("BEFORE METADATA MAP: $metadataMap") + + metadataMap = metadataMap.filter { (_, value) -> + var ok = false + + if (value != null) { + val metaData = ManagedIndexMetaData.fromMap(value) + ok = request.explainFilter.validMetaData(metaData) + if (!ok) { + indexNamesToUUIDs.remove(metaData.index) + } + } + + ok + } + + log.info("FILTERED METADATA MAP: $metadataMap") + log.info("FILTED INDEX NAMES TO UUID: $indexNamesToUUIDs") + buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index 4442ccc82..6c5a45694 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -39,6 +39,7 @@ const val MANAGED_INDEX_FIELD = "managed_index" const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword" const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index" const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid" +const val MANAGED_INDEX_POLICY_ID_FIELD = "$MANAGED_INDEX_FIELD.policy_id" const val DEFAULT_JOB_SORT_FIELD = MANAGED_INDEX_INDEX_FIELD const val DEFAULT_POLICY_SORT_FIELD = "policy.policy_id.keyword" From 73af3b7d1040ea073e1a3984d1dfb69b60ed7fff Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Thu, 2 Nov 2023 12:54:22 -0400 Subject: [PATCH 4/5] Filtered on indexNames in metadata Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 22 ++----------------- .../action/explain/TransportExplainAction.kt | 7 ++---- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 83ab63a58..a7301eac8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,7 +5,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.model -import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable @@ -19,8 +18,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_PO import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException -private val log = LogManager.getLogger(ExplainFilter::class.java) - data class ExplainFilter( val policyID: String?, val state: String?, @@ -67,30 +64,17 @@ data class ExplainFilter( } fun validMetaData(metaData: ManagedIndexMetaData): Boolean { - log.info("----FILTER---- $metaData") // : ${metaData.stateMetaData!!.name} ${metaData.actionMetaData!!.name} ${metaData.actionMetaData!!.failed}") - val stateMetaData = metaData.stateMetaData - if (stateMetaData != null) { - log.info("STATE META DATA: ${stateMetaData.name} : $state") - } - if (state != null && (stateMetaData == null || stateMetaData.name != state)) { - log.info("filter error on state") return false } val actionMetaData = metaData.actionMetaData - if (actionMetaData != null) { - log.info("ACTION META DATA: ${actionMetaData.name} : $actionType, ${actionMetaData.failed} : $failed") - } - if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { - log.info("filter error on action") return false } if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { - log.info("filter error on failed") return false } @@ -102,7 +86,7 @@ data class ExplainFilter( const val POLICY_ID_FIELD = "policy_id" const val STATE_FIELD = "state" const val ACTION_FIELD = "action_type" - const val FAILURE_FIELD = "failure" + const val FAILED_FIELD = "failed" @JvmStatic @Throws(IOException::class) @@ -128,15 +112,13 @@ data class ExplainFilter( POLICY_ID_FIELD -> policyID = xcp.text() STATE_FIELD -> state = xcp.text() ACTION_FIELD -> actionType = xcp.text() - FAILURE_FIELD -> failed = xcp.booleanValue() + FAILED_FIELD -> failed = xcp.booleanValue() } } } } } - log.info("[PARSE] pid: $policyID, s: $state, a: $actionType, f: $failed") - return ExplainFilter(policyID, state, actionType, failed) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index f12b8bc91..6bfdc2927 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -307,8 +307,6 @@ class TransportExplainAction @Inject constructor( var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } - log.info("BEFORE METADATA MAP: $metadataMap") - metadataMap = metadataMap.filter { (_, value) -> var ok = false @@ -316,16 +314,15 @@ class TransportExplainAction @Inject constructor( val metaData = ManagedIndexMetaData.fromMap(value) ok = request.explainFilter.validMetaData(metaData) if (!ok) { + indexNames.remove(metaData.index) indexNamesToUUIDs.remove(metaData.index) + totalManagedIndices-- } } ok } - log.info("FILTERED METADATA MAP: $metadataMap") - log.info("FILTED INDEX NAMES TO UUID: $indexNamesToUUIDs") - buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) } From 871952ca08d1111685b20d0fb33694171e85dedc Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 8 Nov 2023 00:04:20 -0500 Subject: [PATCH 5/5] Fixed github workflow check errors Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index a7301eac8..75856b566 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -34,10 +34,10 @@ data class ExplainFilter( @Throws(IOException::class) constructor(sin: StreamInput) : this( - policyID = sin.readString(), - state = sin.readString(), - actionType = sin.readString(), - failed = sin.readBoolean() + policyID = sin.readOptionalString(), + state = sin.readOptionalString(), + actionType = sin.readOptionalString(), + failed = sin.readOptionalBoolean() ) @Throws(IOException::class) @@ -64,21 +64,23 @@ data class ExplainFilter( } fun validMetaData(metaData: ManagedIndexMetaData): Boolean { + var ok = true + val stateMetaData = metaData.stateMetaData if (state != null && (stateMetaData == null || stateMetaData.name != state)) { - return false + ok = false } val actionMetaData = metaData.actionMetaData if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { - return false + ok = false } if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { - return false + ok = false } - return true + return ok } companion object {