Skip to content

Commit

Permalink
Added filtering on query and metadata map
Browse files Browse the repository at this point in the history 
Signed-off-by: Joshua Au <[email protected]>
  • Loading branch information
@Joshua152
Joshua152 committed Oct 26, 2023
1 parent c58d8d1 commit 2be9253
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 44 deletions.
3 changes: 2 additions & 1 deletion ...ain/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde
import java.io.ByteArrayInputStream
import java.nio.charset.StandardCharsets

data class StateMetaData(
data class
StateMetaData(
val name: String,
val startTime: Long
) : Writeable, ToXContentFragment {
Expand Down
93 changes: 68 additions & 25 deletions src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,72 +5,112 @@

package org.opensearch.indexmanagement.indexstatemanagement.model

import org.apache.logging.log4j.LogManager
import org.opensearch.core.common.io.stream.StreamInput
import org.opensearch.core.common.io.stream.StreamOutput
import org.opensearch.core.common.io.stream.Writeable
import org.opensearch.core.xcontent.XContentParser
import org.opensearch.core.xcontent.XContentParser.Token
import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken
import org.opensearch.index.query.BoolQueryBuilder
import org.opensearch.index.query.Operator
import org.opensearch.index.query.QueryBuilders
import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_POLICY_ID_FIELD
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData
import java.io.IOException

private val log = LogManager.getLogger(ExplainFilter::class.java)

data class ExplainFilter(
val filterPolicyID: String?,
val filterState: String?,
val filterAction: String?
val policyID: String?,
val state: String?,
val actionType: String?,
val failed: Boolean?
) : Writeable {

constructor() : this(
filterPolicyID = null,
filterState = null,
filterAction = null
policyID = null,
state = null,
actionType = null,
failed = null
)

@Throws(IOException::class)
constructor(sin: StreamInput) : this(
filterPolicyID = sin.readOptionalString(),
filterState = sin.readOptionalString(),
filterAction = sin.readOptionalString()
policyID = sin.readString(),
state = sin.readString(),
actionType = sin.readString(),
failed = sin.readBoolean()
)

@Throws(IOException::class)
override fun writeTo(out: StreamOutput) {
out.writeOptionalString(filterPolicyID)
out.writeOptionalString(filterState)
out.writeOptionalString(filterAction)
out.writeOptionalString(policyID)
out.writeOptionalString(state)
out.writeOptionalString(actionType)
out.writeOptionalBoolean(failed)
}

fun convertToBoolQueryBuilder(): BoolQueryBuilder {
val boolQuery = QueryBuilders.boolQuery()

if (policyID != null) {
boolQuery.must(
QueryBuilders
.queryStringQuery(policyID)
.field(MANAGED_INDEX_POLICY_ID_FIELD)
.defaultOperator(Operator.AND)
)
}

return boolQuery
}

fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean {
var valid = true
fun validMetaData(metaData: ManagedIndexMetaData): Boolean {
log.info("----FILTER---- $metaData") // : ${metaData.stateMetaData!!.name} ${metaData.actionMetaData!!.name} ${metaData.actionMetaData!!.failed}")

if (filterPolicyID != null && filterPolicyID != managedIndexConfig.policyID) {
valid = false
val stateMetaData = metaData.stateMetaData
if (stateMetaData != null) {
log.info("STATE META DATA: ${stateMetaData.name} : $state")
}

val policy = managedIndexConfig.policy
if (state != null && (stateMetaData == null || stateMetaData.name != state)) {
log.info("filter error on state")
return false
}

if (filterState != null && policy != null && !policy.states.any { it.name == filterState }) {
valid = false
val actionMetaData = metaData.actionMetaData
if (actionMetaData != null) {
log.info("ACTION META DATA: ${actionMetaData.name} : $actionType, ${actionMetaData.failed} : $failed")
}

if (filterAction != null && policy != null && !policy.states.any { state -> state.actions.any { it.type == filterAction } }) {
valid = false
if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) {
log.info("filter error on action")
return false
}

return valid
if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) {
log.info("filter error on failed")
return false
}

return true
}

companion object {
const val FILTER_FIELD = "filter"
const val POLICY_ID_FIELD = "policy_id"
const val STATE_FIELD = "state"
const val ACTION_FIELD = "action_type"
const val FAILURE_FIELD = "failure"

@JvmStatic
@Throws(IOException::class)
fun parse(xcp: XContentParser): ExplainFilter {
var policyID: String? = null
var state: String? = null
var action: String? = null
var actionType: String? = null
var failed: Boolean? = null

ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp)
while (xcp.nextToken() != Token.END_OBJECT) {
Expand All @@ -87,14 +127,17 @@ data class ExplainFilter(
when (filter) {
POLICY_ID_FIELD -> policyID = xcp.text()
STATE_FIELD -> state = xcp.text()
ACTION_FIELD -> action = xcp.text()
ACTION_FIELD -> actionType = xcp.text()
FAILURE_FIELD -> failed = xcp.booleanValue()
}
}
}
}
}

return ExplainFilter(policyID, state, action)
log.info("[PARSE] pid: $policyID, s: $state, a: $actionType, f: $failed")

return ExplainFilter(policyID, state, actionType, failed)
}
}
}
67 changes: 49 additions & 18 deletions ...h/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,24 @@ class TransportExplainAction @Inject constructor(
private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List<String>, searchSize: Int): SearchRequest {
val sortBuilder = params.getSortBuilder()

val queryBuilder = QueryBuilders.boolQuery()
// change query builder based off of the given json body
// val queryBuilder = QueryBuilders.boolQuery()
// .must(
// QueryBuilders
// .queryStringQuery(params.queryString)
// .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD)
// .defaultOperator(Operator.AND)
// ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs))

log.info("QUERY STRING: ${params.queryString} :: $params")

// can I string multiple bool queries together

val queryBuilder = request.explainFilter.convertToBoolQueryBuilder()
.must(
QueryBuilders
.queryStringQuery(params.queryString)
.defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD)
.field(MANAGED_INDEX_NAME_KEYWORD_FIELD)
.defaultOperator(Operator.AND)
).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs))

Expand Down Expand Up @@ -190,21 +203,19 @@ class TransportExplainAction @Inject constructor(
}

parseSearchHits(response.hits.hits).forEach { managedIndex ->
if (request.explainFilter.isValid(managedIndex)) {
managedIndices.add(managedIndex.index)
enabledState[managedIndex.index] = managedIndex.enabled
managedIndicesMetaDataMap[managedIndex.index] = mapOf(
"index" to managedIndex.index,
"index_uuid" to managedIndex.indexUuid,
"policy_id" to managedIndex.policyID,
"enabled" to managedIndex.enabled.toString()
)
if (showPolicy) {
managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it }
}
if (validateAction) {
managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it }
}
managedIndices.add(managedIndex.index)
enabledState[managedIndex.index] = managedIndex.enabled
managedIndicesMetaDataMap[managedIndex.index] = mapOf(
"index" to managedIndex.index,
"index_uuid" to managedIndex.indexUuid,
"policy_id" to managedIndex.policyID,
"enabled" to managedIndex.enabled.toString()
)
if (showPolicy) {
managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it }
}
if (validateAction) {
managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it }
}
}

Expand Down Expand Up @@ -293,8 +304,28 @@ class TransportExplainAction @Inject constructor(
mgetMetadataReq,
object : ActionListener<MultiGetResponse> {
override fun onResponse(response: MultiGetResponse) {
val metadataMap: Map<ManagedIndexMetadataDocUUID, ManagedIndexMetadataMap?> =
var metadataMap: Map<ManagedIndexMetadataDocUUID, ManagedIndexMetadataMap?> =
response.responses.associate { it.id to getMetadata(it.response)?.toMap() }

log.info("BEFORE METADATA MAP: $metadataMap")

metadataMap = metadataMap.filter { (_, value) ->
var ok = false

if (value != null) {
val metaData = ManagedIndexMetaData.fromMap(value)
ok = request.explainFilter.validMetaData(metaData)
if (!ok) {
indexNamesToUUIDs.remove(metaData.index)
}
}

ok
}

log.info("FILTERED METADATA MAP: $metadataMap")
log.info("FILTED INDEX NAMES TO UUID: $indexNamesToUUIDs")

buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext)
}

Expand Down
1 change: 1 addition & 0 deletions src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const val MANAGED_INDEX_FIELD = "managed_index"
const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword"
const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index"
const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid"
const val MANAGED_INDEX_POLICY_ID_FIELD = "$MANAGED_INDEX_FIELD.policy_id"

const val DEFAULT_JOB_SORT_FIELD = MANAGED_INDEX_INDEX_FIELD
const val DEFAULT_POLICY_SORT_FIELD = "policy.policy_id.keyword"
Expand Down

0 comments on commit 2be9253

Please sign in to comment.