Merge branch 'opensearch-project:main' into main

.github/workflows/bwc-test-workflow.yml

@@ -9,29 +9,39 @@ on:
       - "*"
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: ./.github/workflows/get-ci-image-tag.yml
+    with:
+      platform: centos7
+      usage: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [ 11 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
       - name: Run Alerting Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests..."
-          ./gradlew bwcTestSuite
+          chown -R opensearch.opensearch `pwd`
+          su opensearch -c "whoami && java -version && ./gradlew bwcTestSuite"

.github/workflows/multi-node-test-workflow.yml

@@ -9,14 +9,27 @@ on:
       - "*"
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [ 11, 17 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -27,4 +40,6 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -PnumNodes=3"

.github/workflows/test-workflow.yml

@@ -9,14 +9,68 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
+    env:
+      BUILD_ARGS: ${{ matrix.os_build_args }}
+      WORKING_DIR: ${{ matrix.working_directory }}.
+    strategy:
+      matrix:
+        java: [11, 17]
+    # Job name
+    name: Build Alerting with JDK ${{ matrix.java }} on Linux
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Build and run with Gradle
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew assemble integTest"
+      - name: Create Artifact Path
+        run: |
+          mkdir -p alerting-artifacts
+          cp ./alerting/build/distributions/*.zip alerting-artifacts
+      # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: alerting-plugin-${{ matrix.os }}
+          path: alerting-artifacts
+
   build:
+    needs: Get-CI-Image-Tag
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
         java: [11, 17]
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
             os_build_args: -x integTest
@@ -49,12 +103,6 @@ jobs:
         run: |
           mkdir -p alerting-artifacts
           cp ./alerting/build/distributions/*.zip alerting-artifacts
-      # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
-      - name: Upload Coverage Report
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
       - name: Upload Artifacts
         uses: actions/upload-artifact@v1

alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt

@@ -139,7 +139,8 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             )
 
             // cleanup old indices that are not monitored anymore from the same monitor
-            for (ind in updatedLastRunContext.keys) {
+            val runContextKeys = updatedLastRunContext.keys.toMutableSet()
+            for (ind in runContextKeys) {
                 if (!concreteIndices.contains(ind)) {
                     updatedLastRunContext.remove(ind)
                 }

alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt

@@ -1273,6 +1273,51 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         }
     }
 
+    fun `test execute monitor with indices removed after first run`() {
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val index1 = createTestIndex()
+        val index2 = createTestIndex()
+        val index4 = createTestIndex()
+        val index5 = createTestIndex()
+
+        val docQuery = DocLevelQuery(query = "\"us-west-2\"", fields = listOf(), name = "3")
+        var docLevelInput = DocLevelMonitorInput("description", listOf(index1, index2, index4, index5), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        indexDoc(index1, "1", testDoc)
+        indexDoc(index2, "1", testDoc)
+        indexDoc(index4, "1", testDoc)
+        indexDoc(index5, "1", testDoc)
+
+        var response = executeMonitor(monitor.id)
+
+        var output = entityAsMap(response)
+        assertEquals(monitor.name, output["monitor_name"])
+
+        assertEquals(1, output.objectMap("trigger_results").values.size)
+        deleteIndex(index1)
+        deleteIndex(index2)
+
+        indexDoc(index4, "1", testDoc)
+        response = executeMonitor(monitor.id)
+
+        output = entityAsMap(response)
+        assertEquals(1, output.objectMap("trigger_results").values.size)
+    }
+
     @Suppress("UNCHECKED_CAST")
     /** helper that returns a field in a json map whose values are all json objects */
     private fun Map<String, Any>.objectMap(key: String): Map<String, Map<String, Any>> {

alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt

@@ -9,10 +9,10 @@ import org.apache.hc.core5.http.ContentType
 import org.apache.hc.core5.http.HttpHeaders
 import org.apache.hc.core5.http.io.entity.StringEntity
 import org.apache.hc.core5.http.message.BasicHeader
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix
 import org.junit.After
 import org.junit.Before
 import org.junit.BeforeClass
-import org.junit.Ignore
 import org.opensearch.alerting.ALERTING_BASE_URI
 import org.opensearch.alerting.ALERTING_DELETE_WORKFLOW_ACCESS
 import org.opensearch.alerting.ALERTING_EXECUTE_WORKFLOW_ACCESS
@@ -63,7 +63,7 @@ import org.opensearch.test.junit.annotations.TestLogging
 import java.time.Instant
 
 // TODO investigate flaky nature of tests. not reproducible in local but fails in jenkins CI
-@Ignore
+@AwaitsFix(bugUrl = "https://github.com/opensearch-project/alerting/issues/1246")
 @TestLogging("level:DEBUG", reason = "Debug for tests.")
 @Suppress("UNCHECKED_CAST")
 class SecureWorkflowRestApiIT : AlertingRestTestCase() {

release-notes/opensearch-alerting.release-notes-2.11.0.0.md

@@ -0,0 +1,22 @@
+## Version 2.11.0.0 2023-10-11
+Compatible with OpenSearch 2.11.0
+
+### Maintenance
+* Increment version to 2.11.0-SNAPSHOT. ([#1116](https://github.com/opensearch-project/alerting/pull/1116))
+
+### Bug Fixes
+* Fix workflow execution for first run. ([#1227](https://github.com/opensearch-project/alerting/pull/1227))
+
+### Enhancements
+* Add logging for execution and indexes of monitors and workflows. ([#1223](https://github.com/opensearch-project/alerting/pull/1223))
+
+### Refactoring
+* Optimize doc-level monitor workflow for index patterns. ([#1122](https://github.com/opensearch-project/alerting/pull/1122))
+* Add workflow null or empty check only when empty workflow id passed. ([#1139(https://github.com/opensearch-project/alerting/pull/1139))
+* Add primary first calls for different monitor types. ([#1205](https://github.com/opensearch-project/alerting/pull/1205))
+
+### Infrastructure
+* Ignore flaky security test suites. ([#1188](https://github.com/opensearch-project/alerting/pull/1188))
+
+### Documentation
+* Added 2.11 release notes ([#1251](https://github.com/opensearch-project/alerting/pull/1251))