Merge branch 'opensearch-project:main' into kp/flaky-tests-it

.ci/bwcVersions

@@ -32,4 +32,5 @@ BWC_VERSION:
   - "2.13.0"
   - "2.13.1"
   - "2.14.0"
+  - "2.14.1"
   - "2.15.0"

.github/workflows/dependabot_pr.yml

@@ -7,7 +7,7 @@ jobs:
     permissions:
       pull-requests: write
       contents: write
-    if: ${{ github.actor == 'dependabot[bot]' }}
+    if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' }}
     steps:
       - name: GitHub App token
         id: github_app_token

.github/workflows/gradle-check.yml

@@ -30,11 +30,15 @@ jobs:
       - name: Setup environment variables (PR)
         if: github.event_name == 'pull_request_target'
         run: |
+          echo "event_name=pull_request_target" >> $GITHUB_ENV
+          echo "branch_name=$(jq --raw-output .pull_request.base.ref $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output .pull_request.user.login $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
@@ -43,11 +47,14 @@ jobs:
           ref_id=$(git rev-parse HEAD)
           branch_name=$(git rev-parse --abbrev-ref HEAD)
           echo "branch_name=$branch_name" >> $GITHUB_ENV
+          echo "event_name=push" >> $GITHUB_ENV
           echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
           echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
           echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
+          echo "pr_owner=$(jq --raw-output '.commits[0].author.username' $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
           echo "pr_number=Null" >> $GITHUB_ENV
+          echo "pr_or_commit_description=$(jq --ascii-output .head_commit.message $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
         uses: actions/checkout@v4

.github/workflows/pull-request-checks.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: peternied/[email protected]
-        if: github.actor != 'dependabot[bot]'
+        if: github.event.pull_request.user.login != 'dependabot[bot]'
         with:
           checklist-items: |
             New functionality includes testing.

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
 - Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
+- Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))

README.md

@@ -7,7 +7,7 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.14.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.0")
+[![2.14.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.1")
 [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)

buildSrc/version.properties

@@ -57,7 +57,7 @@ bouncycastle=1.78
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.11.0
+mockito           = 5.12.0
 objenesis         = 3.2
 bytebuddy         = 1.14.9
 

libs/core/src/main/java/org/opensearch/Version.java

@@ -103,6 +103,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_13_1 = new Version(2130199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;

plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java

@@ -32,54 +32,67 @@
 
 package org.opensearch.ingest.attachment;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
 import org.opensearch.common.io.PathUtils;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 /**
- * Evil test-coverage cheat, we parse a bunch of docs from tika
- * so that we have a nice grab-bag variety, and assert some content
- * comes back and no exception.
+ * Parse sample tika documents and assert the contents has not changed according to previously recorded checksums.
+ * Uncaught changes to tika parsing could potentially pose bwc issues.
+ * Note: In some cases tika will access a user's locale to inform the parsing of a file.
+ * The checksums of these files are left empty, and we only validate that parsed content is not null.
  */
 @SuppressFileSystems("ExtrasFS") // don't try to parse extraN
 public class TikaDocTests extends OpenSearchTestCase {
 
-    /** some test files from tika test suite, zipped up */
+    /** some test files from the apache tika unit test suite with accompanying sha1 checksums */
     static final String TIKA_FILES = "/org/opensearch/ingest/attachment/test/tika-files/";
+    static final String TIKA_CHECKSUMS = "/org/opensearch/ingest/attachment/test/.checksums";
 
-    public void testFiles() throws Exception {
-        Path tmp = createTempDir();
-        logger.debug("unzipping all tika sample files");
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(TIKA_FILES).toURI()))) {
-            for (Path doc : stream) {
-                String filename = doc.getFileName().toString();
-                TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES + filename), tmp);
-            }
-        }
+    public void testParseSamples() throws Exception {
+        String checksumJson = Files.readString(PathUtils.get(getClass().getResource(TIKA_CHECKSUMS).toURI()));
+        Map<String, Object> checksums = XContentHelper.convertToMap(JsonXContent.jsonXContent, checksumJson, false);
+        DirectoryStream<Path> stream = Files.newDirectoryStream(unzipToTemp(TIKA_FILES));
 
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmp)) {
-            for (Path doc : stream) {
-                logger.debug("parsing: {}", doc);
-                assertParseable(doc);
+        for (Path doc : stream) {
+            String parsedContent = tryParse(doc);
+            assertNotNull(parsedContent);
+            assertFalse(parsedContent.isEmpty());
+
+            String check = checksums.get(doc.getFileName().toString()).toString();
+            if (!check.isEmpty()) {
+                assertEquals(check, DigestUtils.sha1Hex(parsedContent));
             }
         }
+
+        stream.close();
     }
 
-    void assertParseable(Path fileName) throws Exception {
-        try {
-            byte bytes[] = Files.readAllBytes(fileName);
-            String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1);
-            assertNotNull(parsedContent);
-            assertFalse(parsedContent.isEmpty());
-            logger.debug("extracted content: {}", parsedContent);
-        } catch (Exception e) {
-            throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e);
+    private Path unzipToTemp(String zipDir) throws Exception {
+        Path tmp = createTempDir();
+        DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(zipDir).toURI()));
+
+        for (Path doc : stream) {
+            String filename = doc.getFileName().toString();
+            TestUtil.unzip(getClass().getResourceAsStream(zipDir + filename), tmp);
         }
+
+        stream.close();
+        return tmp;
+    }
+
+    private String tryParse(Path doc) throws Exception {
+        byte bytes[] = Files.readAllBytes(doc);
+        return TikaImpl.parse(bytes, new Metadata(), -1);
     }
 }