Jira Source Configuration and Filter Changes and add license headers (#…

…5306) * jira yaml changes and tests, authentication and hosts Signed-off-by: Maxwell Brown <[email protected]> * filter config file changes Signed-off-by: Maxwell Brown <[email protected]> * jira filter config changes and implementation Signed-off-by: Maxwell Brown <[email protected]> * filter change test corrections Signed-off-by: Maxwell Brown <[email protected]> * project name -> key for now Signed-off-by: Maxwell Brown <[email protected]> * removed unfishished function Signed-off-by: Maxwell Brown <[email protected]> * address pr comments Signed-off-by: Maxwell Brown <[email protected]> * Add license headers to all jira source files Signed-off-by: Maxwell Brown <[email protected]> --------- Signed-off-by: Maxwell Brown <[email protected]>
opensearch-project · Jan 9, 2025 · bad172c · bad172c
1 parent 791e355
commit bad172c
Show file tree

Hide file tree

Showing 53 changed files with 1,042 additions and 201 deletions.
diff --git a/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java b/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java
@@ -1,3 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import com.fasterxml.jackson.core.JsonProcessingException;

diff --git a/...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java
@@ -1,3 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import lombok.Getter;

diff --git a/...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java b/...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java
@@ -1,3 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 

diff --git a/...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java
@@ -1,3 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import io.micrometer.core.instrument.Counter;
@@ -14,8 +24,10 @@
 import javax.inject.Named;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Queue;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -25,9 +37,12 @@
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.GREATER_THAN_EQUALS;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PREFIX;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.SUFFIX;
 
 
@@ -117,26 +132,41 @@ private void addItemsToQueue(List<IssueBean> issueList, Queue<ItemInfo> itemInfo
     private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, Instant ts) {
 
         log.info("Creating issue filter criteria");
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) {
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration)) ) {
             validateProjectFilters(configuration);
         }
         StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli());
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) {
-            jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectKeyFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration))) {
+            jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectNameIncludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration))) {
+            jiraQl.append(PROJECT_NOT_IN).append(JiraConfigHelper.getProjectNameExcludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeIncludeFilter(configuration))) {
+            jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeIncludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeExcludeFilter(configuration))) {
+            jiraQl.append(ISSUE_TYPE_NOT_IN).append(JiraConfigHelper.getIssueTypeExcludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeFilter(configuration))) {
-            jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusIncludeFilter(configuration))) {
+            jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusIncludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusFilter(configuration))) {
-            jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusExcludeFilter(configuration))) {
+            jiraQl.append(STATUS_NOT_IN).append(JiraConfigHelper.getIssueStatusExcludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        log.trace("Created issue filter criteria JiraQl query: {}", jiraQl);
+        log.error("Created issue filter criteria JiraQl query: {}", jiraQl);
         return jiraQl;
     }
 
@@ -148,9 +178,21 @@ private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration,
     private void validateProjectFilters(JiraSourceConfig configuration) {
         log.trace("Validating project filters");
         List<String> badFilters = new ArrayList<>();
+        Set<String> includedProjects = new HashSet<>();
+        List<String> includedAndExcludedProjects = new ArrayList<>();
         Pattern regex = Pattern.compile("[^A-Z0-9]");
-        JiraConfigHelper.getProjectKeyFilter(configuration).forEach(projectFilter -> {
+        JiraConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> {
             Matcher matcher = regex.matcher(projectFilter);
+            includedProjects.add(projectFilter);
+            if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
+                badFilters.add(projectFilter);
+            }
+        });
+        JiraConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> {
+            Matcher matcher = regex.matcher(projectFilter);
+            if (includedProjects.contains(projectFilter)) {
+                includedAndExcludedProjects.add(projectFilter);
+            }
             if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
                 badFilters.add(projectFilter);
             }
@@ -162,6 +204,14 @@ private void validateProjectFilters(JiraSourceConfig configuration) {
                     "Invalid project key found in filter configuration for "
                     + filters);
         }
+        if (!includedAndExcludedProjects.isEmpty()) {
+            String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", ");
+            log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects);
+            throw new BadRequestException("Bad request exception occurred " +
+                    "Project filters is invalid because the following projects are listed in both include and exclude"
+                    + filters);
+        }
+
     }
 
 }
diff --git a/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java b/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java
@@ -1,3 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 

diff --git a/...source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java b/...source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java
@@ -1,16 +1,24 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.constraints.Size;
+import jakarta.validation.Valid;
 import lombok.Getter;
+import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig;
+import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
 
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-
-import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
 
 @Getter
 public class JiraSourceConfig implements CrawlerSourceConfig {
@@ -20,50 +28,23 @@ public class JiraSourceConfig implements CrawlerSourceConfig {
     /**
      * Jira account url
      */
-    @JsonProperty("account_url")
-    private String accountUrl;
+    @JsonProperty("hosts")
+    private List<String> hosts;
 
     /**
-     * A map of connector credentials specific to this source
+     * Authentication Config to Access Jira
      */
-    @JsonProperty("connector_credentials")
-    private Map<String, String> connectorCredentials;
+    @JsonProperty("authentication")
+    @Valid
+    private AuthenticationConfig authenticationConfig;
 
-    /**
-     * List of projects to ingest
-     */
-    @JsonProperty("projects")
-    @Size(max = 1000, message = "Project type filter should not be more than 1000")
-    private List<String> project = new ArrayList<>();
 
     /**
-     * List of specific issue types to ingest.
-     * Ex: Story, Epic, Task etc
+     * Filter Config to filter what tickets get ingested
      */
-    @JsonProperty("issue_types")
-    @Size(max = 1000, message = "Issue type filter should be less than 1000")
-    private List<String> issueType = new ArrayList<>();
+    @JsonProperty("filter")
+    private FilterConfig filterConfig;
 
-    /**
-     * Optional Inclusion patterns for filtering some tickets
-     */
-    @JsonProperty("inclusion_patterns")
-    @Size(max = 100, message = "inclusion pattern filters should not be more than 1000")
-    private List<String> inclusionPatterns;
-
-    /**
-     * Optional Exclusion patterns for excluding some tickets
-     */
-    @JsonProperty("exclusion_patterns")
-    @Size(max = 1000, message = "exclusion pattern filter should be less than 1000")
-    private List<String> exclusionPatterns;
-
-    /**
-     * Optional Status filter to ingest the tickets
-     */
-    @JsonProperty("statuses")
-    @Size(max = 1000, message = "Status filter should be less than 1000")
-    private List<String> status = new ArrayList<>();
 
     /**
      * Number of worker threads to spawn to parallel source fetching
@@ -78,43 +59,11 @@ public class JiraSourceConfig implements CrawlerSourceConfig {
     @JsonProperty("backoff_time")
     private Duration backOff = DEFAULT_BACKOFF_MILLIS;
 
-    public String getJiraId() {
-        return this.getConnectorCredentials().get("jira_id");
-    }
-
-    public String getJiraCredential() {
-        return this.getConnectorCredentials().get("jira_credential");
+    public String getAccountUrl() {
+        return this.getHosts().get(0);
     }
 
     public String getAuthType() {
-        return this.getConnectorCredentials().get("auth_type");
-    }
-
-    public String getAccessToken() {
-        return fetchGivenOAuthAttribute("access_token");
-    }
-
-    public String getRefreshToken() {
-        return fetchGivenOAuthAttribute("refresh_token");
-    }
-
-    public String getClientId() {
-        return fetchGivenOAuthAttribute("client_id");
-    }
-
-    public String getClientSecret() {
-        return fetchGivenOAuthAttribute("client_secret");
+        return this.getAuthenticationConfig().getAuthType();
     }
-
-    private String fetchGivenOAuthAttribute(String givenAttribute) {
-        if (!OAUTH2.equals(getAuthType())) {
-            throw new RuntimeException("Authentication Type is not OAuth2.");
-        }
-        String attributeValue = this.getConnectorCredentials().get(givenAttribute);
-        if (attributeValue == null || attributeValue.isEmpty()) {
-            throw new RuntimeException(String.format("%s is required for OAuth2 AuthType", givenAttribute));
-        }
-        return attributeValue;
-    }
-
 }
diff --git a/...va/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java b/...va/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
+
+@Getter
+public class AuthenticationConfig {
+    @JsonProperty("basic")
+    @Valid
+    private BasicConfig basicConfig;
+
+    @JsonProperty("oauth2")
+    @Valid
+    private Oauth2Config oauth2Config;
+
+    @AssertTrue(message = "Authentication config should have either basic or oauth2")
+    private boolean isValidAuthenticationConfig() {
+        boolean hasBasic = basicConfig != null;
+        boolean hasOauth = oauth2Config != null;
+        return hasBasic ^ hasOauth;
+    }
+
+    public String getAuthType() {
+        if (basicConfig != null) {
+            return BASIC;
+        } else {
+            return OAUTH2;
+        }
+    }
+}
diff --git a/...c/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java b/...c/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+@Getter
+public class BasicConfig {
+    @JsonProperty("username")
+    private String username;
+
+    @JsonProperty("password")
+    private String password;
+
+    @AssertTrue(message = "Username and Password are both required for Basic Auth")
+    private boolean isBasicConfigValid() {
+        return username != null && password != null;
+    }
+}