tests for SOQL IN clause based on CSV file values, fix for intermitte…

…nt config props issue tests for SOQL IN clause based on CSV file values, fix for intermittent config props issue
ashitsalesforce · Jan 19, 2025 · bc625b2 · bc625b2
1 parent 1c4679a
commit bc625b2
Show file tree

Hide file tree

Showing 7 changed files with 246 additions and 4 deletions.
diff --git a/runtests.sh b/runtests.sh
@@ -88,4 +88,4 @@ done
 #decryptedPassword="$(java ${debugEncryption} -cp ${jarname} com.salesforce.dataloader.process.DataLoaderRunner run.mode=encrypt -d ${encryptedPassword} ${encryptionFile} | tail -1)"
 #echo "decryptedPassword = ${decryptedPassword}"
 
-mvn ${failfast} -Dtest.endpoint=${1} -Dtest.user.default=${2} -Dtest.user.restricted=${3} -Dtest.password=${encryptedPassword} -Dtest.encryptionFile=${encryptionFile} verify ${debug} ${test} ${additionalOptions}
+mvn clean ${failfast} -Dtest.endpoint=${1} -Dtest.user.default=${2} -Dtest.user.restricted=${3} -Dtest.password=${encryptedPassword} -Dtest.encryptionFile=${encryptionFile} verify ${debug} ${test} ${additionalOptions}
diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java
@@ -426,9 +426,9 @@ private String constructSoqlFromFile(String soql, CSVFileReader csvReader, Strin
     }
 
     private static final String IN_CLAUSE_REGEX = "\\s+IN\\s+\\(\\s*\\{\\s*([^}]+)\\s*\\}\\s*,\\s*\\{\\s*([^}]+)\\s*\\}\\s*\\)";
-    private List<String> parseInClauseForFileAndColumnName(String input) {
+    static List<String> parseInClauseForFileAndColumnName(String input) {
         List<String> values = new ArrayList<>();
-        Pattern pattern = Pattern.compile(IN_CLAUSE_REGEX);
+        Pattern pattern = Pattern.compile(IN_CLAUSE_REGEX,  Pattern.CASE_INSENSITIVE);
         Matcher matcher = pattern.matcher(input);
 
         if (matcher.find()) {

diff --git a/src/test/java/com/salesforce/dataloader/ConfigTestBase.java b/src/test/java/com/salesforce/dataloader/ConfigTestBase.java
@@ -33,6 +33,7 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 public abstract class ConfigTestBase extends TestBase {
@@ -79,6 +80,17 @@ protected Map<String, String> getTestConfig() {
             configBase.put(AppConfig.PROP_PROXY_USERNAME, proxyUsername);
             configBase.put(AppConfig.PROP_PROXY_PASSWORD, proxyPassword);
         }
+
+
+        Properties systemProperties = System.getProperties();
+
+        // Iterate through system properties
+        systemProperties.forEach((key, value) -> {
+            System.out.println(key + " : " + value);
+            if (key.toString().startsWith("test.")) {
+                configBase.put(key.toString(), value.toString());
+            }
+        });
         return configBase;
     }
 

diff --git a/src/test/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitorTest.java b/src/test/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitorTest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2015, salesforce.com, inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *    Redistributions of source code must retain the above copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ *    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
+ *    the following disclaimer in the documentation and/or other materials provided with the distribution.
+ *
+ *    Neither the name of salesforce.com, inc. nor the names of its contributors may be used to endorse or
+ *    promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.dataloader.action.visitor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import org.junit.Test;
+
+
+public class AbstractQueryVisitorTest {
+    @Test
+    public void testParseInClauseForFileAndColumnName_IN_ValidInput() {
+        String input = "SELECT name FROM Account WHERE id IN ({c:\\users\\me\\dataloader\\accounts.csv}, {acctid})";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertEquals(2, result.size());
+        assertEquals("c:\\users\\me\\dataloader\\accounts.csv", result.get(0));
+        assertEquals("acctid", result.get(1));
+    }
+
+    @Test
+    public void testParseInClauseForFileAndColumnName_in_ValidInput() {
+        String input = "SELECT name FROM Account WHERE id in ({c:\\users\\me\\dataloader\\accounts.csv}, {acctid})";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertEquals(2, result.size());
+        assertEquals("c:\\users\\me\\dataloader\\accounts.csv", result.get(0));
+        assertEquals("acctid", result.get(1));
+    }
+
+    @Test
+    public void testParseInClauseForFileAndColumnName_iN_ValidInput() {
+        String input = "SELECT name FROM Account WHERE id iN ({c:\\users\\me\\dataloader\\accounts.csv}, {acctid})";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertEquals(2, result.size());
+        assertEquals("c:\\users\\me\\dataloader\\accounts.csv", result.get(0));
+        assertEquals("acctid", result.get(1));
+    }
+
+    @Test
+    public void testParseInClauseForFileAndColumnName_InvalidInput() {
+        String input = "SELECT name FROM Account WHERE id IN ()";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertTrue(result.isEmpty());
+    }
+
+    @Test
+    public void testParseInClauseForFileAndColumnName_EmptyInput() {
+        String input = "";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertTrue(result.isEmpty());
+    }
+
+    @Test
+    public void testParseInClauseForFileAndColumnName_MissingBraces() {
+        String input = "SELECT name FROM Account WHERE id IN ('c:\\users\\me\\dataloader\\accounts.csv', 'acctid')";
+        List<String> result = AbstractQueryVisitor.parseInClauseForFileAndColumnName(input);
+        assertTrue(result.isEmpty());
+    }
+}
diff --git a/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java b/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2015, salesforce.com, inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *    Redistributions of source code must retain the above copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ *    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
+ *    the following disclaimer in the documentation and/or other materials provided with the distribution.
+ *
+ *    Neither the name of salesforce.com, inc. nor the names of its contributors may be used to endorse or
+ *    promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.dataloader.process;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.salesforce.dataloader.TestSetting;
+import com.salesforce.dataloader.TestVariant;
+import com.salesforce.dataloader.action.OperationInfo;
+import com.salesforce.dataloader.action.visitor.AbstractQueryVisitor;
+import com.salesforce.dataloader.config.AppConfig;
+import com.salesforce.dataloader.dao.csv.CSVFileReader;
+import com.salesforce.dataloader.exception.DataAccessObjectException;
+import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
+import com.salesforce.dataloader.exception.ProcessInitializationException;
+import com.salesforce.dataloader.model.TableRow;
+import com.sforce.soap.partner.sobject.SObject;
+import com.sforce.ws.ConnectionException;
+
+@RunWith(Parameterized.class)
+@SuppressWarnings("unused")
+public class SOQLInClauseFromCSVTest extends ProcessTestBase {
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> getParameters() {
+        return Arrays.asList(
+                // partner API
+                TestVariant.forSettings(TestSetting.BULK_API_DISABLED, TestSetting.BULK_V2_API_DISABLED),
+                // Bulk API
+                TestVariant.forSettings(TestSetting.BULK_API_ENABLED, TestSetting.BULK_V2_API_DISABLED),
+                // Bulk V2 Query API
+                TestVariant.forSettings(TestSetting.BULK_V2_API_ENABLED));
+    }
+
+    private Map<String,String> testConfig;
+
+    @Before
+    public void setupTestConfig() {
+        testConfig = getTestConfig(OperationInfo.extract, true);
+        testConfig.put(AppConfig.PROP_ENTITY, "Account");
+        testConfig.put(AppConfig.PROP_ENABLE_EXTRACT_STATUS_OUTPUT, AppConfig.TRUE);
+        testConfig.remove(AppConfig.PROP_MAPPING_FILE);
+    }
+
+    public SOQLInClauseFromCSVTest(Map<String, String> config) {
+        super(config);
+    }
+
+    private static final String COL_IN_CSV = "Oracle_Id__c";
+    private static final String ORACLE_ID_VAL = "oracleIdXyz";
+    private static final String NAME_VAL = "acctNameXyz";
+
+    @Test
+    public void testSoqlInClauseUsingCSV() throws Exception {
+        Map<String, String> configMap = getTestConfig(OperationInfo.insert, false);
+        String accountId = insertAccount(NAME_VAL, ORACLE_ID_VAL);
+        String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv";
+        runExtraction("select name from Account where "
+                + COL_IN_CSV
+                + " in ({" 
+                + extractionFileName 
+                + "}, {"
+                + COL_IN_CSV
+                + "})");
+        validateAccountNameInOutputFile("acctNameXyz");
+    }
+
+    private void runExtraction(String extractionQuery) throws ProcessInitializationException, DataAccessObjectException {
+        testConfig.put(AppConfig.PROP_EXTRACT_SOQL, extractionQuery);
+        testConfig.put(AppConfig.PROP_LIMIT_OUTPUT_TO_QUERY_FIELDS, AppConfig.TRUE);
+        runProcess(testConfig, 1, true);
+    }
+
+    private String insertAccount(String name, String oracleId) throws ConnectionException {
+        final SObject account = new SObject();
+        account.setType("Account");
+        account.setField("Name", name);
+        account.setField("Oracle_Id__c", oracleId);
+        String id = getBinding().create(new SObject[]{account})[0].getId();
+        assertNotNull(id);
+        return id;
+    }
+
+    private void validateAccountNameInOutputFile(final String accountName) throws IOException {
+
+        FileInputStream fis = new FileInputStream(new File(testConfig.get(AppConfig.PROP_DAO_NAME)));
+        try {
+            CSVFileReader rdr = new CSVFileReader(new File(testConfig.get(AppConfig.PROP_DAO_NAME)),
+                    this.getController().getAppConfig(), false, true);
+            rdr.open();
+            TableRow row = rdr.readTableRow();
+            String extractedNameVal = (String)row.get("Name");
+            assertEquals(accountName, extractedNameVal);
+        } catch (DataAccessObjectInitializationException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (DataAccessObjectException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } finally {
+            IOUtils.closeQuietly(fis);
+        }
+    }
+}
diff --git a/src/test/java/com/salesforce/dataloader/util/TestRecordsUtil.java b/src/test/java/com/salesforce/dataloader/util/TestRecordsUtil.java
@@ -72,7 +72,7 @@ private void execute(String[] args) throws Exception {
             whereClause = ConfigTestBase.ACCOUNT_WHERE_CLAUSE;
         }
 
-        setupController();
+        setupController(argMap);
         if("insert".equals(operation)) {
             insertSfdcAccounts(numRecords, true);
         } else if("upsert".equals(operation)) {

diff --git a/src/test/resources/testfiles/data/SoqlInClauseFromCSV.csv b/src/test/resources/testfiles/data/SoqlInClauseFromCSV.csv
@@ -0,0 +1,2 @@
+name,Oracle_Id__c
+"acctNameXyz","oracleIdXyz"