correctly show query processed rows count + tests for multiple jobs i…

…n single query operation

correctly show query processed rows count + tests for multiple jobs in single query operation

src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java

@@ -109,6 +109,11 @@ public final void visit() throws DataAccessObjectException, OperationException {
                     flushResults();
                 }
                 soql = getSoqlForNextBatch();
+                if (soql != null) {
+                    this.resetCalculations();
+                    this.batchRows.clear();
+                    this.batchIds.clear();
+                }
             } 
         } catch (final ApiFault e) {
             throw new ExtractException(e.getExceptionMessage(), e);
@@ -135,7 +140,8 @@ protected boolean writeStatus() {
     private CSVFileReader csvReader = null;
     private String inClauseColName = null;
     private int numRows = 0;
-    private int maxSoqlCharLength = DEFAULT_MAX_SOQL_CHAR_LENGTH;
+    private static int maxSoqlCharLength = DEFAULT_MAX_SOQL_CHAR_LENGTH;
+
     private String getSoqlForNextBatch() throws OperationException {
         List<String> inClauseFileAndColumnNameList = parseInClauseForFileAndColumnName(soql);
         if (inClauseFileAndColumnNameList.size() == 2) {
@@ -381,8 +387,8 @@ protected SOQLMapper getMapper() {
         return (SOQLMapper)super.getMapper();
     }
 
-    public void setMaxSoqlCharLength(int maxSoqlCharLength) {
-        this.maxSoqlCharLength = maxSoqlCharLength;
+    public static void setMaxSoqlCharLength(int maxSoqlCharLength) {
+        AbstractQueryVisitor.maxSoqlCharLength = maxSoqlCharLength;
     }
 
     private static final String IN_CLAUSE = " IN ";

src/main/java/com/salesforce/dataloader/action/visitor/AbstractVisitor.java

@@ -179,4 +179,9 @@ public Map<String, InputStream> getAttachments() {
         return null;
     }
 
+    protected void resetCalculations() {
+        this.rateCalculator = new LoadRateCalculator();
+        this.errors = 0;
+        this.successes = 0;
+    }
 }

src/test/java/com/salesforce/dataloader/ConfigTestBase.java

@@ -33,6 +33,7 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
 
@@ -73,24 +74,16 @@ protected Map<String, String> getTestConfig() {
         for (TestProperties prop : getDefaultTestPropertiesSet()) {
             prop.putConfigSetting(configBase);
         }
+        for (Entry<Object, Object> prop : getTestBaseProperties().entrySet()) {
+            configBase.put(prop.getKey().toString(), prop.getValue().toString());
+        }
         configBase.put(AppConfig.CLI_OPTION_CONFIG_DIR_PROP, TEST_CONF_DIR);
         String proxyUsername = System.getProperty(AppConfig.PROP_PROXY_USERNAME);
         String proxyPassword = System.getProperty(AppConfig.PROP_PROXY_PASSWORD);
         if (proxyUsername != null && proxyPassword != null) {
             configBase.put(AppConfig.PROP_PROXY_USERNAME, proxyUsername);
             configBase.put(AppConfig.PROP_PROXY_PASSWORD, proxyPassword);
         }
-
-
-        Properties systemProperties = System.getProperties();
-
-        // Iterate through system properties
-        systemProperties.forEach((key, value) -> {
-            System.out.println(key + " : " + value);
-            if (key.toString().startsWith("test.")) {
-                configBase.put(key.toString(), value.toString());
-            }
-        });
         return configBase;
     }
 

src/test/java/com/salesforce/dataloader/TestBase.java

@@ -71,7 +71,7 @@
  * @author Alex Warshavsky
  * @since 8.0
  */
-abstract class TestBase {
+abstract public class TestBase {
 
     private static final Pattern INSIDE_BRACKETS_TEST_PARAMETERS = Pattern.compile("\\[.+\\]");
     @Rule
@@ -83,6 +83,7 @@ abstract class TestBase {
      * *********
      */
     private static final Properties TEST_PROPS;
+    private static final Properties TEST_PROPS_NO_OVERRIDES = loadTestProperties();
     private static final String TEST_FILES_DIR;
     protected static final String TEST_CONF_DIR;
     private static final String TEST_DATA_DIR;
@@ -110,6 +111,15 @@ abstract class TestBase {
 
         Map<String, String> argsMap = new HashMap<String, String>();
         argsMap.put(AppConfig.CLI_OPTION_CONFIG_DIR_PROP, getTestConfDir());
+
+        // Iterate through system properties to check for test properties settings
+        Properties systemProperties = System.getProperties();
+        systemProperties.forEach((key, value) -> {
+            System.out.println(key + " : " + value);
+            if (key.toString().startsWith("test.")) {
+                TEST_PROPS.put(key.toString(), value.toString());
+            }
+        });
     }
 
     private static Properties loadTestProperties() {
@@ -133,6 +143,18 @@ protected static String getProperty(String testProperty) {
         return TEST_PROPS.getProperty(testProperty);
     }
 
+    protected static String getPropertyNoOverriedes(String testProperty) {
+        return TEST_PROPS_NO_OVERRIDES.getProperty(testProperty);
+    }
+
+    protected static Properties getTestBaseProperties() {
+        return TEST_PROPS;
+    }
+
+    protected static Properties getTestBasePropertiesNoOverrides() {
+        return TEST_PROPS_NO_OVERRIDES;
+    }
+
     private static final String API_CLIENT_NAME = "DataLoaderTestBatch/" + Controller.APP_VERSION;
 
     protected static final String DEFAULT_CONTACT_EXT_ID_FIELD = "NumberId__c";

src/test/java/com/salesforce/dataloader/config/ConfigTest.java

@@ -103,7 +103,7 @@ public void testProperty_OAUTH_SERVER() {
             AppConfig appConfig = AppConfig.getInstance(getTestConfig());
             appConfig.setServerEnvironment(AppConfig.SERVER_PROD_ENVIRONMENT_VAL);
             String configuredOAuthServer = appConfig.getAuthEndpointForCurrentEnv();
-            String expectedOAuthServer = getProperty("test.endpoint");
+            String expectedOAuthServer = getPropertyNoOverriedes("test.endpoint");
             if (expectedOAuthServer == null || expectedOAuthServer.isBlank()) {
                 logger.info("Expected prefix is " + expectedOAuthServer);
                 logger.info("Actual prefix is " + configuredOAuthServer);
@@ -129,7 +129,7 @@ public void testProperty_OAUTH_REDIRECTURI() {
             AppConfig appConfig = AppConfig.getInstance(testConfigMap);
             appConfig.setServerEnvironment(AppConfig.SERVER_PROD_ENVIRONMENT_VAL);
             String configuredOAuthRedirectURI = appConfig.getOAuthRedirectURIForCurrentEnv();
-            String expectedOAuthRedirectURIPrefix = getProperty("test.endpoint");
+            String expectedOAuthRedirectURIPrefix = getPropertyNoOverriedes("test.endpoint");
             if (expectedOAuthRedirectURIPrefix == null || expectedOAuthRedirectURIPrefix.isBlank()) {
                 logger.info("Expected prefix is " + expectedOAuthRedirectURIPrefix);
                 logger.info("Actual prefix is " + configuredOAuthRedirectURI);

src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java

@@ -45,6 +45,7 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import com.salesforce.dataloader.TestBase;
 import com.salesforce.dataloader.TestSetting;
 import com.salesforce.dataloader.TestVariant;
 import com.salesforce.dataloader.action.OperationInfo;
@@ -93,8 +94,8 @@ public SOQLInClauseFromCSVTest(Map<String, String> config) {
     @Test
     public void testSoqlInClauseUsingCSV() throws Exception {
         Map<String, String> configMap = getTestConfig(OperationInfo.insert, false);
-        String accountId1 = insertAccount(NAME_VAL, ORACLE_ID_VAL);
-        String accountId2 = insertAccount(NAME_VAL+"2", ORACLE_ID_VAL+"2");
+        String accountId1 = insertAccount(NAME_VAL+"1", ORACLE_ID_VAL+"1", TestBase.ACCOUNT_NUMBER_PREFIX+"1");
+        String accountId2 = insertAccount(NAME_VAL+"2", ORACLE_ID_VAL+"2", TestBase.ACCOUNT_NUMBER_PREFIX+"2");
         String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv";
         runExtraction("select name from Account where "
                 + COL_IN_CSV
@@ -104,7 +105,7 @@ public void testSoqlInClauseUsingCSV() throws Exception {
                 + COL_IN_CSV
                 + "}) OR Name = '" + NAME_VAL + "2' ORDER BY Name", 2);
         validateAccountNamesInOutputFile(NAME_VAL, 2);
-        runExtraction("select name from Account where Name = '" + NAME_VAL + "' AND "
+        runExtraction("select name from Account where Name = '" + NAME_VAL + "1' AND "
                 + COL_IN_CSV
                 + " in ({" 
                 + extractionFileName 
@@ -113,18 +114,33 @@ public void testSoqlInClauseUsingCSV() throws Exception {
                 + "})", 1);
         validateAccountNamesInOutputFile(NAME_VAL, 1);
     }
+
+    @Test
+    public void testSoqlInClauseUsingCSVMultiBatch() throws Exception {
+        AbstractQueryVisitor.setMaxSoqlCharLength(500);
+        Map<String, String> configMap = getTestConfig(OperationInfo.insert, false);
+        for (int i = 0; i < 20; i++) {
+            insertAccount(NAME_VAL + i, ORACLE_ID_VAL + i, TestBase.ACCOUNT_NUMBER_PREFIX + i);
+        }
+        String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv";
+        runExtraction("select name from Account where " + COL_IN_CSV + " in ({" + extractionFileName + "}, {"
+                + COL_IN_CSV + "})", 20);
+        validateAccountNamesInOutputFile(NAME_VAL, 20);
+        AbstractQueryVisitor.setMaxSoqlCharLength(AbstractQueryVisitor.DEFAULT_MAX_SOQL_CHAR_LENGTH);
+    }
 
     private void runExtraction(String extractionQuery, int numSuccesses) throws ProcessInitializationException, DataAccessObjectException {
         testConfig.put(AppConfig.PROP_EXTRACT_SOQL, extractionQuery);
         testConfig.put(AppConfig.PROP_LIMIT_OUTPUT_TO_QUERY_FIELDS, AppConfig.TRUE);
         runProcess(testConfig, numSuccesses, true);
     }
 
-    private String insertAccount(String name, String oracleId) throws ConnectionException {
+    private String insertAccount(String name, String oracleId, String acctId) throws ConnectionException {
         final SObject account = new SObject();
         account.setType("Account");
         account.setField("Name", name);
         account.setField("Oracle_Id__c", oracleId);
+        account.setField("AccountNumber__c", acctId); // to make sure that records are deleted before/after test execution
         String id = getBinding().create(new SObject[]{account})[0].getId();
         assertNotNull(id);
         return id;

src/test/resources/testfiles/data/SoqlInClauseFromCSV.csv

@@ -1,2 +1,21 @@
-name,Oracle_Id__c
-"acctNameXyz","oracleIdXyz"
+name,Oracle_Id__c,AccountNumber__c
+"acctNameXyz0","oracleIdXyz0","ACCT0"
+"acctNameXyz1","oracleIdXyz1","ACCT1"
+"acctNameXyz2","oracleIdXyz2","ACCT2"
+"acctNameXyz3","oracleIdXyz3","ACCT3"
+"acctNameXyz4","oracleIdXyz4","ACCT4"
+"acctNameXyz5","oracleIdXyz5","ACCT5"
+"acctNameXyz6","oracleIdXyz6","ACCT6"
+"acctNameXyz7","oracleIdXyz7","ACCT7"
+"acctNameXyz8","oracleIdXyz8","ACCT8"
+"acctNameXyz9","oracleIdXyz9","ACCT9"
+"acctNameXyz10","oracleIdXyz10","ACCT10"
+"acctNameXyz11","oracleIdXyz11","ACCT11"
+"acctNameXyz12","oracleIdXyz12","ACCT12"
+"acctNameXyz13","oracleIdXyz13","ACCT13"
+"acctNameXyz14","oracleIdXyz14","ACCT14"
+"acctNameXyz15","oracleIdXyz15","ACCT15"
+"acctNameXyz16","oracleIdXyz16","ACCT16"
+"acctNameXyz17","oracleIdXyz17","ACCT17"
+"acctNameXyz18","oracleIdXyz18","ACCT18"
+"acctNameXyz19","oracleIdXyz19","ACCT19"