config.property to specify max chars in soql on the client side.

config property "sfdc.soql.maxlength" to override default max chars in soql on the client side.
ashitsalesforce · Jan 21, 2025 · 76de107 · 76de107
1 parent 33591d1
commit 76de107
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 11 deletions.
diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java
@@ -134,13 +134,11 @@ protected boolean writeStatus() {
         return getConfig().getBoolean(AppConfig.PROP_ENABLE_EXTRACT_STATUS_OUTPUT);
     }
 
-    public static final int DEFAULT_MAX_SOQL_CHAR_LENGTH = 100000;
     public static final int MAX_IDLOOKUP_FIELD_LENGTH = 255;
     private int daoLastProcessedRow = 0;
     private CSVFileReader csvReader = null;
     private String inClauseColName = null;
     private int numRows = 0;
-    private static int maxSoqlCharLength = DEFAULT_MAX_SOQL_CHAR_LENGTH;
 
     private String getSoqlForNextBatch() throws OperationException {
         List<String> inClauseFileAndColumnNameList = parseInClauseForFileAndColumnName(soql);
@@ -386,10 +384,6 @@ protected void startWriteExtraction(int size) {
     protected SOQLMapper getMapper() {
         return (SOQLMapper)super.getMapper();
     }
-
-    public static void setMaxSoqlCharLength(int maxSoqlCharLength) {
-        AbstractQueryVisitor.maxSoqlCharLength = maxSoqlCharLength;
-    }
 
     private static final String IN_CLAUSE = " IN ";
     private String constructSoqlFromFile(String soql, CSVFileReader csvReader, String columnName) throws IOException, DataAccessObjectException {
@@ -411,8 +405,15 @@ private String constructSoqlFromFile(String soql, CSVFileReader csvReader, Strin
 
         boolean firstRowOfCurrentBatch = true;
         int soqlLength = soqlBuilder.length() + MAX_IDLOOKUP_FIELD_LENGTH + 4 + soqlAfterInClause.length();
+        int maxSoqlLength = AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH;
+        try {
+            maxSoqlLength = this.controller.getAppConfig().getInt(AppConfig.PROP_SOQL_MAX_LENGTH);
+        } catch (ParameterLoadException e) {
+            logger.warn("Error getting max soql length: " + e.getMessage());
+            maxSoqlLength = AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH;
+        }
         while (daoLastProcessedRow < numRows
-                && soqlLength  < this.maxSoqlCharLength) {
+                && soqlLength  < maxSoqlLength) {
             if (firstRowOfCurrentBatch) {
                 firstRowOfCurrentBatch = false;
             }  else {

diff --git a/src/main/java/com/salesforce/dataloader/config/AppConfig.java b/src/main/java/com/salesforce/dataloader/config/AppConfig.java
@@ -335,6 +335,7 @@ public class AppConfig {
     public static final String PROP_IDLOOKUP_FIELD = "sfdc.externalIdField"; //$NON-NLS-1$
     public static final String PROP_EXPORT_BATCH_SIZE = "sfdc.extractionRequestSize"; //$NON-NLS-1$
     public static final String PROP_EXTRACT_SOQL = "sfdc.extractionSOQL"; //$NON-NLS-1$
+    public static final String PROP_SOQL_MAX_LENGTH = "sfdc.soql.maxlength"; //$NON-NLS-1$
     public static final String PROP_SORT_EXTRACT_FIELDS = "sfdc.sortExtractionFields"; //$NON-NLS-1$
     public static final String PROP_EXTRACT_ALL_CAPS_HEADERS="sfdc.extraction.allCapsHeaders";
     public static final String PROP_EXTRACT_CSV_OUTPUT_BOM="sfdc.extraction.outputByteOrderMark";
@@ -553,6 +554,7 @@ public class AppConfig {
             PROP_MIN_RETRY_SLEEP_SECS,
             PROP_REUSE_CLIENT_CONNECTION,
             CLI_OPTION_RUN_MODE,
+            PROP_SOQL_MAX_LENGTH,
             AppConfig.CLI_OPTION_CONFIG_DIR_PROP,
             AppConfig.PROP_GMT_FOR_DATE_FIELD_VALUE,
             AppConfig.CLI_OPTION_INSTALLATION_CREATE_DESKTOP_SHORTCUT_PROP,
@@ -580,6 +582,7 @@ public class AppConfig {
             PROP_SERVER_ENVIRONMENTS,
             PROP_SELECTED_SERVER_ENVIRONMENT,
             PROP_DAO_SKIP_TOTAL_COUNT,
+            PROP_SOQL_MAX_LENGTH,
             AppConfig.CLI_OPTION_SWT_NATIVE_LIB_IN_JAVA_LIB_PATH,
             AppConfig.CLI_OPTION_INSTALLATION_FOLDER_PROP,
             AppConfig.CLI_OPTION_SYSTEM_PROXY_HOST,
@@ -789,6 +792,7 @@ private void setDefaults(Map<String, String> cliOptionsMap) {
         setDefaultValue(PROP_EXTRACT_ALL_CAPS_HEADERS, false);
         setDefaultValue(PROP_EXTRACT_CSV_OUTPUT_BOM, true);
         setDefaultValue(PROP_LOAD_REMOVE_LEADING_TRAILING_WHITESPACE_IN_IDLOOKUP_FIELD, true);
+        setDefaultValue(PROP_SOQL_MAX_LENGTH, DEFAULT_MAX_SOQL_CHAR_LENGTH);
     }
 
     /**
@@ -1793,6 +1797,7 @@ private synchronized static String getDefaultCharsetForCsvReadWrite() {
     }
 
     private final List<ConfigListener> listeners = new ArrayList<ConfigListener>();
+    public static final int DEFAULT_MAX_SOQL_CHAR_LENGTH = 100000;
 
     public synchronized void addListener(ConfigListener l) {
         listeners.add(l);

diff --git a/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java b/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java
@@ -93,7 +93,6 @@ public SOQLInClauseFromCSVTest(Map<String, String> config) {
 
     @Test
     public void testSoqlInClauseUsingCSV() throws Exception {
-        Map<String, String> configMap = getTestConfig(OperationInfo.insert, false);
         String accountId1 = insertAccount(NAME_VAL+"1", ORACLE_ID_VAL+"1", TestBase.ACCOUNT_NUMBER_PREFIX+"1");
         String accountId2 = insertAccount(NAME_VAL+"2", ORACLE_ID_VAL+"2", TestBase.ACCOUNT_NUMBER_PREFIX+"2");
         String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv";
@@ -117,16 +116,15 @@ public void testSoqlInClauseUsingCSV() throws Exception {
 
     @Test
     public void testSoqlInClauseUsingCSVMultiBatch() throws Exception {
-        AbstractQueryVisitor.setMaxSoqlCharLength(500);
-        Map<String, String> configMap = getTestConfig(OperationInfo.insert, false);
+        testConfig.put(AppConfig.PROP_SOQL_MAX_LENGTH, "500");
         for (int i = 0; i < 20; i++) {
             insertAccount(NAME_VAL + i, ORACLE_ID_VAL + i, TestBase.ACCOUNT_NUMBER_PREFIX + i);
         }
         String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv";
         runExtraction("select name from Account where " + COL_IN_CSV + " in ({" + extractionFileName + "}, {"
                 + COL_IN_CSV + "})", 20);
         validateAccountNamesInOutputFile(NAME_VAL, 20);
-        AbstractQueryVisitor.setMaxSoqlCharLength(AbstractQueryVisitor.DEFAULT_MAX_SOQL_CHAR_LENGTH);
+        testConfig.put(AppConfig.PROP_SOQL_MAX_LENGTH, Integer.toString(AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH));
     }
 
     private void runExtraction(String extractionQuery, int numSuccesses) throws ProcessInitializationException, DataAccessObjectException {