From 76de10751fb8460ab0e9cb349f24edea7c5d5b98 Mon Sep 17 00:00:00 2001 From: ashitsalesforce Date: Mon, 20 Jan 2025 21:32:34 -0800 Subject: [PATCH] config.property to specify max chars in soql on the client side. config property "sfdc.soql.maxlength" to override default max chars in soql on the client side. --- .../action/visitor/AbstractQueryVisitor.java | 15 ++++++++------- .../salesforce/dataloader/config/AppConfig.java | 5 +++++ .../process/SOQLInClauseFromCSVTest.java | 6 ++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java index e1ba5bcd..93448452 100644 --- a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java +++ b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java @@ -134,13 +134,11 @@ protected boolean writeStatus() { return getConfig().getBoolean(AppConfig.PROP_ENABLE_EXTRACT_STATUS_OUTPUT); } - public static final int DEFAULT_MAX_SOQL_CHAR_LENGTH = 100000; public static final int MAX_IDLOOKUP_FIELD_LENGTH = 255; private int daoLastProcessedRow = 0; private CSVFileReader csvReader = null; private String inClauseColName = null; private int numRows = 0; - private static int maxSoqlCharLength = DEFAULT_MAX_SOQL_CHAR_LENGTH; private String getSoqlForNextBatch() throws OperationException { List inClauseFileAndColumnNameList = parseInClauseForFileAndColumnName(soql); @@ -386,10 +384,6 @@ protected void startWriteExtraction(int size) { protected SOQLMapper getMapper() { return (SOQLMapper)super.getMapper(); } - - public static void setMaxSoqlCharLength(int maxSoqlCharLength) { - AbstractQueryVisitor.maxSoqlCharLength = maxSoqlCharLength; - } private static final String IN_CLAUSE = " IN "; private String constructSoqlFromFile(String soql, CSVFileReader csvReader, String columnName) throws IOException, DataAccessObjectException { @@ -411,8 +405,15 @@ private String constructSoqlFromFile(String soql, CSVFileReader csvReader, Strin boolean firstRowOfCurrentBatch = true; int soqlLength = soqlBuilder.length() + MAX_IDLOOKUP_FIELD_LENGTH + 4 + soqlAfterInClause.length(); + int maxSoqlLength = AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH; + try { + maxSoqlLength = this.controller.getAppConfig().getInt(AppConfig.PROP_SOQL_MAX_LENGTH); + } catch (ParameterLoadException e) { + logger.warn("Error getting max soql length: " + e.getMessage()); + maxSoqlLength = AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH; + } while (daoLastProcessedRow < numRows - && soqlLength < this.maxSoqlCharLength) { + && soqlLength < maxSoqlLength) { if (firstRowOfCurrentBatch) { firstRowOfCurrentBatch = false; } else { diff --git a/src/main/java/com/salesforce/dataloader/config/AppConfig.java b/src/main/java/com/salesforce/dataloader/config/AppConfig.java index 4427f8a6..4f21f347 100644 --- a/src/main/java/com/salesforce/dataloader/config/AppConfig.java +++ b/src/main/java/com/salesforce/dataloader/config/AppConfig.java @@ -335,6 +335,7 @@ public class AppConfig { public static final String PROP_IDLOOKUP_FIELD = "sfdc.externalIdField"; //$NON-NLS-1$ public static final String PROP_EXPORT_BATCH_SIZE = "sfdc.extractionRequestSize"; //$NON-NLS-1$ public static final String PROP_EXTRACT_SOQL = "sfdc.extractionSOQL"; //$NON-NLS-1$ + public static final String PROP_SOQL_MAX_LENGTH = "sfdc.soql.maxlength"; //$NON-NLS-1$ public static final String PROP_SORT_EXTRACT_FIELDS = "sfdc.sortExtractionFields"; //$NON-NLS-1$ public static final String PROP_EXTRACT_ALL_CAPS_HEADERS="sfdc.extraction.allCapsHeaders"; public static final String PROP_EXTRACT_CSV_OUTPUT_BOM="sfdc.extraction.outputByteOrderMark"; @@ -553,6 +554,7 @@ public class AppConfig { PROP_MIN_RETRY_SLEEP_SECS, PROP_REUSE_CLIENT_CONNECTION, CLI_OPTION_RUN_MODE, + PROP_SOQL_MAX_LENGTH, AppConfig.CLI_OPTION_CONFIG_DIR_PROP, AppConfig.PROP_GMT_FOR_DATE_FIELD_VALUE, AppConfig.CLI_OPTION_INSTALLATION_CREATE_DESKTOP_SHORTCUT_PROP, @@ -580,6 +582,7 @@ public class AppConfig { PROP_SERVER_ENVIRONMENTS, PROP_SELECTED_SERVER_ENVIRONMENT, PROP_DAO_SKIP_TOTAL_COUNT, + PROP_SOQL_MAX_LENGTH, AppConfig.CLI_OPTION_SWT_NATIVE_LIB_IN_JAVA_LIB_PATH, AppConfig.CLI_OPTION_INSTALLATION_FOLDER_PROP, AppConfig.CLI_OPTION_SYSTEM_PROXY_HOST, @@ -789,6 +792,7 @@ private void setDefaults(Map cliOptionsMap) { setDefaultValue(PROP_EXTRACT_ALL_CAPS_HEADERS, false); setDefaultValue(PROP_EXTRACT_CSV_OUTPUT_BOM, true); setDefaultValue(PROP_LOAD_REMOVE_LEADING_TRAILING_WHITESPACE_IN_IDLOOKUP_FIELD, true); + setDefaultValue(PROP_SOQL_MAX_LENGTH, DEFAULT_MAX_SOQL_CHAR_LENGTH); } /** @@ -1793,6 +1797,7 @@ private synchronized static String getDefaultCharsetForCsvReadWrite() { } private final List listeners = new ArrayList(); + public static final int DEFAULT_MAX_SOQL_CHAR_LENGTH = 100000; public synchronized void addListener(ConfigListener l) { listeners.add(l); diff --git a/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java b/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java index d6942ae4..de307976 100644 --- a/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java +++ b/src/test/java/com/salesforce/dataloader/process/SOQLInClauseFromCSVTest.java @@ -93,7 +93,6 @@ public SOQLInClauseFromCSVTest(Map config) { @Test public void testSoqlInClauseUsingCSV() throws Exception { - Map configMap = getTestConfig(OperationInfo.insert, false); String accountId1 = insertAccount(NAME_VAL+"1", ORACLE_ID_VAL+"1", TestBase.ACCOUNT_NUMBER_PREFIX+"1"); String accountId2 = insertAccount(NAME_VAL+"2", ORACLE_ID_VAL+"2", TestBase.ACCOUNT_NUMBER_PREFIX+"2"); String extractionFileName = getTestDataDir() + File.separator + "SoqlInClauseFromCSV.csv"; @@ -117,8 +116,7 @@ public void testSoqlInClauseUsingCSV() throws Exception { @Test public void testSoqlInClauseUsingCSVMultiBatch() throws Exception { - AbstractQueryVisitor.setMaxSoqlCharLength(500); - Map configMap = getTestConfig(OperationInfo.insert, false); + testConfig.put(AppConfig.PROP_SOQL_MAX_LENGTH, "500"); for (int i = 0; i < 20; i++) { insertAccount(NAME_VAL + i, ORACLE_ID_VAL + i, TestBase.ACCOUNT_NUMBER_PREFIX + i); } @@ -126,7 +124,7 @@ public void testSoqlInClauseUsingCSVMultiBatch() throws Exception { runExtraction("select name from Account where " + COL_IN_CSV + " in ({" + extractionFileName + "}, {" + COL_IN_CSV + "})", 20); validateAccountNamesInOutputFile(NAME_VAL, 20); - AbstractQueryVisitor.setMaxSoqlCharLength(AbstractQueryVisitor.DEFAULT_MAX_SOQL_CHAR_LENGTH); + testConfig.put(AppConfig.PROP_SOQL_MAX_LENGTH, Integer.toString(AppConfig.DEFAULT_MAX_SOQL_CHAR_LENGTH)); } private void runExtraction(String extractionQuery, int numSuccesses) throws ProcessInitializationException, DataAccessObjectException {