From fd73263188be7652073dbc89abe2c9e47053451b Mon Sep 17 00:00:00 2001 From: ashitsalesforce Date: Fri, 27 Dec 2024 13:15:04 -0800 Subject: [PATCH] correctly display processing information if bulk v2 has multiple jobs correctly display processing information if bulk v2 has multiple jobs --- .../dataloader/action/AbstractAction.java | 8 ++-- .../dataloader/action/AbstractLoadAction.java | 9 +++-- .../dataloader/action/BulkExtractAction.java | 3 +- .../dataloader/action/BulkLoadAction.java | 5 ++- .../dataloader/action/DeleteAction.java | 3 +- .../dataloader/action/InsertAction.java | 3 +- .../action/PartnerExtractAction.java | 3 +- .../action/PartnerExtractAllAction.java | 3 +- .../dataloader/action/UndeleteAction.java | 3 +- .../dataloader/action/UpdateAction.java | 3 +- .../dataloader/action/UpsertAction.java | 3 +- .../action/visitor/AbstractVisitor.java | 17 +++++++- .../action/visitor/DAOLoadVisitor.java | 11 ++++- .../action/visitor/bulk/BulkLoadVisitor.java | 14 +++++-- .../visitor/bulk/BulkV2LoadVisitor.java | 7 +++- .../dataloader/util/LoadRateCalculator.java | 40 ++++++++++++------- 16 files changed, 96 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/salesforce/dataloader/action/AbstractAction.java b/src/main/java/com/salesforce/dataloader/action/AbstractAction.java index 65732bc6..940410c1 100644 --- a/src/main/java/com/salesforce/dataloader/action/AbstractAction.java +++ b/src/main/java/com/salesforce/dataloader/action/AbstractAction.java @@ -31,7 +31,7 @@ import org.apache.logging.log4j.Logger; import com.salesforce.dataloader.util.DLLogManager; - +import com.salesforce.dataloader.util.LoadRateCalculator; import com.salesforce.dataloader.action.progress.ILoaderProgress; import com.salesforce.dataloader.action.visitor.IVisitor; import com.salesforce.dataloader.config.AppConfig; @@ -89,7 +89,9 @@ protected AbstractAction(Controller controller, ILoaderProgress monitor) this.successWriter = null; this.errorWriter = null; } - this.visitor = createVisitor(true); + // Let the IVisitor instance create LoadRateCalculator instance for the first job by passing + // null as the first argument. + this.visitor = createVisitor(null, true); int retries = -1; this.enableRetries = controller.getAppConfig().getBoolean(AppConfig.PROP_ENABLE_RETRIES); if (this.enableRetries) { @@ -107,7 +109,7 @@ protected AbstractAction(Controller controller, ILoaderProgress monitor) protected abstract void checkDao(DataAccessObject dao) throws DataAccessObjectInitializationException; /** @return a new IVisitor object to be used by this action */ - protected abstract IVisitor createVisitor(boolean isFirstJob); + protected abstract IVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob); /** flushes any remaining records to or from the dao * @throws BatchSizeLimitException */ diff --git a/src/main/java/com/salesforce/dataloader/action/AbstractLoadAction.java b/src/main/java/com/salesforce/dataloader/action/AbstractLoadAction.java index fd59079a..54524336 100644 --- a/src/main/java/com/salesforce/dataloader/action/AbstractLoadAction.java +++ b/src/main/java/com/salesforce/dataloader/action/AbstractLoadAction.java @@ -42,6 +42,7 @@ import com.salesforce.dataloader.mapping.LoadMapper; import com.salesforce.dataloader.model.TableRow; import com.salesforce.dataloader.util.DAORowUtil; +import com.salesforce.dataloader.util.LoadRateCalculator; import com.sforce.ws.ConnectionException; import java.util.List; @@ -58,7 +59,7 @@ protected AbstractLoadAction(Controller controller, ILoaderProgress monitor) } @Override - protected abstract DAOLoadVisitor createVisitor(boolean isFirstJob); + protected abstract DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob); @Override protected void checkDao(DataAccessObject dao) throws DataAccessObjectInitializationException { @@ -97,8 +98,10 @@ protected boolean visit() throws DataAccessObjectException, ParameterLoadExcepti // retry the same row again try { if (this.getConfig().isBulkV2APIEnabled()) { - // create a new visitor for a new job - setVisitor(this.createVisitor(false)); + // BulkV2 completed a job. + // Create a new visitor for a new job. However, it should use + // the LoadRateCalculator instance from the current visitor. + setVisitor(this.createVisitor(this.getVisitor().getLoadRateCalculator(), false)); } successfulVisit = getVisitor().visit(daoRow); } catch (BatchSizeLimitException e) { diff --git a/src/main/java/com/salesforce/dataloader/action/BulkExtractAction.java b/src/main/java/com/salesforce/dataloader/action/BulkExtractAction.java index 0ef8fe69..fe24da27 100644 --- a/src/main/java/com/salesforce/dataloader/action/BulkExtractAction.java +++ b/src/main/java/com/salesforce/dataloader/action/BulkExtractAction.java @@ -32,6 +32,7 @@ import com.salesforce.dataloader.action.visitor.bulk.BulkV2QueryVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * Bulk api extract action. @@ -47,7 +48,7 @@ public BulkExtractAction(Controller controller, ILoaderProgress monitor) } @Override - protected IVisitor createVisitor(boolean isFirstJob) { + protected IVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { if (getController().getAppConfig().isBulkV2APIEnabled() ) { return new BulkV2QueryVisitor(this, getController(), getMonitor(), getDao(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/BulkLoadAction.java b/src/main/java/com/salesforce/dataloader/action/BulkLoadAction.java index a6d67a95..d42790f0 100644 --- a/src/main/java/com/salesforce/dataloader/action/BulkLoadAction.java +++ b/src/main/java/com/salesforce/dataloader/action/BulkLoadAction.java @@ -36,6 +36,7 @@ import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; import com.salesforce.dataloader.mapping.LoadMapper; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Jesper Joergensen, Colin Jarvis @@ -49,9 +50,9 @@ public BulkLoadAction(Controller controller, ILoaderProgress monitor) } @Override - protected DAOLoadVisitor createVisitor(boolean isFirstJob) { + protected DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { if (this.getConfig().isBulkV2APIEnabled()) { - return new BulkV2LoadVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter(), isFirstJob); + return new BulkV2LoadVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter(), rateCalculator, isFirstJob); } return new BulkLoadVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/DeleteAction.java b/src/main/java/com/salesforce/dataloader/action/DeleteAction.java index 72f44d99..4bcad7ec 100644 --- a/src/main/java/com/salesforce/dataloader/action/DeleteAction.java +++ b/src/main/java/com/salesforce/dataloader/action/DeleteAction.java @@ -34,6 +34,7 @@ import com.salesforce.dataloader.config.AppConfig; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Lexi Viripaeff @@ -45,7 +46,7 @@ public DeleteAction(Controller controller, ILoaderProgress monitor) throws DataA } @Override - protected DAOLoadVisitor createVisitor(boolean isFirstJob) { + protected DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { if (getController().getAppConfig().isRESTAPIEnabled() && getController().getAppConfig().getBoolean(AppConfig.PROP_DELETE_WITH_EXTERNALID)) { return new RESTDeleteVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); diff --git a/src/main/java/com/salesforce/dataloader/action/InsertAction.java b/src/main/java/com/salesforce/dataloader/action/InsertAction.java index 48da370a..711725ac 100644 --- a/src/main/java/com/salesforce/dataloader/action/InsertAction.java +++ b/src/main/java/com/salesforce/dataloader/action/InsertAction.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.action.visitor.partner.PartnerInsertVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Lexi Viripaeff @@ -43,7 +44,7 @@ public InsertAction(Controller controller, ILoaderProgress monitor) throws DataA } @Override - protected DAOLoadVisitor createVisitor(boolean isFirstJob) { + protected DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { return new PartnerInsertVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/PartnerExtractAction.java b/src/main/java/com/salesforce/dataloader/action/PartnerExtractAction.java index cfc7c4f0..de402216 100644 --- a/src/main/java/com/salesforce/dataloader/action/PartnerExtractAction.java +++ b/src/main/java/com/salesforce/dataloader/action/PartnerExtractAction.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.action.visitor.partner.PartnerQueryVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Lexi Viripaeff @@ -44,7 +45,7 @@ public PartnerExtractAction(Controller controller, ILoaderProgress monitor) } @Override - protected IQueryVisitor createVisitor(boolean isFirstJob) { + protected IQueryVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { return new PartnerQueryVisitor(this, getController(), getMonitor(), getDao(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/PartnerExtractAllAction.java b/src/main/java/com/salesforce/dataloader/action/PartnerExtractAllAction.java index 6c6ff182..3fa8d2f5 100644 --- a/src/main/java/com/salesforce/dataloader/action/PartnerExtractAllAction.java +++ b/src/main/java/com/salesforce/dataloader/action/PartnerExtractAllAction.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.action.visitor.partner.PartnerQueryAllVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * Dataloader action which does a partner api query all operation. @@ -46,7 +47,7 @@ public PartnerExtractAllAction(Controller controller, ILoaderProgress monitor) } @Override - protected IVisitor createVisitor(boolean isFirstJob) { + protected IVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { return new PartnerQueryAllVisitor(this, getController(), getMonitor(), getDao(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/UndeleteAction.java b/src/main/java/com/salesforce/dataloader/action/UndeleteAction.java index 5eccd74e..17b9f262 100644 --- a/src/main/java/com/salesforce/dataloader/action/UndeleteAction.java +++ b/src/main/java/com/salesforce/dataloader/action/UndeleteAction.java @@ -32,6 +32,7 @@ import com.salesforce.dataloader.action.visitor.partner.PartnerUndeleteVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Lexi Viripaeff @@ -43,7 +44,7 @@ public UndeleteAction(Controller controller, ILoaderProgress monitor) throws Dat } @Override - protected DAOLoadVisitor createVisitor(boolean isFirstJob) { + protected DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { return new PartnerUndeleteVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/UpdateAction.java b/src/main/java/com/salesforce/dataloader/action/UpdateAction.java index 6a579874..b79ae86f 100644 --- a/src/main/java/com/salesforce/dataloader/action/UpdateAction.java +++ b/src/main/java/com/salesforce/dataloader/action/UpdateAction.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.action.visitor.rest.RESTUpdateVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Lexi Viripaeff @@ -42,7 +43,7 @@ public UpdateAction(Controller controller, ILoaderProgress monitor) throws DataA } @Override - protected DAOLoadVisitor createVisitor(boolean isFirstJob) { + protected DAOLoadVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { if (getController().getAppConfig().isRESTAPIEnabled()) { return new RESTUpdateVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); } else { diff --git a/src/main/java/com/salesforce/dataloader/action/UpsertAction.java b/src/main/java/com/salesforce/dataloader/action/UpsertAction.java index da46170a..ed3d520b 100644 --- a/src/main/java/com/salesforce/dataloader/action/UpsertAction.java +++ b/src/main/java/com/salesforce/dataloader/action/UpsertAction.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.action.visitor.partner.PartnerUpsertVisitor; import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; +import com.salesforce.dataloader.util.LoadRateCalculator; /** * @author Alex Warshavsky @@ -43,7 +44,7 @@ public UpsertAction(Controller controller, ILoaderProgress monitor) throws DataA } @Override - protected PartnerUpsertVisitor createVisitor(boolean isFirstJob) { + protected PartnerUpsertVisitor createVisitor(LoadRateCalculator rateCalculator, boolean isFirstJob) { return new PartnerUpsertVisitor(getController(), getMonitor(), getSuccessWriter(), getErrorWriter()); } diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractVisitor.java index 1585d421..20e8f9b4 100644 --- a/src/main/java/com/salesforce/dataloader/action/visitor/AbstractVisitor.java +++ b/src/main/java/com/salesforce/dataloader/action/visitor/AbstractVisitor.java @@ -51,16 +51,29 @@ public abstract class AbstractVisitor implements IVisitor { private DataWriter errorWriter; private long errors; private long successes; - private final LoadRateCalculator rateCalculator; + private LoadRateCalculator rateCalculator; public AbstractVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, DataWriter errorWriter) { + this(controller, monitor, successWriter, errorWriter, null); + } + + public AbstractVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, + DataWriter errorWriter, LoadRateCalculator rateCalculator) { this.logger = DLLogManager.getLogger(getClass()); this.controller = controller; this.monitor = monitor; this.successWriter = successWriter; this.errorWriter = errorWriter; - this.rateCalculator = new LoadRateCalculator(); + if (rateCalculator == null) { + this.rateCalculator = new LoadRateCalculator(); + } else { + this.rateCalculator = rateCalculator; + } + } + + public LoadRateCalculator getLoadRateCalculator() { + return this.rateCalculator; } protected abstract boolean writeStatus(); diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java index 5045986e..8a8694ac 100644 --- a/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java +++ b/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java @@ -36,6 +36,8 @@ import org.apache.commons.beanutils.*; import org.apache.commons.text.StringEscapeUtils; import com.salesforce.dataloader.util.DLLogManager; +import com.salesforce.dataloader.util.LoadRateCalculator; + import org.apache.logging.log4j.Logger; import java.util.regex.Matcher; @@ -89,10 +91,15 @@ public abstract class DAOLoadVisitor extends AbstractVisitor implements DAORowVi // - https://www.geeksforgeeks.org/how-to-validate-html-tag-using-regular-expression/# private String richTextRegex = AppConfig.DEFAULT_RICHTEXT_REGEX; private Field[] cachedFieldAttributesForOperation = null; - + protected DAOLoadVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, DataWriter errorWriter) { - super(controller, monitor, successWriter, errorWriter); + this(controller, monitor, successWriter, errorWriter, null); + } + + protected DAOLoadVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, + DataWriter errorWriter, LoadRateCalculator rateCalculator) { + super(controller, monitor, successWriter, errorWriter, rateCalculator); this.columnNames = ((DataReader)controller.getDao()).getColumnNames(); diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java index 402612b0..1ac7e33a 100644 --- a/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java +++ b/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java @@ -51,7 +51,7 @@ import org.apache.commons.beanutils.DynaProperty; import org.apache.logging.log4j.Logger; import com.salesforce.dataloader.util.DLLogManager; - +import com.salesforce.dataloader.util.LoadRateCalculator; import com.salesforce.dataloader.action.progress.ILoaderProgress; import com.salesforce.dataloader.action.visitor.DAOLoadVisitor; import com.salesforce.dataloader.client.DescribeRefObject; @@ -79,6 +79,7 @@ import com.sforce.async.BatchInfo; import com.sforce.async.BatchStateEnum; import com.sforce.async.CSVReader; +import com.sforce.async.JobStateEnum; /** * Visitor for operations using the bulk API client @@ -138,7 +139,12 @@ private static class RowResult { public BulkLoadVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, DataWriter errorWriter) { - super(controller, monitor, successWriter, errorWriter); + this(controller, monitor, successWriter, errorWriter, null); + } + + public BulkLoadVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, + DataWriter errorWriter, LoadRateCalculator rateCalculator) { + super(controller, monitor, successWriter, errorWriter, rateCalculator); this.isDelete = getController().getAppConfig().getOperationInfo().isDelete(); this.jobUtil = new BulkApiVisitorUtil(getController(), getProgressMonitor(), getRateCalculator()); } @@ -417,7 +423,9 @@ private void createBatch(ByteArrayOutputStream os, int numRecords) throws AsyncA } protected void closeJob() throws OperationException, DataAccessObjectException { - if (this.jobUtil.hasJob()) { + if (this.jobUtil.hasJob() + && this.jobUtil.getJobInfo().getState() != JobStateEnum.JobComplete + && this.jobUtil.getJobInfo().getState() != JobStateEnum.Closed) { try { this.jobUtil.awaitCompletionAndCloseJob(); } catch (final AsyncApiException e) { diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkV2LoadVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkV2LoadVisitor.java index e0f9d39f..5996a381 100644 --- a/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkV2LoadVisitor.java +++ b/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkV2LoadVisitor.java @@ -44,6 +44,7 @@ import com.salesforce.dataloader.exception.LoadException; import com.salesforce.dataloader.exception.OperationException; import com.salesforce.dataloader.util.DLLogManager; +import com.salesforce.dataloader.util.LoadRateCalculator; import com.sforce.async.AsyncApiException; public class BulkV2LoadVisitor extends BulkLoadVisitor { @@ -52,8 +53,8 @@ public class BulkV2LoadVisitor extends BulkLoadVisitor { private boolean isFirstJob; public BulkV2LoadVisitor(Controller controller, ILoaderProgress monitor, DataWriter successWriter, - DataWriter errorWriter, boolean isFirstJob) { - super(controller, monitor, successWriter, errorWriter); + DataWriter errorWriter, LoadRateCalculator rateCalculator, boolean isFirstJob) { + super(controller, monitor, successWriter, errorWriter, rateCalculator); this.isFirstJob = isFirstJob; } @@ -84,11 +85,13 @@ protected void getResults() throws AsyncApiException, OperationException, DataAc this.getVisitorUtil().getBulkV2LoadSuccessResults(successWriterFile, !this.isFirstJob); CSVFileReader csvReader = new CSVFileReader(new File(successWriterFile), appConfig, true, false); this.setSuccesses(csvReader.getTotalRows()); + this.getLoadRateCalculator().setNumSuccessesAcrossCompletedJobs(csvReader.getTotalRows()); csvReader.close(); this.getVisitorUtil().getBulkV2LoadErrorResults(errorWriterFile); csvReader = new CSVFileReader(new File(errorWriterFile), appConfig, true, false); this.setErrors(csvReader.getTotalRows()); + this.getLoadRateCalculator().setNumErrorsAcrossCompletedJobs(csvReader.getTotalRows()); csvReader.close(); gotUploadResultsFromServer = true; } diff --git a/src/main/java/com/salesforce/dataloader/util/LoadRateCalculator.java b/src/main/java/com/salesforce/dataloader/util/LoadRateCalculator.java index 78ef901d..112fa615 100644 --- a/src/main/java/com/salesforce/dataloader/util/LoadRateCalculator.java +++ b/src/main/java/com/salesforce/dataloader/util/LoadRateCalculator.java @@ -40,8 +40,10 @@ public class LoadRateCalculator { // TODO: we can probably move all references to this code into a base ProgressMonitor class private Date startTime = null; - private long totalRecordsInJob = 0; + private long totalRecordsAcrossAllJobs = 0; private boolean started = false; + private long numSuccessesAcrossCompletedJobs = 0; + private long numErrorsAcrossCompletedJobs = 0; public LoadRateCalculator() { // do nothing @@ -51,14 +53,16 @@ public synchronized void start(int numRecords) { if (!started) { started = true; this.startTime = new Date(); - this.totalRecordsInJob = numRecords; + this.totalRecordsAcrossAllJobs = numRecords; } } public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob) { final Date currentLoadTime = new Date(); - final long numSuccessInJob = processedRecordsInJob - numErrorsInJob; + final long totalProcessedRecords = processedRecordsInJob + this.numErrorsAcrossCompletedJobs + this.numSuccessesAcrossCompletedJobs; + final long totalErrors = this.numErrorsAcrossCompletedJobs + numErrorsInJob; + final long totalSuccesses = totalProcessedRecords - totalErrors; //final long currentPerMin = numSuccess * 60 * 60; long hourlyProcessingRate; @@ -67,14 +71,14 @@ public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob) if (totalElapsedTimeInSec == 0) { hourlyProcessingRate = 0; } else { - hourlyProcessingRate = (processedRecordsInJob * 60 * 60) / totalElapsedTimeInSec; + hourlyProcessingRate = (totalProcessedRecords * 60 * 60) / totalElapsedTimeInSec; } long remainingTimeInSec = 0; long estimatedTotalTimeInSec = 0; - if (this.totalRecordsInJob > 0 && processedRecordsInJob > 0) { + if (this.totalRecordsAcrossAllJobs > 0 && totalProcessedRecords > 0) { // can estimate remaining time only if a few records are processed already. - estimatedTotalTimeInSec = (long) (totalElapsedTimeInSec * this.totalRecordsInJob / processedRecordsInJob); + estimatedTotalTimeInSec = (long) (totalElapsedTimeInSec * this.totalRecordsAcrossAllJobs / totalProcessedRecords); remainingTimeInSec = estimatedTotalTimeInSec - totalElapsedTimeInSec; } @@ -85,24 +89,32 @@ public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob) // LoadRateCalculator.processedTimeUnknown=Processed {0} of {1} total records. // There are {2} successes and {3} errors. return Messages.getMessage(getClass(), "processedTimeUnknown", - processedRecordsInJob, // {0} - this.totalRecordsInJob, // {1} - numSuccessInJob, // {2} - numErrorsInJob); // {3} + totalProcessedRecords, // {0} + this.totalRecordsAcrossAllJobs, // {1} + totalSuccesses, // {2} + totalErrors); // {3} } // LoadRateCalculator.processed=Processed {0} of {1} total records in {8} minutes, {7} seconds. // There are {5} successes and {6} errors. \nRate: {2} records per hour. // Estimated time to complete: {3} minutes and {4} seconds. return Messages.getMessage(getClass(), "processed", - processedRecordsInJob, // {0} - this.totalRecordsInJob, // {1} + totalProcessedRecords, // {0} + this.totalRecordsAcrossAllJobs, // {1} hourlyProcessingRate, // {2} remainingTimeInMinutes, // {3} remainingSeconds, // {4} - numSuccessInJob, // {5} - numErrorsInJob, // {6} + totalSuccesses, // {5} + totalErrors, // {6} totalElapsedTimeInSec - (60 * elapsedTimeInMinutes), // {7} elapsedTimeInMinutes // {8} ); } + + public void setNumSuccessesAcrossCompletedJobs(long num) { + this.numSuccessesAcrossCompletedJobs = num; + } + + public void setNumErrorsAcrossCompletedJobs(long num) { + this.numErrorsAcrossCompletedJobs = num; + } } \ No newline at end of file