diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java index f44455457..992516ae7 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java @@ -109,7 +109,6 @@ private void printOrders() { public void mergeEvents(DataEvent event, DataEvent bg1, DataEvent bg2) { if(!event.hasBank("RUN::config") || !bg1.hasBank("RUN::config") || !bg2.hasBank("RUN::config")) { - System.out.println("Missing RUN::config bank"); return; } diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java new file mode 100644 index 000000000..b99b90a36 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -0,0 +1,239 @@ +package org.jlab.analysis.postprocess; + +import java.io.File; +import java.nio.file.FileSystems; +import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import org.jlab.logging.DefaultLogger; + +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.data.SchemaFactory; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.io.hipo.HipoDataBank; + +import org.jlab.detector.calib.utils.ConstantsManager; +import org.jlab.detector.scalers.DaqScalers; +import org.jlab.detector.scalers.DaqScalersSequence; +import org.jlab.detector.helicity.HelicityBit; +import org.jlab.detector.helicity.HelicitySequenceDelayed; + +/** + * + * @author baltzell + */ +public class Processor { + + static final Logger logger = Logger.getLogger(Util.class.getName()); + + public static final String CCDB_TABLES[] = {"/runcontrol/fcup","/runcontrol/slm", + "/runcontrol/helicity","/daq/config/scalers/dsc1","/runcontrol/hwp"}; + public static final String DEF_PRELOAD_GLOB = "*.{hipo,h5}"; + + private final String outputPrefix = "tmp_"; + + private boolean initialized; + private ConstantsManager conman = null; + private SchemaFactory schemaFactory = null; + private DaqScalersSequence chargeSequence = null; + private HelicitySequenceDelayed helicitySequence = null; + + public Processor(File file, boolean restream) { + configure(restream, Arrays.asList(file.getAbsolutePath())); + } + + public Processor(String dir, boolean restream) { + configure(restream, findPreloadFiles(dir,DEF_PRELOAD_GLOB)); + } + + public Processor(String dir, String glob, boolean restream) { + configure(restream, findPreloadFiles(dir,glob)); + } + + private void configure(boolean restream, List preloadFiles) { + if (preloadFiles.isEmpty()) { + logger.warning("<<<< No preload files found, postprocessing disabled."); + initialized = false; + } else { + HipoReader r = new HipoReader(); + r.open(preloadFiles.get(0)); + conman = new ConstantsManager(); + conman.init(CCDB_TABLES); + schemaFactory = r.getSchemaFactory(); + helicitySequence = Util.getHelicity(preloadFiles, schemaFactory, restream, conman); + chargeSequence = DaqScalersSequence.readSequence(preloadFiles); + r.close(); + initialized = true; + } + } + + /** + * Get a list of files to preload, from one directory and a glob. + * @param dir + * @param glob + * @return list of preload files + */ + private static List findPreloadFiles(String dir, String glob) { + List ret = new ArrayList<>(); + if (dir != null) { + PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:"+dir+"/"+glob); + for (File f : (new File(dir)).listFiles()) { + if (matcher.matches(f.toPath())) + ret.add(f.getPath()); + } + } + return ret; + } + + /** + * Modify REC::Event/HEL::scaler with the delay-corrected helicity + * @param event + * @param runConfig + * @param recEvent + */ + private void processEventHelicity(DataEvent event, DataBank runConfig, DataBank recEvent) { + HelicityBit hb = helicitySequence.search(runConfig.getLong("timestamp", 0)); + HelicityBit hbraw = helicitySequence.getHalfWavePlate() ? HelicityBit.getFlipped(hb) : hb; + recEvent.setByte("helicity",0,hb.value()); + recEvent.setByte("helicityRaw",0,hbraw.value()); + DataBank helScaler = event.getBank("HEL::scaler"); + if (helScaler.rows()>0) { + event.removeBank("HEL::scaler"); + Util.assignScalerHelicity(runConfig.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence); + event.appendBank(helScaler); + } + } + + /** + * Modify REC::Event/HEL::scaler with the delay-corrected helicity + * @param event + * @param runConfig + * @param recEvent + */ + private void processEventHelicity(Event event, Bank runConfig, Bank recEvent) { + HelicityBit hb = helicitySequence.search(runConfig.getLong("timestamp", 0)); + HelicityBit hbraw = helicitySequence.getHalfWavePlate() ? HelicityBit.getFlipped(hb) : hb; + recEvent.setByte("helicity",0,hb.value()); + recEvent.setByte("helicityRaw",0,hbraw.value()); + Bank helScaler = new Bank(schemaFactory.getSchema("HEL::scaler")); + event.read(helScaler); + if (helScaler.getRows()>0) { + event.remove(schemaFactory.getSchema("HEL::scaler")); + Util.assignScalerHelicity(runConfig.getLong("timestamp",0), helScaler, helicitySequence); + event.write(helScaler); + } + } + + /** + * Modify REC::Event for beam charge and livetime + * @param runConfig + * @param recEvent + */ + private void processEventScalers(DataBank runConfig, DataBank recEvent) { + DaqScalers ds = chargeSequence.get(runConfig.getLong("timestamp", 0)); + if (ds != null) { + recEvent.setFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated()); + recEvent.setDouble("liveTime",0,ds.dsc2.getLivetime()); + } + } + + /** + * Modify REC::Event for beam charge and livetime + * @param runConfig + * @param recEvent + */ + private void processEventScalers(Bank runConfig, Bank recEvent) { + DaqScalers ds = chargeSequence.get(runConfig.getLong("timestamp", 0)); + if (ds != null) { + recEvent.putFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated()); + recEvent.putDouble("liveTime",0,ds.dsc2.getLivetime()); + } + } + + /** + * Postprocess one event + * @param e + */ + public void processEvent(DataEvent e) { + if (!initialized) return; + if (!e.hasBank("RUN::config")) return; + if (!e.hasBank("REC::Event")) return; + DataBank runConfig = e.getBank("RUN::config"); + DataBank recEvent = e.getBank("REC::Event"); + if (runConfig.rows()<1 || recEvent.rows()<1) return; + e.removeBank("REC::Event"); + if (helicitySequence != null) processEventHelicity(e, runConfig, recEvent); + if (chargeSequence != null) processEventScalers(runConfig, recEvent); + e.appendBank(recEvent); + } + + /** + * Postprocess one event + * @param e + */ + public void processEvent(Event e) { + if (!initialized) return; + if (!e.hasBank(schemaFactory.getSchema("RUN::config"))) return; + if (!e.hasBank(schemaFactory.getSchema("REC::Event"))) return; + Bank runConfig = new Bank(schemaFactory.getSchema("RUN::config")); + Bank recEvent = new Bank(schemaFactory.getSchema("REC::Event")); + e.read(runConfig); + e.read(recEvent); + if (runConfig.getRows()<1 || recEvent.getRows()<1) return; + e.remove(schemaFactory.getSchema("REC::Event")); + if (helicitySequence != null) processEventHelicity(e, runConfig, recEvent); + if (chargeSequence != null) processEventScalers(runConfig, recEvent); + e.write(recEvent); + } + + /** + * Create rebuilt files from preload files. + * @param preloadFiles + * @return map of rebuilt:preload files + */ + private Map rebuild(String dir, List preloadFiles) { + File d = new File(dir); + if (!d.canWrite()) { + throw new RuntimeException("No write permissions on "+dir); + } + Map rebuiltFiles = new HashMap<>(); + for (String preloadFile : preloadFiles) { + String rebuiltFile = dir+"/"+outputPrefix+preloadFile.replace(dir+"/",""); + Util.rebuildScalers(conman, preloadFile, rebuiltFile); + rebuiltFiles.put(rebuiltFile,preloadFile); + } + return rebuiltFiles; + } + + /** + * Replace files with new ones. + * @param files map of new:old filenames + */ + private static void replace(Map files) { + for (String rebuiltFile : files.keySet()) { + new File(files.get(rebuiltFile)).delete(); + new File(rebuiltFile).renameTo(new File(files.get(rebuiltFile))); + } + } + + /** + * Replace preload files with rebuilt ones. + */ + private void rebuildAndReplace(List preloadFiles) { + replace(rebuild(".",preloadFiles)); + } + + public static void main(String args[]) { + DefaultLogger.debug(); + Processor p = new Processor(System.getenv("HOME")+"/tmp","r*.hipo",false); + //p.rebuildAndReplace(); + } + +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java index a471522b8..8e99a8ab6 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java @@ -6,7 +6,6 @@ import java.util.List; import org.jlab.detector.calib.utils.ConstantsManager; import org.jlab.detector.calib.utils.RCDBConstants; -import org.jlab.detector.helicity.HelicitySequence; import org.jlab.detector.scalers.DaqScalers; import org.jlab.detector.helicity.HelicitySequenceManager; import org.jlab.jnp.hipo4.data.Bank; @@ -111,7 +110,7 @@ public static void main(String[] args) { runScalerBank = ds.createRunBank(writer.getSchemaFactory()); helScalerBank = ds.createHelicityBank(writer.getSchemaFactory()); - RebuildScalers.assignScalerHelicity(event, helScalerBank, helSeq); + Util.assignScalerHelicity(event, helScalerBank, helSeq); // put modified HEL/RUN::scaler back in the event: event.write(runScalerBank); @@ -125,65 +124,4 @@ public static void main(String[] args) { writer.close(); } - /** - * Assign the delay-corrected helicity to the HEL::scaler bank's rows - * @param event the event containing the scaler reading - * @param bank the HEL::scaler bank - * @param seq previously initialized helicity sequence - */ - public static void assignScalerHelicity(Event event, Bank bank, HelicitySequenceManager seq) { - - // Struck (helicity) scaler readout is always slightly after the helicity - // state change, i.e., as registered in the FADCs, so its true helicity - // is offset by one state from its event: - final int readoutStateOffset = -1; - - // Rows in the HEL::scaler bank correspond to the most recent, consecutive, - // time-ordered, T-stable intervals. The first row is the earliest in - // time, and the last row is the latest. Here we loop over them: - for (int row=0; row>> Initializing helicity configuration from CCDB ...\n"); ConstantsManager conman = new ConstantsManager(); conman.init("/runcontrol/hwp","/runcontrol/helicity"); - final int run = getRunNumber(parser.getInputList().get(0)); + final int run = Util.getRunNumber(parser.getInputList().get(0)); IndexedTable helTable = conman.getConstants(run, "/runcontrol/helicity"); // Initialize the scaler sequence from tag-1 events: @@ -137,7 +137,7 @@ public static void main(String[] args) { if (doHelicityDelay) { recEventBank.putByte("helicity",0,hb.value()); recEventBank.putByte("helicityRaw",0,hbraw.value()); - RebuildScalers.assignScalerHelicity(runConfigBank.getLong("timestamp",0), helScalerBank, helSeq); + Util.assignScalerHelicity(runConfigBank.getLong("timestamp",0), helScalerBank, helSeq); } // Write beam charge to REC::Event: @@ -154,7 +154,7 @@ public static void main(String[] args) { writer.addEvent(event, event.getEventTag()); // Copy config banks to new, tag-1 events: - createTag1Events(writer, event, configEvent, configBanks); + Util.createTag1Events(writer, event, configEvent, configBanks); } reader.close(); @@ -171,32 +171,5 @@ public static void main(String[] args) { LOGGER.info(String.format("Tag1ToEvent: Good Charge Fraction: %.2f%%",100*(float)goodCharge/(goodCharge+badCharge))); } - private static void createTag1Events(HipoWriterSorted writer, Event source, Event destination, Bank... banks) { - destination.reset(); - for (Bank bank : banks) { - source.read(bank); - if (bank.getRows()>0) - destination.write(bank); - } - if (!destination.isEmpty()) - writer.addEvent(destination,1); - } - - private static int getRunNumber(String... filenames) { - Event event = new Event(); - for (String filename : filenames) { - HipoReader reader = new HipoReader(); - reader.open(filename); - Bank bank = new Bank(reader.getSchemaFactory().getSchema("RUN::config")); - while (reader.hasNext()) { - reader.nextEvent(event); - event.read(bank); - if (bank.getRows()>0 && bank.getInt("run",0)>0) - return bank.getInt("run",0); - } - } - return -1; - } - } diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Util.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Util.java new file mode 100644 index 000000000..7ac75bcea --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Util.java @@ -0,0 +1,226 @@ +package org.jlab.analysis.postprocess; + +import java.sql.Time; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.data.SchemaFactory; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; + +import org.jlab.utils.groups.IndexedTable; +import org.jlab.detector.calib.utils.ConstantsManager; +import org.jlab.detector.calib.utils.RCDBConstants; + +import org.jlab.detector.scalers.DaqScalers; +import org.jlab.detector.helicity.HelicitySequence; +import org.jlab.detector.helicity.HelicitySequenceDelayed; +import org.jlab.detector.helicity.HelicitySequenceManager; + +/** + * Static utility methods for postprocessing. + * @author baltzell + */ +class Util { + + static final Logger logger = Logger.getLogger(Util.class.getName()); + + /** + * Assign the delay-corrected helicity to the HEL::scaler bank's rows + * @param event the event containing the scaler reading + * @param bank the HEL::scaler bank + * @param seq previously initialized helicity sequence + */ + public static void assignScalerHelicity(Event event, Bank bank, HelicitySequenceManager seq) { + + // Struck (helicity) scaler readout is always slightly after the helicity + // state change, i.e., as registered in the FADCs, so its true helicity + // is offset by one state from its event: + final int readoutStateOffset = -1; + + // Rows in the HEL::scaler bank correspond to the most recent, consecutive, + // time-ordered, T-stable intervals. The first row is the earliest in + // time, and the last row is the latest. Here we loop over them: + for (int row=0; row filenames, SchemaFactory schema, boolean restream, ConstantsManager conman) { + final int run = getRunNumber(filenames); + IndexedTable helTable = conman.getConstants(run, "/runcontrol/helicity"); + HelicitySequenceDelayed seq = null; + if (helTable.getIntValue("delay", 0,0,0) == 0) { + logger.warning("CCDB's helicity delay is zero, disabling helicity postprocessing."); + } + else { + seq = new HelicitySequenceDelayed(helTable); + if (restream) seq.addStream(schema, conman, filenames); + else seq.initialize(filenames); + } + return seq; + } + + /** + * Rebuild the RUN::scaler and HEL::scaler banks from RAW::scaler + * @param schema + * @param conman + * @param runConfig a RUN::config bank + * @param rawScaler a RAW::scaler bank + * @return rebuilt banks + */ + public static Bank[] rebuildScalers(SchemaFactory schema, ConstantsManager conman, Bank runConfig, Bank rawScaler) { + if (runConfig.getRows()>0 && rawScaler.getRows()>0) { + int run = runConfig.getInt("run", 0); + IndexedTable fcup = conman.getConstants(run, "/runcontrol/fcup"); + IndexedTable slm = conman.getConstants(run, "/runcontrol/slm"); + IndexedTable hel = conman.getConstants(run, "/runcontrol/helicity"); + IndexedTable dsc = conman.getConstants(run, "/daq/config/scalers/dsc1"); + if (fcup != null) { + DaqScalers ds; + if (dsc.getIntValue("frequency",0,0,0) < 2e5) { + ds = DaqScalers.create(rawScaler, fcup, slm, hel, dsc); + } + else { + RCDBConstants rcdb = conman.getRcdbConstants(run); + Time rst = rcdb.getTime("run_start_time"); + Date uet = new Date(runConfig.getInt("unixtime",0)*1000L); + ds = DaqScalers.create(rawScaler, fcup, slm, hel, rst, uet); + } + Bank runScaler = ds.createRunBank(schema); + Bank helScaler = ds.createHelicityBank(schema); + return new Bank[]{runScaler,helScaler}; + } + } + return new Bank[]{}; + } + + /** + * Create a new file with all RUN::scaler and HEL::scaler banks rebuilt + * from RAW::scaler + * @param conman + * @param inputFile + * @param outputFile + */ + public static void rebuildScalers(ConstantsManager conman, String inputFile, String outputFile) { + try (HipoWriterSorted w = new HipoWriterSorted()) { + HipoReader r = new HipoReader(); + r.open(inputFile); + Event event = new Event(); + w.setCompressionType(2); + w.getSchemaFactory().copy(r.getSchemaFactory()); + w.open(outputFile); + Bank runConfig = new Bank(r.getSchemaFactory().getSchema("RUN::config")); + Bank rawScaler = new Bank(r.getSchemaFactory().getSchema("RAW::scaler")); + Bank runScaler = new Bank(r.getSchemaFactory().getSchema("RUN::scaler")); + Bank helScaler = new Bank(r.getSchemaFactory().getSchema("HEL::scaler")); + while (r.hasNext()) { + r.nextEvent(event); + event.read(runConfig); + event.read(rawScaler); + event.remove(runScaler.getSchema()); + event.remove(helScaler.getSchema()); + for (Bank b : rebuildScalers(r.getSchemaFactory(), conman, runConfig, rawScaler)) { + event.write(b); + } + w.addEvent(event, event.getEventTag()); + } + } + } + + /** + * Get the first "good" run number in RUN::config + * @param filenames + * @return run number + */ + public static int getRunNumber(List filenames) { + Event event = new Event(); + for (String filename : filenames) { + HipoReader reader = new HipoReader(); + reader.open(filename); + Bank bank = reader.getBank("RUN::config"); + while (reader.hasNext()) { + reader.nextEvent(event); + event.read(bank); + if (bank.getRows()>0 && bank.getInt("run",0)>0) { + logger.log(Level.INFO, "Found first good run number: {0}", bank.getInt("run",0)); + return bank.getInt("run",0); + } + } + } + return -1; + } + + /** + * @param filenames + * @return first "good" RUN::config.run found + */ + public static int getRunNumber(String... filenames) { + return getRunNumber(Arrays.asList(filenames)); + } + + public static void createTag1Events(HipoWriterSorted writer, Event source, Event destination, Bank... banks) { + destination.reset(); + for (Bank bank : banks) { + source.read(bank); + if (bank.getRows()>0) + destination.write(bank); + } + if (!destination.isEmpty()) + writer.addEvent(destination,1); + } + +} diff --git a/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java b/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java index 97602d06c..b85916273 100644 --- a/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java +++ b/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java @@ -16,7 +16,6 @@ import org.jlab.clara.engine.EngineDataType; import java.util.Arrays; import org.jlab.jnp.hipo4.data.SchemaFactory; -import org.jlab.utils.JsonUtils; import org.json.JSONObject; import org.jlab.logging.DefaultLogger; import org.jlab.utils.ClaraYaml; @@ -27,6 +26,9 @@ */ public class EngineProcessor { + public static final String ENGINE_CLASS_BG = "org.jlab.service.bg.BackgroundEngine"; + public static final String ENGINE_CLASS_PP = "org.jlab.service.postproc.PostprocEngine"; + private final Map processorEngines = new LinkedHashMap<>(); private static final Logger LOGGER = Logger.getLogger(EngineProcessor.class.getPackage().getName()); private boolean updateDictionary = true; @@ -35,6 +37,33 @@ public class EngineProcessor { public EngineProcessor(){} + private ReconstructionEngine findEngine(String clazz) { + for (String k : processorEngines.keySet()) { + if (processorEngines.get(k).getClass().getName().equals(clazz)) { + return processorEngines.get(k); + } + } + return null; + } + + private void setBackgroundFiles(String filenames) { + if (findEngine(ENGINE_CLASS_BG) == null) { + LOGGER.info("Adding BackgroundEngine for -B option."); + addEngine("BG",ENGINE_CLASS_BG); + } + findEngine(ENGINE_CLASS_BG).engineConfigMap.put("filename", filenames); + findEngine(ENGINE_CLASS_BG).init(); + } + + private void setPreloadFiles(String filenames) { + if (findEngine(ENGINE_CLASS_PP) == null) { + LOGGER.info("Adding PostprocEngine for -P option."); + addEngine("BG",ENGINE_CLASS_PP); + } + findEngine(ENGINE_CLASS_PP).engineConfigMap.put("preloadFile", filenames); + findEngine(ENGINE_CLASS_PP).init(); + } + private void updateDictionary(HipoDataSource source, HipoDataSync sync){ SchemaFactory fsync = sync.getWriter().getSchemaFactory(); SchemaFactory fsrc = source.getReader().getSchemaFactory(); @@ -121,7 +150,6 @@ public void initAll(){ "org.jlab.service.raster.RasterEngine", "org.jlab.rec.cvt.services.CVTEngine", "org.jlab.service.ctof.CTOFEngine", - //"org.jlab.service.cnd.CNDEngine", "org.jlab.service.cnd.CNDCalibrationEngine", "org.jlab.service.band.BANDEngine", "org.jlab.service.htcc.HTCCReconstructionService", @@ -324,7 +352,8 @@ public static void main(String[] args){ parser.addOption("-u","true","update dictionary from writer ? "); parser.addOption("-d","1","Debug level [0 - OFF, 1 - ON/default]"); parser.addOption("-S",null,"schema directory"); - parser.setDescription("previously known as notsouseful-util"); + parser.addOption("-B",null,"background file"); + parser.addOption("-P",null,"preload file for post-processing"); parser.parse(args); @@ -380,6 +409,15 @@ else if (config>0){ if (parser.getOption("-S").stringValue() != null) proc.setBanksToKeep(parser.getOption("-S").stringValue()); + // command-line filename for background merging overrides YAML: + if (parser.getOption("-B").stringValue() != null) + proc.setBackgroundFiles(parser.getOption("-B").stringValue()); + + // command-line filename for post-processing overrides YAML: + if (parser.getOption("-P").stringValue() != null) + proc.setPreloadFiles(parser.getOption("-P").stringValue()); + proc.processFile(inputFile,outputFile,nskip,nevents); } + } diff --git a/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java b/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java index 55541cfdd..79b4fc6f1 100644 --- a/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java +++ b/common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java @@ -129,6 +129,10 @@ public void requireConstants(List tables){ } } + public void requireConstants(String... tables){ + requireConstants(Arrays.asList(tables)); + } + public final String getEngineConfiguration(){ return this.engineConfiguration; } @@ -137,12 +141,12 @@ public ConstantsManager getConstantsManager(){ return constManagerMap.get(this.getClass().getName()); } + public String getEngineConfigString(String key, String def) { + return engineConfigMap.containsKey(key) ? engineConfigMap.get(key) : def; + } + public String getEngineConfigString(String key) { - String val=null; - if (this.engineConfigMap.containsKey(key)) { - val=this.engineConfigMap.get(key); - } - return val; + return getEngineConfigString(key, null); } /** diff --git a/common-tools/cnuphys/parent/pom.xml b/common-tools/cnuphys/parent/pom.xml index 6abb26d7e..0590ce905 100644 --- a/common-tools/cnuphys/parent/pom.xml +++ b/common-tools/cnuphys/parent/pom.xml @@ -49,7 +49,7 @@ 3.13.0 UTF-8 - 11 + 17 diff --git a/common-tools/coat-lib/pom.xml b/common-tools/coat-lib/pom.xml index 1c6853147..0cbc71102 100644 --- a/common-tools/coat-lib/pom.xml +++ b/common-tools/coat-lib/pom.xml @@ -187,7 +187,7 @@ 3.8.0 UTF-8 - 11 + 17 diff --git a/etc/ejml/ejmlclas12.network b/etc/ejml/ejmlclas12.network index 25030fe27..16df05b40 100644 Binary files a/etc/ejml/ejmlclas12.network and b/etc/ejml/ejmlclas12.network differ diff --git a/parent/pom.xml b/parent/pom.xml index b66c7a147..f99a88530 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -71,7 +71,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.8.0 + 3.10.0 coatjavadocs coatjavadocs @@ -83,7 +83,7 @@ maven-compiler-plugin 3.13.0 - 11 + 17 UTF-8 diff --git a/reconstruction/bg/pom.xml b/reconstruction/bg/pom.xml new file mode 100644 index 000000000..ab6945173 --- /dev/null +++ b/reconstruction/bg/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + org.jlab.clas12.detector + clas12detector-bg + 1.0-SNAPSHOT + jar + + + org.jlab.clas + clas12rec + ../../parent/pom.xml + 11.0.2-SNAPSHOT + + + + + + org.jlab.clas + clas-io + 11.0.2-SNAPSHOT + + + + org.jlab.clas + clas-reco + 11.0.2-SNAPSHOT + + + + org.jlab.clas + clas-analysis + 11.0.2-SNAPSHOT + + + + diff --git a/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java b/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java new file mode 100644 index 000000000..9ca9e9406 --- /dev/null +++ b/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java @@ -0,0 +1,92 @@ +package org.jlab.service.bg; + +import java.io.File; +import java.util.LinkedList; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.jlab.analysis.eventmerger.EventMerger; +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.io.base.DataEvent; +import org.jlab.io.hipo.HipoDataSource; + +/** + * + * @author baltzell + */ +public class BackgroundEngine extends ReconstructionEngine { + + public static final String CONF_FILENAME = "filename"; + public static final String CONF_DETECTORS = "detectors"; + public static final String CONF_ORDERS = "orders"; + public static final String CONF_SUPPRESS_DOUBLES = "suppressDoubles"; + public static final String CONF_PRESERVE_ORDER = "preserveOrder"; + public static final String CONF_REUSE_EVENTS = "reuseEvents"; + + static final Logger logger = Logger.getLogger(BackgroundEngine.class.getName()); + + int filesUsed = 0; + boolean reuseEvents = true; + EventMerger bgmerger = null; + HipoDataSource bgreader = null; + LinkedList bgfilenames = new LinkedList<>(); + + public BackgroundEngine() { + super("BG", "baltzell", "1.0"); + } + + @Override + public boolean init() { + if (getEngineConfigString(CONF_FILENAME) != null) + return init(getEngineConfigString(CONF_FILENAME).split(",")); + return true; + } + + public boolean init(String... filenames) { + bgfilenames.clear(); + String detectors = getEngineConfigString(CONF_DETECTORS,"DC,FTOF"); + String orders = getEngineConfigString(CONF_ORDERS,"NOMINAL"); + boolean suppressDoubles = Boolean.valueOf(getEngineConfigString(CONF_SUPPRESS_DOUBLES,"true")); + boolean preserveOrder = Boolean.valueOf(getEngineConfigString(CONF_PRESERVE_ORDER,"true")); + boolean reuseEvents = Boolean.valueOf(getEngineConfigString(CONF_REUSE_EVENTS,"false")); + for (String filename : filenames) { + File f = new File(filename); + if (!f.exists() || !f.isFile() || !f.canRead()) { + logger.log(Level.SEVERE,"BackgroundEngine:: filename {0} invalid.",filename); + return false; + } + logger.log(Level.INFO,"BackgroundEngine:: reading {0}",filename); + bgfilenames.add(filename); + } + bgmerger = new EventMerger(detectors.split(","), orders.split(","), suppressDoubles, preserveOrder); + openNextFile(); + return true; + } + + private void openNextFile() { + if (filesUsed>0 && filesUsed%bgfilenames.size()==0) { + if (reuseEvents) logger.info("BackgroundEngine:: Reopening previously used file."); + else throw new RuntimeException("BackgroundEngine:: Ran out of events."); + } + String filename = bgfilenames.remove(); + bgfilenames.add(filename); + bgreader = new HipoDataSource(); + bgreader.open(filename); + filesUsed++; + } + + synchronized public DataEvent getBackgroundEvent() { + if (!bgreader.hasEvent()) openNextFile(); + return bgreader.getNextEvent(); + } + + @Override + public boolean processDataEvent(DataEvent event) { + if (!bgfilenames.isEmpty()) { + DataEvent a = getBackgroundEvent(); + DataEvent b = getBackgroundEvent(); + bgmerger.mergeEvents(event, a, b); + } + return true; + } + +} diff --git a/reconstruction/pom.xml b/reconstruction/pom.xml index 108255458..a7565a73c 100644 --- a/reconstruction/pom.xml +++ b/reconstruction/pom.xml @@ -34,5 +34,7 @@ vtx urwell alert + bg + postproc diff --git a/reconstruction/postproc/pom.xml b/reconstruction/postproc/pom.xml new file mode 100644 index 000000000..e9ed05d22 --- /dev/null +++ b/reconstruction/postproc/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + org.jlab.clas12.detector + clas12detector-postproc + 1.0-SNAPSHOT + jar + + + org.jlab.clas + clas12rec + ../../parent/pom.xml + 11.0.2-SNAPSHOT + + + + + + org.jlab.clas + clas-io + 11.0.2-SNAPSHOT + + + + org.jlab.clas + clas-reco + 11.0.2-SNAPSHOT + + + + org.jlab.clas + clas-analysis + 11.0.2-SNAPSHOT + + + + diff --git a/reconstruction/postproc/src/main/java/org/jlab/service/postproc/PostprocEngine.java b/reconstruction/postproc/src/main/java/org/jlab/service/postproc/PostprocEngine.java new file mode 100644 index 000000000..66e7819a6 --- /dev/null +++ b/reconstruction/postproc/src/main/java/org/jlab/service/postproc/PostprocEngine.java @@ -0,0 +1,58 @@ +package org.jlab.service.postproc; + +import java.io.File; +import java.util.logging.Logger; +import org.jlab.analysis.postprocess.Processor; +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.io.base.DataEvent; + +/** + * Post-processing as an engine. + * @author baltzell + */ +public class PostprocEngine extends ReconstructionEngine { + + public static final String CONF_PRELOAD_FILE = "preloadFile"; + public static final String CONF_PRELOAD_DIR = "preloadDir"; + public static final String CONF_PRELOAD_GLOB = "preloadGlob"; + public static final String CONF_RESTREAM_HELICITY = "restream"; + public static final String CONF_REBUILD_SCALERS = "rebuild"; + + static final Logger logger = Logger.getLogger(PostprocEngine.class.getName()); + + Processor processor = null; + + public PostprocEngine() { + super("PP", "baltzell", "1.0"); + } + + @Override + public boolean init() { + requireConstants(Processor.CCDB_TABLES); + if (getEngineConfigString(CONF_PRELOAD_FILE) != null) { + if (getEngineConfigString(CONF_PRELOAD_DIR) != null) + logger.warning("PostprocEngine:: Using preloadFile, ignoring preloadDir."); + processor = new Processor(new File(getEngineConfigString(CONF_PRELOAD_FILE)), + Boolean.parseBoolean(getEngineConfigString(CONF_RESTREAM_HELICITY,"false"))); + } + if (getEngineConfigString(CONF_PRELOAD_DIR) != null) { + processor = new Processor( + getEngineConfigString(CONF_PRELOAD_DIR), + getEngineConfigString(CONF_PRELOAD_GLOB, Processor.DEF_PRELOAD_GLOB), + Boolean.parseBoolean(getEngineConfigString(CONF_RESTREAM_HELICITY,"false"))); + } + if (null != getEngineConfigString(CONF_REBUILD_SCALERS)) { + if (Boolean.getBoolean(getEngineConfigString(CONF_REBUILD_SCALERS))) { + //processor.rebuildAndReplace(); + } + } + return true; + } + + @Override + public boolean processDataEvent(DataEvent event) { + if (processor != null) processor.processEvent(event); + return true; + } + +}