From 0158735545cff02431b596c0d9073b2c0d5200c9 Mon Sep 17 00:00:00 2001 From: raffaelladevita Date: Mon, 10 Aug 2020 23:52:51 -0400 Subject: [PATCH] Background merging tools (#562) * moved event merger to clas-analysis, finalized merging code and created random trigger filtering tools * dropped DC::doca in bg merging to avoid inconsistency with number of entries in DC::tdc * fixed bg merging tools usage instructions * added file splitting to create trigger files with fixed number of events * added file splitting to create trigger files with fixed number of events * evio2hipo: added conversion of CND fADC time * added statistics analysis to triggerBitFilter, bgMerger now handles all detectors based in command line list, added comments * bug fix in beam current threshold for random trigger selection * trigger bit filter: moved charge calculation to DaqScalersSequence class * DaqScalerSequence: added Interval class to handle single scaler readouts * added tag-2 bank and saving/transfer of non-zero tag events * bg merging: changed file prefix for file splitter and removed dependency on environment variable * BG merging: modied trigger bit filtering (only selected bit should be set) and added doublee-hit suppression on/off option to merging tool * BG merging: modied trigger bit filtering (only selected bit should be set) and added doublee-hit suppression on/off option to merging tool * BG merging added correction for TDC jitter * BG Merging: more realistic emulation of DC readout, class renaming * EventMerger: reading constants from CCDB * EventMerger: fixed problem with CCDB constants * Event Merger: added missing java docs * clas-analysis: removed unnecessary dependency in pom file * restored accidentally dropped RUN::scaler, modify bg merging script to remove bashisms * renamed bg merging scripts --- bin/bg-merger | 7 + bin/trigger-filter | 7 + bin/trigger-splitter | 7 + .../analysis/eventmerger/ADCTDCMerger.java | 655 ++++++++++++++++++ .../analysis/eventmerger/EventMerger.java | 99 +++ .../eventmerger/EventMergerConstants.java | 162 +++++ .../analysis/eventmerger/EventMergerEnum.java | 22 + .../jlab/analysis/eventmerger/FilterFcup.java | 120 ++++ .../analysis/eventmerger/FilterTrigger.java | 62 ++ .../eventmerger/RandomTriggerFilter.java | 183 +++++ .../eventmerger/RandomTriggerSplit.java | 102 +++ .../eventmerger/SortedWriterUtils.java | 43 ++ .../detector/decode/DaqScalersSequence.java | 68 +- common-tools/clas-eventmerger/pom.xml | 64 -- .../eventmerger/bgmerger/ADCTDCMerger.java | 276 -------- .../eventmerger/bgmerger/RawHitsMerger.java | 101 --- common-tools/pom.xml | 1 - etc/bankdefs/hipo4/header.json | 10 + 18 files changed, 1540 insertions(+), 449 deletions(-) create mode 100755 bin/bg-merger create mode 100755 bin/trigger-filter create mode 100755 bin/trigger-splitter create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerConstants.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerEnum.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterFcup.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterTrigger.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerFilter.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerSplit.java create mode 100644 common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/SortedWriterUtils.java delete mode 100644 common-tools/clas-eventmerger/pom.xml delete mode 100644 common-tools/clas-eventmerger/src/main/java/org/jlab/eventmerger/bgmerger/ADCTDCMerger.java delete mode 100644 common-tools/clas-eventmerger/src/main/java/org/jlab/eventmerger/bgmerger/RawHitsMerger.java diff --git a/bin/bg-merger b/bin/bg-merger new file mode 100755 index 0000000000..5c881d4b94 --- /dev/null +++ b/bin/bg-merger @@ -0,0 +1,7 @@ +#!/bin/sh + +. `dirname $0`/env.sh + +export MALLOC_ARENA_MAX=1 + +java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.EventMerger $* diff --git a/bin/trigger-filter b/bin/trigger-filter new file mode 100755 index 0000000000..d3b00b1566 --- /dev/null +++ b/bin/trigger-filter @@ -0,0 +1,7 @@ +#!/bin/sh + +. `dirname $0`/env.sh + +export MALLOC_ARENA_MAX=1 + +java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.RandomTriggerFilter $* diff --git a/bin/trigger-splitter b/bin/trigger-splitter new file mode 100755 index 0000000000..c2d257f822 --- /dev/null +++ b/bin/trigger-splitter @@ -0,0 +1,7 @@ +#!/bin/sh + +. `dirname $0`/env.sh + +export MALLOC_ARENA_MAX=1 + +java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.RandomTriggerSplit $* diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java new file mode 100644 index 0000000000..b5bea148c9 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java @@ -0,0 +1,655 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.jlab.analysis.eventmerger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.jlab.detector.base.DetectorType; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.io.hipo.HipoDataEvent; + +/** + * Class for merging of ADC and TDC banks from two events + * Operates on selectable list of detectors (default DC, FTOF) + * + * @author ziegler + * @author devita + */ + +public class ADCTDCMerger { + + private boolean debug = false; + private boolean suppressDoubleHits = true; + private String[] detectors; + private EventMergerConstants constants = new EventMergerConstants(); + + public ADCTDCMerger() { + detectors = new String[]{"DC","FTOF"}; + printDetectors(); + } + + public ADCTDCMerger(String[] dets, boolean dhits) { + suppressDoubleHits = dhits; + System.out.println("Double hits suppression flag set to " + suppressDoubleHits); + detectors = dets; + printDetectors(); + } + + /** + * Reads ADC bank and returns corresponding information + * + * @param detector: detector identifier string + * @param bankDGTZ: selected DataBank + * @return + */ + public List ADCbank(String detector,DataBank bankDGTZ) { + List adcStore = new ArrayList(); + + for (int i = 0; i < bankDGTZ.rows(); i++) { + ADC adcData = null; + byte sector = bankDGTZ.getByte("sector", i); + byte layer = bankDGTZ.getByte("layer", i); + short component = bankDGTZ.getShort("component", i); + byte order = bankDGTZ.getByte("order", i); + int adc = bankDGTZ.getInt("ADC", i); + float time = bankDGTZ.getFloat("time", i); + short ped = bankDGTZ.getShort("ped", i); + if(adc<=0) continue; + if(detector == DetectorType.BST.getName()) { + long timestamp = (int)bankDGTZ.getLong("timestamp", i); + adcData = new ADC(sector,layer,component,order,adc,time,ped,timestamp); + } + else if(detector == DetectorType.BMT.getName() || detector==DetectorType.FMT.getName() || detector==DetectorType.FTTRK.getName()) { + long timestamp = (int)bankDGTZ.getLong("timestamp", i); + int integral = bankDGTZ.getInt("integral", i); + adcData = new ADC(sector,layer,component,order,adc,time,ped,timestamp,integral); + } + else if(detector == DetectorType.BAND.getName()) { + int amplitude = bankDGTZ.getInt("amplitude", i); + adcData = new ADC(sector,layer,component,order,adc,time,ped,amplitude); + } + else { + adcData = new ADC(sector,layer,component,order,adc,time,ped); + } + adcStore.add(adcData); + } + + return adcStore; + } + + /** + * Read TDC bank and return corresponding information + * + * @param bankDGTZ: selected DataBank + * @param offset: offset to be applied to TDC values to compensate for jitter + * @return + */ + public List TDCbank(DataBank bankDGTZ, int offset) { + List tdcStore = new ArrayList(); + for (int i = 0; i < bankDGTZ.rows(); i++) { + byte sector = bankDGTZ.getByte("sector", i); + byte layer = bankDGTZ.getByte("layer", i); + short component = bankDGTZ.getShort("component", i); + byte order = bankDGTZ.getByte("order", i); + int tdc = bankDGTZ.getInt("TDC", i); + if(tdc<=0) continue; + TDC tdcData = new TDC(sector,layer,component,order,tdc+offset); + tdcStore.add(tdcData); + } + return tdcStore; + } + + /** + * Merge TDC banks for data (signal) and background events for selected detector + * In case of multiple hit on same detector element, only first hit in time is kept + * unless the double-hit suppression flag, suppressDoubleHits, is set to false + * + * @param Det + * @param event + * @param bg + * @return + */ + public DataBank getTDCBank(String Det, DataEvent event, DataEvent bg){ + + int offset = getTDCOffset(Det, event, bg); + + String TDCString = Det+"::tdc"; + DataBank bank = null; + if(event.hasBank(TDCString)==true && (bg==null || bg.hasBank(TDCString)==false)) { + bank = event.getBank(TDCString); + if(event.hasBank(TDCString)) { + HipoDataEvent de = (HipoDataEvent) event; + de.removeBank(TDCString); + } + } + else if(event.hasBank(TDCString)==false && bg.hasBank(TDCString)==true) { + bank = bg.getBank(TDCString); + } + else if(event.hasBank(TDCString)==true && bg.hasBank(TDCString)==true) { + List bgTDCs = TDCbank(bg.getBank(TDCString), offset); + List TDCs = TDCbank(event.getBank(TDCString), 0); + + List allTDCs = new ArrayList(); + for(TDC tdc : TDCs) allTDCs.add(tdc); + for(TDC tdc : bgTDCs) allTDCs.add(tdc); + Collections.sort(allTDCs); + + List mergedTDCs = new ArrayList(); + for(int i = 0; i < allTDCs.size(); i++) { + TDC tdc = allTDCs.get(i); + if(mergedTDCs.size()==0) { + if(debug) { + System.out.println("Keeping TDC " + i); + tdc.show(); + } + mergedTDCs.add(tdc); + } + else { + TDC tdcOld = mergedTDCs.get(mergedTDCs.size()-1); + if(!tdc.equalTo(tdcOld) || !suppressDoubleHits) { + if(debug) { + System.out.println("Keeping TDC " + i); + tdc.show(); + } + mergedTDCs.add(tdc); + } + else { + if(debug) { + System.out.println("\tSkipping TDC " + i +"\t"); + tdc.show(); + } + } + } + } + + if(event.hasBank(TDCString)) { + HipoDataEvent de = (HipoDataEvent) event; + de.removeBank(TDCString); + } + bank = event.createBank(TDCString, mergedTDCs.size()); + for (int i = 0; i < mergedTDCs.size(); i++) { + bank.setByte("sector", i, mergedTDCs.get(i).getSector()); + bank.setByte("layer", i, mergedTDCs.get(i).getLayer()); + bank.setShort("component", i, mergedTDCs.get(i).getComponent()); + bank.setInt("TDC", i, mergedTDCs.get(i).getTdc()); + bank.setByte("order", i, mergedTDCs.get(i).getOrder()); + } + } + return bank; + } + + /** + * Merge TDC banks for data (signal) and background events for selected detector + * Use two background events shifted in time to extend the time range of the backgrounds + * Multiple hits on the same components are kept if time distance exceed the holdoff time + * + * @param Det + * @param event + * @param bg1: primary background event + * @param bg2: secondary background event that will be shifted to negative times + * @return + */ + public DataBank getTDCBank(String Det, DataEvent event, DataEvent bg1, DataEvent bg2){ + + int offset1 = getTDCOffset(Det, event, bg1); + int offset2 = getTDCOffset(Det, event, bg2); + + int run = bg2.getBank("RUN::config").getInt("run", 0); + + String TDCString = Det+"::tdc"; + DataBank bank = null; + // if no detector bank is found in the background events then keep the event bank + if(event.hasBank(TDCString)==true && bg1.hasBank(TDCString)==false && bg2.hasBank(TDCString)==false) { + bank = event.getBank(TDCString); + if(event.hasBank(TDCString)) { + HipoDataEvent de = (HipoDataEvent) event; + de.removeBank(TDCString); + } + } + // if no detector bank is found in the phyics event, then keep the bank from the primary background event + else if(event.hasBank(TDCString)==false && bg1.hasBank(TDCString)==true) { + bank = bg1.getBank(TDCString); + } + // if both physics and primary background events have the detector bank, then proceed with merging + else if(event.hasBank(TDCString)==true && bg1.hasBank(TDCString)==true) { + List bgTDCs = TDCbank(bg1.getBank(TDCString), offset1); + // if secondary background event has the relevant detector bank, add the hits shifted in time + if(bg2.hasBank(TDCString)) { + List bg2TDCs = TDCbank(bg2.getBank(TDCString), offset2); + for(TDC tdc : bg2TDCs) { + int value = tdc.getTdc(); + int layer = tdc.getLayer(); + int comp = tdc.getComponent(); + int offset = constants.getInt(run, Det, EventMergerEnum.READOUT_WINDOW, 0, layer, comp); + tdc.setTdc(value-offset); + bgTDCs.add(tdc); + } + } + List TDCs = TDCbank(event.getBank(TDCString), 0); + + List allTDCs = new ArrayList(); + for(TDC tdc : TDCs) allTDCs.add(tdc); + for(TDC tdc : bgTDCs) allTDCs.add(tdc); + Collections.sort(allTDCs); + + List mergedTDCs = new ArrayList(); + for(int i = 0; i < allTDCs.size(); i++) { + TDC tdc = allTDCs.get(i); + if(mergedTDCs.size()==0) { + if(debug) { + System.out.println("Keeping TDC " + i); + tdc.show(); + } + mergedTDCs.add(tdc); + } + else { + TDC tdcOld = mergedTDCs.get(mergedTDCs.size()-1); + if(!tdc.equalTo(tdcOld) || !suppressDoubleHits) { + if(debug) { + System.out.println("Keeping TDC " + i); + tdc.show(); + } + mergedTDCs.add(tdc); + } + else { + double delta = constants.getInt(run, Det, EventMergerEnum.READOUT_HOLDOFF, 0, tdc.getLayer(), tdc.getComponent()) + / constants.getDouble(Det, EventMergerEnum.TDC_CONV); + if(tdc.getTdc()-tdcOld.getTdc() filteredTDCs = new ArrayList(); + for(int i = 0; i < mergedTDCs.size(); i++) { + TDC tdc = mergedTDCs.get(i); + int value = tdc.getTdc(); + int layer = tdc.getLayer(); + int comp = tdc.getComponent(); + if(value>0 && value bgADCs = ADCbank(detector,bg.getBank(ADCString)); + List ADCs = ADCbank(detector,event.getBank(ADCString)); + + List allADCs = new ArrayList(); + for(ADC adc : ADCs) allADCs.add(adc); + for(ADC adc : bgADCs) allADCs.add(adc); + Collections.sort(allADCs); + + List mergedADCs = new ArrayList<>(); + + for(int i = 0; i < allADCs.size(); i++) { + ADC adc = allADCs.get(i); + if(mergedADCs.isEmpty()) { + if(debug) { + System.out.println("\tSkipping ADC " + i +"\t"); + adc.show(); + } + mergedADCs.add(adc); + } + else { + ADC adcOld = mergedADCs.get(mergedADCs.size()-1); + if(!adc.equalTo(adcOld)) { + if(debug) { + System.out.println("Keeping ADC " + i); + adc.show(); + } + mergedADCs.add(adc); + } + else { + if(debug) { + System.out.println("\tSkipping ADC " + i +"\t"); + adc.show(); + } + } + } + } + + if(event.hasBank(ADCString)) { + HipoDataEvent de = (HipoDataEvent) event; + de.removeBank(ADCString); + } + + bank = event.createBank(ADCString, mergedADCs.size()); + + for (int i = 0; i < mergedADCs.size(); i++) { + bank.setByte("sector", i, mergedADCs.get(i).getSector()); + bank.setByte("layer", i, mergedADCs.get(i).getLayer()); + bank.setShort("component", i, mergedADCs.get(i).getComponent()); + bank.setInt("ADC", i, mergedADCs.get(i).getAdc()); + bank.setByte("order", i, mergedADCs.get(i).getOrder()); + bank.setFloat("time", i, mergedADCs.get(i).getTime()); + bank.setShort("ped", i, mergedADCs.get(i).getPedestal()); + + if(detector==DetectorType.BST.getName()) { + bank.setLong("timestamp", i, mergedADCs.get(i).getTimestamp()); + } + if(detector == DetectorType.BMT.getName() || detector==DetectorType.FMT.getName() || detector==DetectorType.FTTRK.getName()) { + bank.setLong("timestamp", i, mergedADCs.get(i).getTimestamp()); + bank.setInt("integral", i, mergedADCs.get(i).getIntegral()); + } + if(detector == DetectorType.BAND.getName()) { + bank.setInt("amplitude", 1, mergedADCs.get(i).getAmplitude()); + } + } + } + //bank.show(); + return bank; + } + + private int getTDCOffset(String detector, DataEvent event, DataEvent bg) { + int offset = getJitterCorrection(event, detector) + - getJitterCorrection(bg, detector); + return offset; + } + + private int getJitterCorrection(DataEvent event, String detector) { + // calculate the trigger time jitter correction + int run = event.getBank("RUN::config").getInt("run", 0); + double tdcconv = constants.getDouble(detector, EventMergerEnum.TDC_CONV); + double period = constants.getDouble(run, detector, EventMergerEnum.JITTER_PERIOD); + int phase = constants.getInt(run, detector, EventMergerEnum.JITTER_PHASE); + int cycles = constants.getInt(run, detector, EventMergerEnum.JITTER_CYCLES); + + double triggerphase=0; + long timestamp = event.getBank("RUN::config").getLong("timestamp", 0); + if(cycles > 0) triggerphase=period*((timestamp+phase)%cycles); + int tdcjitter = (int) (triggerphase/tdcconv); + return tdcjitter; + } + + private void printDetectors() { + System.out.print("\nMerging activated for detectors: "); + for(String det:detectors) System.out.print(det + " "); + System.out.println("\n"); + } + + /** + * Append merged banks to hipo event + * + * @param event + * @param bg1 + * @param bg2 + */ + public void updateEventWithMergedBanks(DataEvent event, DataEvent bg1, DataEvent bg2) { + + if(!event.hasBank("RUN::config") || !bg1.hasBank("RUN::config") || !bg2.hasBank("RUN::config")) { + System.out.println("Missing RUN::config bank"); + return; + } + + if(event.hasBank("DC::doca")) event.removeBank("DC::doca"); + + for(String det:detectors) { + if("BMT".equals(det) || "BST".equals(det) || "FTCAL".equals(det) || "FTHODO".equals(det) || "FMT".equals(det) || "FTTRK".equals(det) || "HTCC".equals(det) || "LTCC".equals(det)) { + event.appendBanks(this.getADCBank(det, event, bg1)); + } + else if("DC".equals(det)) { + event.appendBanks(this.getTDCBank(det, event, bg1, bg2)); + } + else if("BAND".equals(det) || "CND".equals(det) || "CTOF".equals(det) || "ECAL".equals(det) || "FTOF".equals(det)) { + event.appendBanks(this.getADCBank(det, event, bg1),this.getTDCBank(det, event, bg1)); + } + else { + System.out.println("Unknown detector:" + det); + } + } + } + + private class ADC implements Comparable { + private byte sector; + private byte layer; + private short component; + private byte order; + private int adc; + private float time; + private short pedestal; + private long timestamp; + private int integral; + private int amplitude; + + public ADC(byte sector, byte layer, short component, byte order, int adc, float time, short ped) { + this.sector = sector; + this.layer = layer; + this.component = component; + this.order = order; + this.adc = adc; + this.time = time; + this.pedestal = ped; + } + + public ADC(byte sector, byte layer, short component, byte order, int adc, float time, short ped, int amplitude) { + this.sector = sector; + this.layer = layer; + this.component = component; + this.order = order; + this.adc = adc; + this.time = time; + this.pedestal = ped; + this.amplitude = amplitude; + } + + public ADC(byte sector, byte layer, short component, byte order, int adc, float time, short ped, long timestamp) { + this.sector = sector; + this.layer = layer; + this.component = component; + this.order = order; + this.adc = adc; + this.time = time; + this.pedestal = ped; + this.timestamp = timestamp; + } + + public ADC(byte sector, byte layer, short component, byte order, int adc, float time, short ped, long timestamp, int integral) { + this.sector = sector; + this.layer = layer; + this.component = component; + this.order = order; + this.adc = adc; + this.time = time; + this.pedestal = ped; + this.timestamp = timestamp; + this.integral = integral; + } + + public byte getSector() { + return sector; + } + + public byte getLayer() { + return layer; + } + + public short getComponent() { + return component; + } + + public byte getOrder() { + return order; + } + + public int getAdc() { + return adc; + } + + public float getTime() { + return time; + } + + public short getPedestal() { + return pedestal; + } + + public long getTimestamp() { + return timestamp; + } + + public int getIntegral() { + return integral; + } + + public int getAmplitude() { + return amplitude; + } + + public boolean equalTo(ADC o){ + if(this.getSector() == o.getSector() && + this.getLayer() == o.getLayer()&& + this.getComponent() == o.getComponent() && + this.getOrder() == o.getOrder()) + return true; + else return false; + } + + @Override + public int compareTo(ADC o) { + int value = 0; + int CompSec = this.getSector() < o.getSector() ? -1 : this.getSector() == o.getSector() ? 0 : 1; + int CompLay = this.getLayer() < o.getLayer() ? -1 : this.getLayer() == o.getLayer() ? 0 : 1; + int CompCom = this.getComponent() < o.getComponent() ? -1 : this.getComponent()== o.getComponent()? 0 : 1; + int CompOrd = this.getOrder() < o.getOrder() ? -1 : this.getOrder() == o.getOrder()? 0 : 1; + int CompTim = this.getTime() < o.getTime() ? -1 : this.getTime() == o.getTime()? 0 : 1; + + int value3 = ((CompOrd == 0) ? CompTim : CompOrd); + int value2 = ((CompCom == 0) ? value3 : CompCom); + int value1 = ((CompLay == 0) ? value2 : CompLay); + value = ((CompSec == 0) ? value1 : CompSec); + + return value; + } + + public void show() { + System.out.println("Sector/Layer/Component/Order: " + sector + "/"+ layer + "/" + component + "/" + order + " ADC: " + adc + " time: " + time); + } + } + + private class TDC implements Comparable{ + private byte sector; + private byte layer; + private short component; + private byte order; + private int tdc; + + public TDC(byte sector, byte layer, short component, byte order, int tdc) { + this.sector = sector; + this.layer = layer; + this.component = component; + this.order = order; + this.tdc = tdc; + } + + public byte getSector() { + return sector; + } + + public byte getLayer() { + return layer; + } + + public short getComponent() { + return component; + } + + public byte getOrder() { + return order; + } + + public int getTdc() { + return tdc; + } + + public void setTdc(int tdc) { + this.tdc = tdc; + } + + public boolean equalTo(TDC o){ + if(this.getSector() == o.getSector() && + this.getLayer() == o.getLayer()&& + this.getComponent() == o.getComponent() && + this.getOrder() == o.getOrder()) + return true; + else return false; + } + + @Override + public int compareTo(TDC o) { + int value = 0; + int CompSec = this.getSector() < o.getSector() ? -1 : this.getSector() == o.getSector() ? 0 : 1; + int CompLay = this.getLayer() < o.getLayer() ? -1 : this.getLayer() == o.getLayer() ? 0 : 1; + int CompCom = this.getComponent() < o.getComponent() ? -1 : this.getComponent()== o.getComponent()? 0 : 1; + int CompOrd = this.getOrder() < o.getOrder() ? -1 : this.getOrder() == o.getOrder()? 0 : 1; + int CompTim = this.getTdc() < o.getTdc() ? -1 : this.getTdc() == o.getTdc()? 0 : 1; + + int value3 = ((CompOrd == 0) ? CompTim : CompOrd); + int value2 = ((CompCom == 0) ? value3 : CompCom); + int value1 = ((CompLay == 0) ? value2 : CompLay); + value = ((CompSec == 0) ? value1 : CompSec); + + return value; + } + + public void show() { + System.out.println("Sector/Layer/Component/Order: " + sector + "/"+ layer + "/" + component + "/" + order + " TDC: " + tdc); + } + } + +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java new file mode 100644 index 0000000000..5fbe566f0b --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java @@ -0,0 +1,99 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.jlab.analysis.eventmerger; +import org.jlab.io.base.DataEvent; +import org.jlab.io.hipo.HipoDataSource; +import org.jlab.io.hipo.HipoDataSync; +import org.jlab.utils.benchmark.ProgressPrintout; +import org.jlab.utils.options.OptionParser; + +/** + * Tool for merging of signal and background events + * + * Usage : bgMerger -b [background file] -i [input data file] -o [merged file] + * Options : + * -d : list of detectors, for example "DC,FTOF,HTCC" (default = DC,FTOF) + * -n : maximum number of events to process (default = -1) + * + * @author ziegler + * @author devita + * + * FIXME: event tags are not preserved + */ + + +public class EventMerger { + + + public EventMerger() { + } + + + + public static void main(String[] args) { + + OptionParser parser = new OptionParser("bgMerger"); + parser.addRequired("-o" ,"merged file"); + parser.addRequired("-i" ,"input data file"); + parser.addRequired("-b" ,"background file"); + parser.setRequiresInputList(false); + parser.addOption("-n" ,"-1", "maximum number of events to process"); + parser.addOption("-d" ,"DC,FTOF", "list of detectors, for example \"DC,FTOF,HTCC\""); + parser.addOption("-s" ,"1", "suppress double TDC hits on the same component, 0-no suppression, 1-suppression"); + parser.parse(args); + + if(parser.hasOption("-i")==true&&parser.hasOption("-o")==true&&parser.hasOption("-b")==true){ + + String dataFile = parser.getOption("-i").stringValue(); + String outputFile = parser.getOption("-o").stringValue(); + String bgFile = parser.getOption("-b").stringValue(); + + int maxEvents = parser.getOption("-n").intValue(); + String detectors = parser.getOption("-d").stringValue(); + boolean doubleHits = true; + if(parser.getOption("-s").intValue()==0) doubleHits = false; + + EventMerger en = new EventMerger(); + ADCTDCMerger adctdcMerger = new ADCTDCMerger(detectors.split(","),doubleHits); + + int counter = 0; + + // Readers for event and background + HipoDataSource readerData = new HipoDataSource(); + readerData.open(dataFile); + HipoDataSource readerBg = new HipoDataSource(); + readerBg.open(bgFile); + + //Writer + HipoDataSync writer = new HipoDataSync(); + writer.setCompressionType(2); + writer.open(outputFile); + + ProgressPrintout progress = new ProgressPrintout(); + while (readerData.hasEvent()&& readerBg.hasEvent()) { + + counter++; + + //System.out.println("************************************************************* "); + DataEvent eventData = readerData.getNextEvent(); + DataEvent eventBg1 = readerBg.getNextEvent(); + if(!readerBg.hasEvent()) break; + DataEvent eventBg2 = readerBg.getNextEvent(); + + adctdcMerger.updateEventWithMergedBanks(eventData, eventBg1, eventBg2); + writer.writeEvent(eventData); + progress.updateStatus(); + if(maxEvents>0){ + if(counter>=maxEvents) break; + } + } + progress.showStatus(); + writer.close(); + } + + } + +} \ No newline at end of file diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerConstants.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerConstants.java new file mode 100644 index 0000000000..b3a53618e2 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerConstants.java @@ -0,0 +1,162 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.jlab.analysis.eventmerger; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.jlab.detector.calib.utils.ConstantsManager; +import org.jlab.utils.groups.IndexedTable; + +/** + * Class organizing CCDB constants for background-merging purposes + * Includes TDC conversion factor, time jitter constants and DC + * readout parameters + * + * @author devita + */ +public class EventMergerConstants { + + private ConstantsManager manager = new ConstantsManager(); + private final Map > constants = new HashMap<>(); + private final Map > tables = new HashMap<>(); + private final Map links = new HashMap<>(); + private final Map items = new HashMap<>(); + + private static final String[] tableNames={ + "/daq/config/dc", + "/calibration/dc/time_jitter", + "/calibration/ftof/time_jitter", + "/calibration/ec/time_jitter", + "/calibration/cnd/time_jitter", + "/calibration/ctof/time_jitter", + }; + + public EventMergerConstants() { + + // fill table map + // time jitter + setTable("DC", EventMergerEnum.TIME_JITTER, "/calibration/dc/time_jitter"); + setTable("FTOF", EventMergerEnum.TIME_JITTER, "/calibration/ftof/time_jitter"); + setTable("ECAL", EventMergerEnum.TIME_JITTER, "/calibration/ec/time_jitter"); + setTable("CND", EventMergerEnum.TIME_JITTER, "/calibration/cnd/time_jitter"); + setTable("CTOF", EventMergerEnum.TIME_JITTER, "/calibration/ctof/time_jitter"); + // readout parameters + setTable("DC",EventMergerEnum.READOUT_PAR,"/daq/config/dc"); + + // fill table items map + setLinks(EventMergerEnum.JITTER_CYCLES, EventMergerEnum.TIME_JITTER); + setLinks(EventMergerEnum.JITTER_PERIOD, EventMergerEnum.TIME_JITTER); + setLinks(EventMergerEnum.JITTER_PHASE, EventMergerEnum.TIME_JITTER); + setLinks(EventMergerEnum.READOUT_WINDOW, EventMergerEnum.READOUT_PAR); + setLinks(EventMergerEnum.READOUT_HOLDOFF, EventMergerEnum.READOUT_PAR); + setItems(EventMergerEnum.JITTER_CYCLES, "cycles"); + setItems(EventMergerEnum.JITTER_PERIOD, "period"); + setItems(EventMergerEnum.JITTER_PHASE, "phase"); + setItems(EventMergerEnum.READOUT_WINDOW, "window"); + setItems(EventMergerEnum.READOUT_HOLDOFF, "holdoff"); + + // define additional constants + setDouble("DC", EventMergerEnum.TDC_CONV, 1.0); + setDouble("FTOF",EventMergerEnum.TDC_CONV, 0.02345); + setDouble("ECAL",EventMergerEnum.TDC_CONV, 0.02345); + setDouble("CTOF",EventMergerEnum.TDC_CONV, 0.02345); + setDouble("CND", EventMergerEnum.TDC_CONV, 0.02345); + + // initialize manager + manager.setVariation("default"); + manager.init(Arrays.asList(tableNames)); + } + + private void setTable(String detector, EventMergerEnum key, String path) { + if (!tables.containsKey(detector)) tables.put(detector,new HashMap<>()); + tables.get(detector).put(key,path); + } + + private void setLinks(EventMergerEnum item, EventMergerEnum value) { + if (!links.containsKey(item)) links.put(item,value); + } + + private void setItems(EventMergerEnum item, String value) { + if (!items.containsKey(item)) items.put(item,value); + } + + private void setDouble(String detector, EventMergerEnum key, Double value) { + if(!constants.containsKey(detector)) constants.put(detector, new HashMap<>()); + constants.get(detector).put(key,value); + } + + /** + * Get double value for selected detector and constant (used for run-independent constants) + * + * @param detector: detector identifier string + * @param key: constant + * @return + */ + public double getDouble(String detector, EventMergerEnum key) { + if (!constants.containsKey(detector)) + throw new RuntimeException("Missing Integer Key: "+detector); + if (!constants.get(detector).containsKey(key)) + throw new RuntimeException("Missing Integer Key: "+key); + return constants.get(detector).get(key); + } + + /** + * Get double value for selected run number, detector and constant + * + * @param run: run number + * @param detector: detector identifier string + * @param item: constant + * @return + */ + public double getDouble(int run, String detector, EventMergerEnum item) { + IndexedTable table = getTable(run, detector, links.get(item)); + return table.getDoubleValue(items.get(item), 0, 0, 0); + } + + /** + * Get integer value for selected run number, detector and constant + * + * @param run: run number + * @param detector: detector identifier string + * @param item: constant + * @return + */ + public int getInt(int run, String detector, EventMergerEnum item) { + IndexedTable table = getTable(run, detector, links.get(item)); + return table.getIntValue(items.get(item), 0, 0, 0); + } + + /** + * Get integer value for selected run number, detector component and constant + * + * @param run: run number + * @param detector: detector identifier string + * @param item: constant + * @param sector + * @param layer + * @param component + * @return + */ + public int getInt(int run, String detector, EventMergerEnum item, int sector, int layer, int component) { + IndexedTable table = getTable(run, detector, links.get(item)); + return table.getIntValue(items.get(item), sector, layer, component); + } + + private String getTable(String detector, EventMergerEnum key) { + if (!tables.containsKey(detector)) + throw new RuntimeException("Missing Key: "+detector); + if (!tables.get(detector).containsKey(key)) + throw new RuntimeException("Missing Key: "+key); + return tables.get(detector).get(key); + } + + private IndexedTable getTable(int run, String detector, EventMergerEnum key) { + String path = getTable(detector, key); + return manager.getConstants(run, path); + } + +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerEnum.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerEnum.java new file mode 100644 index 0000000000..7a9d4e4b9d --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerEnum.java @@ -0,0 +1,22 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.jlab.analysis.eventmerger; + +/** + * Event merger constants enum + * + * @author devita + */ +public enum EventMergerEnum { + TIME_JITTER, + READOUT_PAR, + TDC_CONV, + JITTER_PHASE, + JITTER_CYCLES, + JITTER_PERIOD, + READOUT_WINDOW, + READOUT_HOLDOFF +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterFcup.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterFcup.java new file mode 100644 index 0000000000..7bca1c4486 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterFcup.java @@ -0,0 +1,120 @@ +package org.jlab.analysis.eventmerger; +import java.util.LinkedHashMap; +import java.util.Map; +import org.jlab.detector.decode.DaqScalersSequence; +import org.jlab.jnp.hipo4.data.*; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.utils.data.TextHistogram; + +/** + * Hipo Reduce Worker: filters events based on beam current + * calculated from integrating scalers + * + * Inputs: minimum accepted beam current + * + * @author devita + */ + +public class FilterFcup implements Worker { + + Bank runConfigBank = null; + DaqScalersSequence chargeSeq = null; + private double charge = -1; + private double current = -1; + private int[] currentBuffer = new int[21]; + private int currentMax = 80; + + public FilterFcup(double current){ + this.current=current; + System.out.println("\nInitializing Faraday Cup reduction: threshold current set to " + this.current + "\n"); + } + + + /** + * Initialize bank schema + * + * @param reader + */ + @Override + public void init(HipoReader reader) { + runConfigBank = new Bank(reader.getSchemaFactory().getSchema("RUN::config")); + } + + /** + * Set sequence of scaler readings + * + * @param sequence + */ + public void setScalerSequence(DaqScalersSequence sequence) { + this.chargeSeq=sequence; + } + + /** + * Event filter: selects events in scaler interval with beam current greater than threshold and build statistics information + * + * @param event + * @return + */ + @Override + public boolean processEvent(Event event) { + event.read(runConfigBank); + + if(runConfigBank.getRows()>0){ + long timeStamp = runConfigBank.getLong("timestamp",0); + + // get beam current + double value=chargeSeq.getInterval(timeStamp).getBeamCurrent(); + + // fill statistics array + int currentBins = currentBuffer.length-1; + if(value>currentMax){ + currentBuffer[currentBins] = currentBuffer[currentBins] + 1; + } else if(value<0){ + currentBuffer[0] = currentBuffer[0]; + } else{ + int bin = (int) (currentBins*value/(currentMax)); + currentBuffer[bin] = currentBuffer[bin] + 1; + } + + // set filter value + if(value>current) return true; + } + return false; + } + + // This function has to be implemented, but not used if + // HipoStream is not trying to classify the events. + @Override + public long clasifyEvent(Event event) { return 0L; } + + /** + * Get Map of beam current values + * @return + */ + public Map getCurrentMap(){ + Map sizeMap = new LinkedHashMap(); + int currentBins = currentBuffer.length-1; + double step = ((double) currentMax)/currentBins; + for(int i = 0; i < currentBins; i++){ + String key = String.format("[%6.1f -%6.1f]", (i*step),(i+1)*step); + sizeMap.put(key, (double) currentBuffer[i]); + } + sizeMap.put("overflow", (double) currentBuffer[currentBins] ); + return sizeMap; + } + + /** + * Show beam current histogram + */ + public void showStats() { + System.out.println("\n\n"); + System.out.println(" BEAM CURRENT HISTOGRAM (ENTRIES ARE EVENTS)\n"); + TextHistogram histo = new TextHistogram(); + Map sizeMap = this.getCurrentMap(); + histo.setPrecision(0); + histo.setMinDecriptorWidth(28); + histo.setWidth(80); + histo.setData(sizeMap); + histo.print(); + } +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterTrigger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterTrigger.java new file mode 100644 index 0000000000..c9119f80ae --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/FilterTrigger.java @@ -0,0 +1,62 @@ +package org.jlab.analysis.eventmerger; +import org.jlab.detector.decode.DaqScalersSequence; +import org.jlab.jnp.hipo4.data.*; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.utils.data.*; + +/** + * Hipo Reduce Worker: filter event based on trigger bit + * + * Inputs: selected trigger bit (0-63) + * Returns "true" is selected bit is set in the trigger bit word and no other bits are set + * @author devita + */ +public class FilterTrigger implements Worker { + + Bank triggerBank = null; + DaqScalersSequence chargeSeq = null; + int bit = -1; + + public FilterTrigger(int bit){ + this.bit=bit; + System.out.println("\nInitializing trigger reduction: bit set to " + this.bit + "\n"); + } + + /** + * Initialize bank schema + * + * @param reader + */ + @Override + public void init(HipoReader reader) { + triggerBank = new Bank(reader.getSchemaFactory().getSchema("RUN::config")); + } + + /** + * Event filter: select events according to trigger bit + * + * @param event + * @return + */ + @Override + public boolean processEvent(Event event) { + event.read(triggerBank); + if(triggerBank.getRows()>0){ + long triggerBit = triggerBank.getLong("trigger",0); + long timeStamp = triggerBank.getLong("timestamp",0); + // Value will be >0 if selected bit is 1 in triggerBit + int value = DataByteUtils.readLong(triggerBit, bit, bit); + // Check that no other bit is set + long thisBit = value*((long) Math.pow(2, bit)); + // If returned true, the event will be write to the output + if(value>0 && thisBit==triggerBit) return true; + } + return false; + } + + // This function has to be implemented, but not used if + // HipoStream is not trying to classify the events. + @Override + public long clasifyEvent(Event event) { return 0L; } + +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerFilter.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerFilter.java new file mode 100644 index 0000000000..bcecdf2546 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerFilter.java @@ -0,0 +1,183 @@ +package org.jlab.analysis.eventmerger; +import java.util.List; +import org.jlab.detector.decode.DaqScalersSequence; +import org.jlab.jnp.hipo4.data.*; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; +import org.jlab.jnp.utils.json.JsonObject; +import org.jlab.utils.benchmark.ProgressPrintout; +import org.jlab.utils.options.OptionParser; +import org.jlab.utils.system.ClasUtilsFile; + + +/** + * Random trigger filtering tool: filters hipo event according to trigger bit + * and beam current to create random-trigger files for background merging + * + * Usage: triggerBitFilter -b [trigger bit] -o [output file] + * Options: + * -c : minimum beam current (default = -1) + * -n : maximum number of events to process (default = -1) + * + * Event is filtered if selected trigger bit is set and no other bit is + * + * @author devita + */ + +public class RandomTriggerFilter { + + FilterTrigger triggerFilter = null; + FilterFcup fcupFilter = null; + + public RandomTriggerFilter(int bit, double current){ + triggerFilter = new FilterTrigger(bit); + fcupFilter = new FilterFcup(current); + } + + private FilterTrigger getTriggerFilter() { + return triggerFilter; + } + + private FilterFcup getFcupFilter() { + return fcupFilter; + } + + private void init(HipoReader reader) { + triggerFilter.init(reader); + fcupFilter.init(reader); + } + + public boolean processEvent(Event event) { + if(triggerFilter.processEvent(event) + && fcupFilter.processEvent(event)) { + return true; + } + else { + return false; + } + } + + /** + * Create Json object with filter settings + * @param triggerBit + * @param minCurrent + * @return + */ + public JsonObject settingsToJson(int triggerBit, double minCurrent){ + + JsonObject filterData = new JsonObject(); + filterData.add("trigger-bit", triggerBit); + filterData.add("current-threshold", minCurrent); + JsonObject json = new JsonObject(); + json.add("filter", filterData); + return json; + } + + /** + * Create hipo bank with Json string saved as byte array + * @param writer + * @param json + * @return + */ + public Bank createFilterBank(HipoWriterSorted writer, JsonObject json){ + + if(writer.getSchemaFactory().hasSchema("RUN::filter")==false) return null; + String jsonString = json.toString(); + //create bank + Bank bank = new Bank(writer.getSchemaFactory().getSchema("RUN::filter"), jsonString.length()); + for (int ii=0; ii inputList = parser.getInputList(); + + if(parser.hasOption("-o")==true&&parser.hasOption("-b")==true){ + + String outputFile = parser.getOption("-o").stringValue(); + int triggerBit = parser.getOption("-b").intValue(); + double minCurrent = parser.getOption("-c").doubleValue(); + int maxEvents = parser.getOption("-n").intValue(); + + if(inputList.isEmpty()==true){ + parser.printUsage(); + System.out.println("\n >>>> error : no input file is specified....\n"); + System.exit(0); + } + + if(triggerBit<0 || triggerBit>63) { + parser.printUsage(); + System.out.println("\n >>>> error : invalid trigger bit....\n"); + System.exit(0); + } + + int counter = 0; + + DaqScalersSequence chargeSeq = DaqScalersSequence.readSequence(inputList); + + //Writer + HipoWriterSorted writer = new HipoWriterSorted(); + writer.getSchemaFactory().initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); + writer.setCompressionType(2); + writer.open(outputFile); + + RandomTriggerFilter filter = new RandomTriggerFilter(triggerBit, minCurrent); + filter.getFcupFilter().setScalerSequence(chargeSeq); + + ProgressPrintout progress = new ProgressPrintout(); + for(String inputFile : inputList){ + + // write tag-1 events + SortedWriterUtils utils = new SortedWriterUtils(); +// utils.writeTag(writer, utils.SCALERTAG, inputFile); + + // Reader + HipoReader reader = new HipoReader(); + reader.setTags(0); + reader.open(inputFile); + filter.init(reader); + + // create tag 1 event with trigger filter information + JsonObject json = filter.settingsToJson(triggerBit, minCurrent); + // write tag-1 event + Event tagEvent = new Event(); + tagEvent.write(filter.createFilterBank(writer, json)); + tagEvent.setEventTag(utils.CONFIGTAG); + writer.addEvent(tagEvent,tagEvent.getEventTag()); + System.out.println("\nAdding tag-1 bank with filter settings..."); + System.out.println(json); + + while (reader.hasNext()) { + + Event event = new Event(); + reader.nextEvent(event); + + counter++; + + if(filter.processEvent(event)) { + writer.addEvent(event, event.getEventTag()); + } + progress.updateStatus(); + if(maxEvents>0){ + if(counter>maxEvents) break; + } + } + progress.showStatus(); + } + writer.close(); + + filter.getFcupFilter().showStats(); + } + } +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerSplit.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerSplit.java new file mode 100644 index 0000000000..f9e9dfdb4d --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/RandomTriggerSplit.java @@ -0,0 +1,102 @@ +package org.jlab.analysis.eventmerger; +import java.util.List; +import org.jlab.jnp.hipo4.data.*; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; +import org.jlab.utils.benchmark.ProgressPrintout; +import org.jlab.utils.options.OptionParser; +import org.jlab.utils.system.ClasUtilsFile; + +/** + * Random trigger file splitting tool: splits a list of input files + * into output files with a given number of tag-0 events + * + * Usage : fileSplitter -o [output file prefix] + * Options : + * -n : maximum number of events to process (default = -1) + * -s : number of events per output file (default = -1) + * + * @author devita + */ +public class RandomTriggerSplit { + + + public RandomTriggerSplit(){ + } + + private HipoWriterSorted openOutputFile(String outputfile){ + HipoWriterSorted writer = null; + writer = new HipoWriterSorted(); + writer.getSchemaFactory().initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); + writer.setCompressionType(2); + writer.open(outputfile); + System.out.println("Open output file " + outputfile); + return writer; + } + + public static void main(String[] args){ + + OptionParser parser = new OptionParser("fileSplitter"); + parser.addOption("-o" ,"", "output file prefix"); + parser.addOption("-n" ,"-1", "maximum number of events to process"); + parser.addOption("-s" ,"-1", "number of events per output file"); + parser.parse(args); + + List inputList = parser.getInputList(); + + if(parser.hasOption("-o")==true){ + + String outputFile = parser.getOption("-o").stringValue(); + int maxInEvents = parser.getOption("-n").intValue(); + int maxOutEvents = parser.getOption("-s").intValue(); + + if(inputList.isEmpty()==true){ + parser.printUsage(); + System.out.println("\n >>>> error : no input file is specified....\n"); + System.exit(0); + } + + int eventCounter = 0; + int fileCounter = 1; + + //Writer + HipoWriterSorted writer = null; + SortedWriterUtils utils = new SortedWriterUtils(); + + RandomTriggerSplit splitter = new RandomTriggerSplit(); + + ProgressPrintout progress = new ProgressPrintout(); + for(String inputFile : inputList){ + // Reader + HipoReader reader = new HipoReader(); + reader.setTags(0); + reader.open(inputFile); + + while (reader.hasNext()) { + // open/close ouputfiles + if(((eventCounter%maxOutEvents)==0 && maxOutEvents!=-1) || writer==null) { + if(writer!=null) writer.close(); + String filename = String.format("%05d", fileCounter) + ".hipo"; + if(!outputFile.equals("")) filename = outputFile + "_" + filename; + writer = splitter.openOutputFile(filename); + utils.writeTag(writer, utils.SCALERTAG, inputFile); + utils.writeTag(writer, utils.CONFIGTAG, inputFile); + fileCounter++; + } + Event event = new Event(); + reader.nextEvent(event); + + eventCounter++; + writer.addEvent(event, event.getEventTag()); + + progress.updateStatus(); + if(maxInEvents>0){ + if(eventCounter>maxInEvents) break; + } + } + progress.showStatus(); + } + writer.close(); + } + } +} diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/SortedWriterUtils.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/SortedWriterUtils.java new file mode 100644 index 0000000000..c9a761d885 --- /dev/null +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/SortedWriterUtils.java @@ -0,0 +1,43 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.jlab.analysis.eventmerger; + +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; + +/** + * Utility methods for the HipoWriterSorted + * + * @author devita + */ +public class SortedWriterUtils { + + public static final int SCALERTAG = 1; + public static final int CONFIGTAG = 2; + + /** + * Write all selected tag events from inputfile, requires writer file to be already open + * @param writer + * @param tag + * @param inputfile + */ + public void writeTag(HipoWriterSorted writer, int tag, String inputfile){ + if(writer==null) return; + HipoReader reader = new HipoReader(); + reader.setTags(tag); + reader.open(inputfile); + while(reader.hasNext()) { + Event eventData = new Event(); + reader.nextEvent(eventData); + writer.addEvent(eventData, eventData.getEventTag()); + } + reader.close(); + System.out.println("\nAdding tag-" + tag + " events from file " + inputfile); + } + + +} diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DaqScalersSequence.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DaqScalersSequence.java index 197a9222d6..7de896b1ff 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DaqScalersSequence.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DaqScalersSequence.java @@ -20,17 +20,55 @@ * @author baltzell */ public class DaqScalersSequence implements Comparator { - - private final List scalers=new ArrayList<>(); - + + public static final double TI_CLOCK_FREQ = 250e6; // Hz + + protected final List scalers=new ArrayList<>(); + + public class Interval { + private DaqScalers previous = null; + private DaqScalers next = null; + protected Interval(DaqScalersSequence seq, long t1, long t2) { + final int idx1 = seq.findIndex(t1); + final int idx2 = seq.findIndex(t2); + if (idx1>=0 && idx20) { + return this.getBeamCharge()/dt; + } + } + return 0; + } + } + @Override public int compare(DaqScalers o1, DaqScalers o2) { if (o1.getTimestamp() < o2.getTimestamp()) return -1; if (o1.getTimestamp() > o2.getTimestamp()) return +1; return 0; } - - private int findIndex(long timestamp) { + + protected int findIndex(long timestamp) { if (this.scalers.isEmpty()) return -1; if (timestamp < this.scalers.get(0).getTimestamp()) return -1; // assume late timestamps are ok and go with last readout, so comment this out: @@ -76,7 +114,23 @@ public DaqScalers get(long timestamp) { if (n>=0) return this.scalers.get(n); return null; } - + + /** + * @param timestamp TI timestamp (i.e. RUN::config.timestamp) + * @return smallest interval of scaler readings around that timestamp + */ + public Interval getInterval(long timestamp) { + return this.getInterval(timestamp,timestamp); + } + /** + * @param t1 first TI timestamp (i.e. RUN::config.timestamp) + * @param t2 second TI timestamp + * @return an interval of scaler readings around those timestamps + */ + public Interval getInterval(long t1,long t2) { + return new Interval(this,t1,t2); + } + /** * This reads tag=1 events for RUN::scaler banks, and initializes and returns * a {@link DaqScalersSequence} that can be used to access the most recent scaler @@ -183,4 +237,4 @@ public static void main(String[] args) { } } -} +} \ No newline at end of file diff --git a/common-tools/clas-eventmerger/pom.xml b/common-tools/clas-eventmerger/pom.xml deleted file mode 100644 index 7f09d929c7..0000000000 --- a/common-tools/clas-eventmerger/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - 4.0.0 - clas-eventmerger - 6.5.6-SNAPSHOT - jar - - - org.jlab.clas - clas12rec - ../../parent/pom.xml - 6.5.6-SNAPSHOT - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - - - - - org.jlab.clas - clas-utils - 6.5.6-SNAPSHOT - - - - org.jlab.clas - clas-io - 6.5.6-SNAPSHOT - - - - org.jlab.clas - clas-geometry - 6.5.6-SNAPSHOT - - - - org.apache.commons - commons-math3 - 3.6.1 - - - ${project.groupId} - clas-reco - ${project.version} - - - - - 1.8 - 1.8 - - - diff --git a/common-tools/clas-eventmerger/src/main/java/org/jlab/eventmerger/bgmerger/ADCTDCMerger.java b/common-tools/clas-eventmerger/src/main/java/org/jlab/eventmerger/bgmerger/ADCTDCMerger.java deleted file mode 100644 index ecb493d334..0000000000 --- a/common-tools/clas-eventmerger/src/main/java/org/jlab/eventmerger/bgmerger/ADCTDCMerger.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package org.jlab.eventmerger.bgmerger; - -import org.jlab.io.base.DataBank; -import org.jlab.io.base.DataEvent; -import org.jlab.io.hipo.HipoDataEvent; -import org.jlab.io.hipo.HipoDataSync; -import org.jlab.jnp.hipo.data.HipoEvent; -/** - * - * @author ziegler - */ -public class ADCTDCMerger { - private HipoDataSync dataSync = new HipoDataSync(); - - public int[][] ADCbank(DataBank bankDGTZ) { - int nCol = bankDGTZ.columns(); - int[][] ADCs = new int[nCol][bankDGTZ.rows()]; - int[] sector = new int[bankDGTZ.rows()]; - int[] layer = new int[bankDGTZ.rows()]; - int[] component = new int[bankDGTZ.rows()]; - int[] adc = new int[bankDGTZ.rows()]; - int[] order = new int[bankDGTZ.rows()]; - int[] time = new int[bankDGTZ.rows()]; - int[] ped = new int[bankDGTZ.rows()]; - int[] integral = new int[bankDGTZ.rows()]; - int[] timstmp = new int[bankDGTZ.rows()]; - - for (int i = 0; i < bankDGTZ.rows(); i++) { - sector[i] = bankDGTZ.getByte("sector", i); - layer[i] = bankDGTZ.getByte("layer", i); - component[i] = bankDGTZ.getShort("component", i); - order[i] = bankDGTZ.getByte("order", i); - adc[i] = bankDGTZ.getInt("ADC", i); - time[i] = (int)(bankDGTZ.getFloat("time", i)*1000); - ped[i] = bankDGTZ.getShort("ped", i); - if(nCol==8) { - timstmp[i] = (int)bankDGTZ.getLong("timestamp", i); - } - if(nCol==9) { - timstmp[i] = (int)bankDGTZ.getLong("timestamp", i); - integral[i] = bankDGTZ.getInt("integral", i); - } - } - ADCs[0] = sector; - ADCs[1] = layer; - ADCs[2] = component; - ADCs[3] = adc; - ADCs[4] = order; - ADCs[5] = time; - ADCs[6] = ped; - if(nCol==8) { - ADCs[7] = timstmp; - } - if(nCol==9) { - ADCs[7] = timstmp; - ADCs[8] = integral; - } - return ADCs; - } - - public int[][] TDCbank(DataBank bankDGTZ) { - int[][] TDCs = new int[5][bankDGTZ.rows()]; - int[] sector = new int[bankDGTZ.rows()]; - int[] layer = new int[bankDGTZ.rows()]; - int[] component = new int[bankDGTZ.rows()]; - int[] tdc = new int[bankDGTZ.rows()]; - int[] order = new int[bankDGTZ.rows()]; - - for (int i = 0; i < bankDGTZ.rows(); i++) { - sector[i] = bankDGTZ.getByte("sector", i); - layer[i] = bankDGTZ.getByte("layer", i); - component[i] = bankDGTZ.getShort("component", i); - order[i] = bankDGTZ.getByte("order", i); - tdc[i] = bankDGTZ.getInt("TDC", i); - } - TDCs[0] = sector; - TDCs[1] = layer; - TDCs[2] = component; - TDCs[3] = tdc; - TDCs[4] = order; - - return TDCs; - } - - public DataBank getTDCBank(String Det, DataEvent event, DataEvent bg){ - - String TDCString = Det+"::tdc"; - DataBank bank = null; - if(event.hasBank(TDCString)==true && bg.hasBank(TDCString)==false) { - bank = event.getBank(TDCString); - if(event.hasBank(TDCString)) { - HipoDataEvent de = (HipoDataEvent) event; - HipoEvent dde = de.getHipoEvent(); - dde.removeGroup(TDCString); - } - } - if(event.hasBank(TDCString)==false && bg.hasBank(TDCString)==true) { - bank = bg.getBank(TDCString); - } - if(event.hasBank(TDCString)==true && bg.hasBank(TDCString)==true) { - int[][] BgTDCs = TDCbank(bg.getBank(TDCString)); - int[][] TDCs = TDCbank(event.getBank(TDCString)); - - int[][] MgTDCs = new int[5][TDCs[0].length+BgTDCs[0].length]; - - for (int i = 0; i < TDCs[0].length; i++) { - for (int j = 0; j < BgTDCs[0].length; j++) { - if(BgTDCs[0][j]==TDCs[0][i] && BgTDCs[1][j]==TDCs[1][i] && BgTDCs[2][j]==TDCs[2][i] - && BgTDCs[4][j]==TDCs[4][i] ) { - if(BgTDCs[3][j]>TDCs[3][i]) { - BgTDCs[3][j]=-999; - } else { - TDCs[3][i]=-999; - } - } - } - } - - for (int i = 0; i < TDCs[0].length; i++) { - MgTDCs[0][i] = TDCs[0][i]; - MgTDCs[1][i] = TDCs[1][i]; - MgTDCs[2][i] = TDCs[2][i]; - MgTDCs[3][i] = TDCs[3][i]; - MgTDCs[4][i] = TDCs[4][i]; - } - for (int i = 0; i < BgTDCs[0].length; i++) { - MgTDCs[0][TDCs[0].length+i] = BgTDCs[0][i]; - MgTDCs[1][TDCs[0].length+i] = BgTDCs[1][i]; - MgTDCs[2][TDCs[0].length+i] = BgTDCs[2][i]; - MgTDCs[3][TDCs[0].length+i] = BgTDCs[3][i]; - MgTDCs[4][TDCs[0].length+i] = BgTDCs[4][i]; - } - if(event.hasBank(TDCString)) { - HipoDataEvent de = (HipoDataEvent) event; - HipoEvent dde = de.getHipoEvent(); - //HipoGroup group = dde.getGroup(TDCString); - dde.removeGroup(TDCString); - } - bank = event.createBank(TDCString, TDCs[0].length+BgTDCs[0].length); - for (int i = 0; i < TDCs[0].length; i++) { - bank.setByte("sector", i, (byte) TDCs[0][i]); - bank.setByte("layer", i, (byte) TDCs[1][i]); - bank.setShort("component", i, (short) TDCs[2][i]); - bank.setInt("TDC", i, (short) TDCs[3][i]); - bank.setByte("order", i, (byte) TDCs[4][i]); - } - for (int i = 0; i < BgTDCs[0].length; i++) { - bank.setByte("sector", TDCs[0].length+i, (byte) BgTDCs[0][i]); - bank.setByte("layer", TDCs[0].length+i, (byte) BgTDCs[1][i]); - bank.setShort("component", TDCs[0].length+i, (short) BgTDCs[2][i]); - bank.setInt("TDC", TDCs[0].length+i, (short) BgTDCs[3][i]); - bank.setByte("order", TDCs[0].length+i, (byte) BgTDCs[4][i]); - } - } - return bank; - } - - - public DataBank getADCBank(String Det, DataEvent event, DataEvent bg){ - - String ADCString = Det+"::adc"; - DataBank bank = null; - if(event.hasBank(ADCString)==true && bg.hasBank(ADCString)==false) { - bank = event.getBank(ADCString); - if(event.hasBank(ADCString)) { - HipoDataEvent de = (HipoDataEvent) event; - HipoEvent dde = de.getHipoEvent(); - dde.removeGroup(ADCString); - } - } - if(event.hasBank(ADCString)==false && bg.hasBank(ADCString)==true) { - bank = bg.getBank(ADCString); - } - if(event.hasBank(ADCString)==true && bg.hasBank(ADCString)==true) { - int[][] BgADCs = ADCbank(bg.getBank(ADCString)); - int[][] ADCs = ADCbank(event.getBank(ADCString)); - - int[][] MgADCs = new int[event.getBank(ADCString).columns()][ADCs[0].length+BgADCs[0].length]; - - for (int i = 0; i < ADCs[0].length; i++) { - for (int j = 0; j < BgADCs[0].length; j++) { - if(BgADCs[0][j]==ADCs[0][i] && BgADCs[1][j]==ADCs[1][i] && BgADCs[2][j]==ADCs[2][i] - && BgADCs[4][j]==ADCs[4][i] ) { - if(BgADCs[5][j]>ADCs[5][i]) { - BgADCs[3][j]=-999; - } else { - ADCs[3][i]=-999; - } - } - } - } - - - for (int i = 0; i < ADCs[0].length; i++) { - for (int j = 0; j 0 ){ - //event.getBank("BST::adc").show(); - //event2.getBank("BST::adc").show(); - //event.getBank("BMT::adc").show(); - //event2.getBank("BMT::adc").show(); - counter++; - //event.show(); - //System.out.println("********************** "); - writer0.writeEvent(event); - aDCTDCMerge.updateEventWithMergedBanks(event, event2); - writer.writeEvent(event); - if (counter > 0) { - t1 = System.currentTimeMillis(); - } - //if(counter>11) - // break; - //event.show(); - - } - } - } - writer0.close(); - writer.close(); - double t = System.currentTimeMillis() - t1; - System.out.println(t1 + " TOTAL PROCESSING TIME = " + (t / (float) counter)); - } - - @Override - public boolean processDataEvent(DataEvent de) { - return true; - } - - @Override - public boolean init() { - return true; - } - -} \ No newline at end of file diff --git a/common-tools/pom.xml b/common-tools/pom.xml index b9fc2439b4..0cc5052829 100644 --- a/common-tools/pom.xml +++ b/common-tools/pom.xml @@ -37,7 +37,6 @@ clas-analysis swim-tools clas-math - diff --git a/etc/bankdefs/hipo4/header.json b/etc/bankdefs/hipo4/header.json index 5efc5133d3..176606d238 100644 --- a/etc/bankdefs/hipo4/header.json +++ b/etc/bankdefs/hipo4/header.json @@ -1,3 +1,4 @@ + [ { "name": "RUN::config", @@ -46,5 +47,14 @@ {"name":"fcup", "type":"F", "info":"Beam charge, integrated from beginning of run, ungated (nano-Coulomb)"}, {"name":"livetime", "type":"F", "info":"Livetime during one scaler period"} ] + }, + { + "name" : "RUN::filter", + "group": 10000, + "item" : 15, + "info" : "Random trigger filter information", + "entries":[ + { "name":"json" , "type":"B", "info":"json character" } + ] } ]