Background merging tools (#562)

* moved event merger to clas-analysis, finalized merging code and created random trigger filtering tools

* dropped DC::doca in bg merging to avoid inconsistency with number of entries in DC::tdc

* fixed bg merging tools usage instructions

* added file splitting to create trigger files with fixed number of events

* added file splitting to create trigger files with fixed number of events

* evio2hipo: added conversion of CND fADC time

* added statistics analysis to triggerBitFilter, bgMerger now handles all detectors based in command line list, added comments

* bug fix in beam current threshold for random trigger selection

* trigger bit filter: moved charge calculation to DaqScalersSequence class

* DaqScalerSequence: added Interval class to handle single scaler readouts

* added tag-2 bank and saving/transfer of non-zero tag events

* bg merging: changed file prefix for file splitter and removed dependency on environment variable

* BG merging: modied trigger bit filtering (only selected bit should be set) and added doublee-hit suppression  on/off option to merging tool

* BG merging: modied trigger bit filtering (only selected bit should be set) and added doublee-hit suppression  on/off option to merging tool

* BG merging added correction for TDC jitter

* BG Merging: more realistic emulation of DC readout, class renaming

* EventMerger: reading constants from CCDB

* EventMerger: fixed problem with CCDB constants

* Event Merger: added missing java docs

* clas-analysis: removed unnecessary dependency in pom file

* restored accidentally dropped RUN::scaler, modify bg merging script to remove bashisms

* renamed bg merging scripts

bin/bg-merger

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+. `dirname $0`/env.sh 
+
+export MALLOC_ARENA_MAX=1
+
+java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.EventMerger $*

bin/trigger-filter

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+. `dirname $0`/env.sh 
+
+export MALLOC_ARENA_MAX=1
+
+java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.RandomTriggerFilter $*

bin/trigger-splitter

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+. `dirname $0`/env.sh 
+
+export MALLOC_ARENA_MAX=1
+
+java -Xmx1536m -Xms1024m -cp "$CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/services/*:$CLAS12DIR/lib/utils/*" org.jlab.analysis.eventmerger.RandomTriggerSplit $*

common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java

@@ -0,0 +1,99 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.jlab.analysis.eventmerger;
+import org.jlab.io.base.DataEvent;
+import org.jlab.io.hipo.HipoDataSource;
+import org.jlab.io.hipo.HipoDataSync;
+import org.jlab.utils.benchmark.ProgressPrintout;
+import org.jlab.utils.options.OptionParser;
+
+/**
+ * Tool for merging of signal and background events
+ *      
+ * Usage : bgMerger -b [background file] -i [input data file] -o [merged file] 
+ * Options :
+ *      -d : list of detectors, for example "DC,FTOF,HTCC" (default = DC,FTOF)
+ *      -n : maximum number of events to process (default = -1)
+ * 
+ * @author ziegler
+ * @author devita
+ * 
+ * FIXME: event tags are not preserved
+ */
+
+
+public class EventMerger {
+
+
+    public EventMerger() {
+    }
+
+
+
+    public static void main(String[] args)  {
+
+        OptionParser parser = new OptionParser("bgMerger");
+        parser.addRequired("-o"    ,"merged file");
+        parser.addRequired("-i"    ,"input data file");
+        parser.addRequired("-b"    ,"background file");
+        parser.setRequiresInputList(false);
+        parser.addOption("-n"    ,"-1", "maximum number of events to process");
+        parser.addOption("-d"    ,"DC,FTOF", "list of detectors, for example \"DC,FTOF,HTCC\"");
+        parser.addOption("-s"    ,"1", "suppress double TDC hits on the same component, 0-no suppression, 1-suppression");
+        parser.parse(args);
+
+        if(parser.hasOption("-i")==true&&parser.hasOption("-o")==true&&parser.hasOption("-b")==true){
+
+            String dataFile   = parser.getOption("-i").stringValue();
+            String outputFile = parser.getOption("-o").stringValue();
+            String bgFile     = parser.getOption("-b").stringValue();
+
+            int     maxEvents  = parser.getOption("-n").intValue();
+            String  detectors  = parser.getOption("-d").stringValue();
+            boolean doubleHits = true;
+            if(parser.getOption("-s").intValue()==0) doubleHits = false;
+
+            EventMerger en = new EventMerger();
+            ADCTDCMerger adctdcMerger = new ADCTDCMerger(detectors.split(","),doubleHits);
+
+            int counter = 0;
+
+            // Readers for event and background
+            HipoDataSource readerData = new HipoDataSource();
+            readerData.open(dataFile);
+            HipoDataSource readerBg = new HipoDataSource();
+            readerBg.open(bgFile);
+
+            //Writer
+            HipoDataSync writer = new HipoDataSync();
+            writer.setCompressionType(2);
+            writer.open(outputFile);
+
+            ProgressPrintout  progress = new ProgressPrintout();
+            while (readerData.hasEvent()&& readerBg.hasEvent()) {
+
+                counter++;
+
+                //System.out.println("************************************************************* ");
+                DataEvent eventData = readerData.getNextEvent();
+                DataEvent eventBg1  = readerBg.getNextEvent();
+                if(!readerBg.hasEvent()) break;
+                DataEvent eventBg2  = readerBg.getNextEvent();
+
+                adctdcMerger.updateEventWithMergedBanks(eventData, eventBg1, eventBg2);
+                writer.writeEvent(eventData);
+                progress.updateStatus();
+                if(maxEvents>0){
+                    if(counter>=maxEvents) break;
+                }
+            }
+            progress.showStatus();
+            writer.close();
+        }
+
+    }
+
+}

common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerConstants.java

@@ -0,0 +1,162 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.jlab.analysis.eventmerger;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.jlab.detector.calib.utils.ConstantsManager;
+import org.jlab.utils.groups.IndexedTable;
+
+/**
+ * Class organizing CCDB constants for background-merging purposes
+ * Includes TDC conversion factor, time jitter constants and DC 
+ * readout parameters
+ * 
+ * @author devita
+ */
+public class EventMergerConstants {
+
+    private ConstantsManager                                 manager   = new ConstantsManager();
+    private final Map <String, Map <EventMergerEnum,Double>> constants = new HashMap<>();
+    private final Map <String, Map <EventMergerEnum,String>> tables    = new HashMap<>();
+    private final Map <EventMergerEnum, EventMergerEnum>     links     = new HashMap<>();
+    private final Map <EventMergerEnum, String>              items     = new HashMap<>();
+
+    private static final String[] tableNames={
+            "/daq/config/dc",
+            "/calibration/dc/time_jitter",
+            "/calibration/ftof/time_jitter",
+            "/calibration/ec/time_jitter",
+            "/calibration/cnd/time_jitter",
+            "/calibration/ctof/time_jitter",            
+    };
+
+    public EventMergerConstants() {
+
+        // fill table map
+        // time jitter
+        setTable("DC",   EventMergerEnum.TIME_JITTER, "/calibration/dc/time_jitter");
+        setTable("FTOF", EventMergerEnum.TIME_JITTER, "/calibration/ftof/time_jitter");
+        setTable("ECAL", EventMergerEnum.TIME_JITTER, "/calibration/ec/time_jitter");
+        setTable("CND",  EventMergerEnum.TIME_JITTER, "/calibration/cnd/time_jitter");
+        setTable("CTOF", EventMergerEnum.TIME_JITTER, "/calibration/ctof/time_jitter");
+        // readout parameters
+        setTable("DC",EventMergerEnum.READOUT_PAR,"/daq/config/dc");
+
+        // fill table items map
+        setLinks(EventMergerEnum.JITTER_CYCLES,   EventMergerEnum.TIME_JITTER);
+        setLinks(EventMergerEnum.JITTER_PERIOD,   EventMergerEnum.TIME_JITTER);
+        setLinks(EventMergerEnum.JITTER_PHASE,    EventMergerEnum.TIME_JITTER);
+        setLinks(EventMergerEnum.READOUT_WINDOW,  EventMergerEnum.READOUT_PAR);
+        setLinks(EventMergerEnum.READOUT_HOLDOFF, EventMergerEnum.READOUT_PAR);
+        setItems(EventMergerEnum.JITTER_CYCLES,   "cycles");
+        setItems(EventMergerEnum.JITTER_PERIOD,   "period");
+        setItems(EventMergerEnum.JITTER_PHASE,    "phase");
+        setItems(EventMergerEnum.READOUT_WINDOW,  "window");
+        setItems(EventMergerEnum.READOUT_HOLDOFF, "holdoff");
+
+        // define additional constants
+        setDouble("DC",  EventMergerEnum.TDC_CONV,  1.0);
+        setDouble("FTOF",EventMergerEnum.TDC_CONV,  0.02345);
+        setDouble("ECAL",EventMergerEnum.TDC_CONV,  0.02345);
+        setDouble("CTOF",EventMergerEnum.TDC_CONV,  0.02345);
+        setDouble("CND", EventMergerEnum.TDC_CONV,  0.02345);
+
+        // initialize manager
+        manager.setVariation("default");
+        manager.init(Arrays.asList(tableNames));
+    }
+
+    private void setTable(String detector, EventMergerEnum key, String path) {
+        if (!tables.containsKey(detector)) tables.put(detector,new HashMap<>());
+        tables.get(detector).put(key,path);
+    }
+
+    private void setLinks(EventMergerEnum item, EventMergerEnum value) {
+        if (!links.containsKey(item)) links.put(item,value);
+    }
+
+    private void setItems(EventMergerEnum item, String value) {
+        if (!items.containsKey(item)) items.put(item,value);
+    }
+
+    private void setDouble(String detector, EventMergerEnum key, Double value) {
+        if(!constants.containsKey(detector)) constants.put(detector, new HashMap<>());
+        constants.get(detector).put(key,value);
+    }
+
+    /**
+     * Get double value for selected detector and constant (used for run-independent constants)
+     * 
+     * @param detector: detector identifier string
+     * @param key:      constant
+     * @return
+     */
+    public double getDouble(String detector, EventMergerEnum key) {
+        if (!constants.containsKey(detector))
+            throw new RuntimeException("Missing Integer Key:  "+detector);
+        if (!constants.get(detector).containsKey(key))
+            throw new RuntimeException("Missing Integer Key:  "+key);
+        return constants.get(detector).get(key);
+    }
+
+    /**
+     * Get double value for selected run number, detector and constant
+     * 
+     * @param run:      run number
+     * @param detector: detector identifier string
+     * @param item:     constant
+     * @return
+     */
+    public double getDouble(int run, String detector, EventMergerEnum item) {
+        IndexedTable table = getTable(run, detector, links.get(item));
+        return table.getDoubleValue(items.get(item), 0, 0, 0);
+    }
+
+    /**
+     * Get integer value for selected run number, detector and constant
+     * 
+     * @param run:      run number
+     * @param detector: detector identifier string
+     * @param item:     constant
+     * @return
+     */
+    public int getInt(int run, String detector, EventMergerEnum item) {
+        IndexedTable table = getTable(run, detector, links.get(item));
+        return table.getIntValue(items.get(item), 0, 0, 0);
+    }
+
+    /**
+     * Get integer value for selected run number, detector  component and constant
+     *
+     * @param run:      run number
+     * @param detector: detector identifier string
+     * @param item:     constant
+     * @param sector
+     * @param layer
+     * @param component
+     * @return
+     */
+    public int getInt(int run, String detector, EventMergerEnum item, int sector, int layer, int component) {
+        IndexedTable table = getTable(run, detector, links.get(item));
+        return table.getIntValue(items.get(item), sector, layer, component);
+    }
+
+    private String getTable(String detector, EventMergerEnum key) {
+        if (!tables.containsKey(detector))
+            throw new RuntimeException("Missing  Key:  "+detector);
+        if (!tables.get(detector).containsKey(key))
+            throw new RuntimeException("Missing  Key:  "+key);
+        return tables.get(detector).get(key);
+    }
+
+    private IndexedTable getTable(int run, String detector, EventMergerEnum key) {
+        String path = getTable(detector, key);
+        return manager.getConstants(run, path);
+    }
+
+}

common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMergerEnum.java

@@ -0,0 +1,22 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.jlab.analysis.eventmerger;
+
+/**
+ * Event merger constants enum
+ * 
+ * @author devita
+ */
+public enum EventMergerEnum {
+    TIME_JITTER,
+    READOUT_PAR,
+    TDC_CONV,
+    JITTER_PHASE,
+    JITTER_CYCLES,
+    JITTER_PERIOD,
+    READOUT_WINDOW,
+    READOUT_HOLDOFF
+}