JDAQSplit.cc

Go to the documentation of this file.

 
#include <string>
#include <iostream>
#include <iomanip>
#include <algorithm>
 
#include "JDAQ/JDAQTimesliceIO.hh"
#include "JDAQ/JDAQEventIO.hh"
#include "JDAQ/JDAQSummarysliceIO.hh"
#include "JDAQ/JDAQEvaluator.hh"
 
#include "JTrigger/JTriggerParameters.hh"
 
#include "JSupport/JSingleFileScanner.hh"
#include "JSupport/JTreeScanner.hh"
#include "JSupport/JFileRecorder.hh"
#include "JSupport/JTriggerParametersSupportkit.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JSupport.hh"
#include "JSupport/JSupportToolkit.hh"
#include "JROOT/JROOTClassSelector.hh"
 
#include "JLang/JSinglePointer.hh"
 
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"
 
 
namespace {
 
  static const char WILDCARD = '%';     //!< wild card for output files
 
  using namespace JPP;
  using namespace KM3NETDAQ;
 
 
  /**
   * Interface for copying part of data.
   */
  struct JCopyInterface {
    /**
     * Virtual destructor.
     */
    virtual ~JCopyInterface()
    {}
 
    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    virtual void copy(const JDAQUTCTimeRange& range) = 0;
  };
  
 
  /**
   * Implementation for copying part of data.
   */
  template<class T>
  struct JCopy :
    public JCopyInterface
  {
    /**
     * Copy data from input file to output.
     *
     * \param  file_name               file name
     * \param  out                     output
     */
    JCopy(const std::string& file_name, JObjectOutput<T>& out) :
      in (file_name),
      out(out)
    {
      i = in.begin();
    }
 
    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    virtual void copy(const JDAQUTCTimeRange& range)
    {
      for ( ; i != in.end() && i->getTimesliceStart() <  range.getLowerLimit(); ++i) {}
      for ( ; i != in.end() && i->getTimesliceStart() <= range.getUpperLimit(); ++i) { out.put(*i); }
    }
    
 
  private:
    JTreeScanner<T, JDAQEvaluator> in;
    JObjectOutput<T>&              out;
    typename JTreeScanner<T, JDAQEvaluator>::iterator i;
  };
 
 
  /**
   * Copy master.
   */
  template<class JTypelist_t>
  struct JCopyMaster : 
    public std::vector< JSinglePointer<JCopyInterface> >,
    public JFileRecorder<JTYPELIST<JTriggerTypes_t, JMeta>::typelist>
  {
    /**
     * Constructor.
     *
     * \param  input_file              input file
     * \param  selection               selection
     */
    JCopyMaster(const std::string& input_file, const JROOTClassSelection& selection) :
      selection(selection)
    {
      add(JType<JTypelist_t>(), input_file);
    }
 
    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    void copy(const JDAQUTCTimeRange& range)
    {
      for (iterator i = this->begin(); i != this->end(); ++i) {
        (*i)->copy(range);
      }
    }
 
    static int debug;
    const JROOTClassSelection selection;
 
  private:
    /**
     * Recursive method for adding data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    template<class JHead_t, class JTail_t>
    void add(JType< JTypeList<JHead_t, JTail_t> > type, const std::string& input_file)
    {
      add(JType<JHead_t>(), input_file);
      add(JType<JTail_t>(), input_file);
    }
 
    /**
     * Add data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    template<class T>
    void add(JType<T> type, const std::string& input_file)
    {
      if (selection(type)) {
 
        JTreeScanner<T, JDAQEvaluator>::debug = debug;
 
        push_back(new JCopy<T>(input_file, *this));
      }
    }
 
    /**
     * Termination method for adding data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    void add(JType<JNullType> type, const std::string& input_file)
    {}
  };
 
  /**
   * Debug.
   */
  template<class JTypelist_t>
  int JCopyMaster<JTypelist_t>::debug = 0;
 
 
  /**
   * Auxiliary data structure for file splitting.
   */
  struct JSplit {
 
    static const char SEPARATOR = '/';   //!< separator between optional index and total
 
    /**
     * Default constructor.
     */
    JSplit() :
       total(0),
       index(invalid)
    {}
 
    /**
     * Read split from input stream.
     *
     * \param  in         input stream
     * \param  object     split
     * \return            input stream
     */
    friend inline std::istream& operator>>(std::istream& in, JSplit& object)
    {
      using namespace std;
 
      string buffer;
 
      if (in >> buffer) {
 
        const string::size_type pos = buffer.find(SEPARATOR);
 
        if (pos != string::npos) {
 
          if (!(istringstream(buffer.substr(0,pos)) >> object.index)) { in.setstate(ios::badbit); }
          if (!(istringstream(buffer.substr(pos+1)) >> object.total)) { in.setstate(ios::badbit); }
 
        } else {
 
          object.index = invalid;
 
          istringstream(buffer) >> object.total;
        }
      }
 
      return in;
    }
 
    /**
     * Write split to output stream.
     *
     * \param  out        output stream
     * \param  object     split
     * \return            output stream
     */
    friend inline std::ostream& operator<<(std::ostream& out, const JSplit& object)
    {
      using namespace std;
 
      if (object.index != invalid)
        return out << object.index << SEPARATOR << object.total;
      else
        return out << object.total;
    }
 
 
    static constexpr size_t invalid = std::numeric_limits<size_t>::max();  //!< invalid value
  
    size_t total;
    size_t index;
  };
 
 
  /**
   * Compare UTC time ranges.
   *
   * \param  first                   first  UTC time range
   * \param  second                  second UTC time range
   * \param  output_file             file name
   */
  inline bool compare(const JDAQUTCTimeRange& first, const JDAQUTCTimeRange& second)
  {
    return first.getLowerLimit() < second.getLowerLimit();
  }
}
 
 
/**
 * \file
 * Auxiliary program to split DAQ data into multiple output files.
 *
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;
 
  typedef vector<JDAQUTCTimeRange> ranges_type;
 
  std::string         inputFile;
  std::string         outputFile;
  JSplit              numberOfFiles;
  ranges_type         ranges;
  JROOTClassSelection selection      = getROOTClassSelection<JDAQTypes_t>();
  int                 debug;
 
  try {
 
    JParser<> zap("Auxiliary program to split DAQ data into multiple output files.");
 
    zap['f'] = make_field(inputFile);
    zap['o'] = make_field(outputFile);
    zap['N'] = make_field(numberOfFiles)       = JPARSER::initialised();
    zap['r'] = make_field(ranges, 
                          "UTC time range(s)") = JPARSER::initialised();
    zap['C'] = make_field(selection,
                          "Precede name of data structure by a '+' or '-' "
                          "to add or remove data types in the output, respectively."
                          "\nROOT wildcards are accepted.") = JPARSER::initialised();
    zap['d'] = make_field(debug)               = 2;
 
    zap(argc, argv);
  }
  catch(const exception& error) {
    FATAL(error.what() << endl);
  }
 
 
  if ((numberOfFiles.total == 0 &&  ranges.empty()) ||
      (numberOfFiles.total != 0 && !ranges.empty()))  {
    FATAL("Invalid splitting " << numberOfFiles << ' ' << ranges.size() << "; use either option -N or -r." << endl);
  }
 
  const size_t pos = outputFile.find(WILDCARD);
 
  if (pos == string::npos) {
    FATAL("Output file name " << outputFile << " does not contain wild card '" << WILDCARD << "'" << endl);
  }
 
  const JDAQUTCTimeRange total  =  combine(getUTCTimeRange<JDAQSummaryslice>(inputFile),
                                           getUTCTimeRange<JDAQEvent>       (inputFile));
 
  if (!total.is_valid()) {
    FATAL("No (valid) summary or event data in input file " << inputFile << ' ' << total << endl);
  }
 
  NOTICE("Total UTC time range " << total << endl);
 
  if (!ranges.empty()) {
 
    sort(ranges.begin(), ranges.end(), compare);
 
  } else {
 
    const long double T_ns = ((long double) JDAQUTCExtended::getTick() + 1.0e9l * (long double) getTimeDifference(total.getLowerLimit(), total.getUpperLimit())) / (long double) numberOfFiles.total;
 
    for (long double t1 = total.getLowerLimit().getTimeNanoSecond(); t1 < (long double) total.getUpperLimit().getTimeNanoSecond(); t1 += T_ns) {
      ranges.push_back(JDAQUTCTimeRange(JDAQUTCExtended(t1), JDAQUTCExtended(t1 + T_ns)));
    }
  }
 
 
  JCopyMaster<JDAQTypes_t>::debug = debug;
 
  JCopyMaster<JDAQTypes_t> master(inputFile, selection);
 
  const int width = (int) (log10(ranges.size() + 1) + 1);
 
  for (size_t i = 0; i != ranges.size(); ++i) {
 
    if (numberOfFiles.index == numberOfFiles.invalid || numberOfFiles.index == i) {
 
      const string file_name = MAKE_STRING(outputFile.substr(0,pos) << FILL(width,'0') << i << FILL() << outputFile.substr(pos+1));
 
      STATUS("Writing " << file_name << ' ' << ranges[i] << "... " << flush);
 
      master.open(file_name.c_str());
 
      master.put(JMeta(argc, argv));
 
      try {
        master.put(getTriggerParameters(inputFile));
      }
      catch(const exception& error) {}
 
      master.copy(ranges[i]);
 
      master.close();
 
      STATUS("OK" << endl);
    }
  }
}