JDAQSplit.cc

Go to the documentation of this file.

#include <string>
#include <iostream>
#include <iomanip>
#include <algorithm>

#include "JDAQ/JDAQTimesliceIO.hh"
#include "JDAQ/JDAQEventIO.hh"
#include "JDAQ/JDAQSummarysliceIO.hh"
#include "JDAQ/JDAQEvaluator.hh"

#include "JTrigger/JTriggerParameters.hh"

#include "JSupport/JSingleFileScanner.hh"
#include "JSupport/JTreeScanner.hh"
#include "JSupport/JFileRecorder.hh"
#include "JSupport/JTriggerParametersSupportkit.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JSupport.hh"
#include "JSupport/JSupportToolkit.hh"
#include "JROOT/JROOTClassSelector.hh"

#include "JLang/JSinglePointer.hh"

#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"


namespace {

  static const char WILDCARD = '%';     //!< wild card for output files

  using namespace JPP;
  using namespace KM3NETDAQ;


  /**
   * Interface for copying part of data.
   */
  struct JCopyInterface {
    /**
     * Virtual destructor.
     */
    virtual ~JCopyInterface()
    {}

    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    virtual void copy(const JDAQUTCTimeRange& range) = 0;
  };
  

  /**
   * Implementation for copying part of data.
   */
  template<class T>
  struct JCopy :
    public JCopyInterface
  {
    /**
     * Copy data from input file to output.
     *
     * \param  file_name               file name
     * \param  out                     output
     */
    JCopy(const std::string& file_name, JObjectOutput<T>& out) :
      in (file_name),
      out(out)
    {
      i = in.begin();
    }

    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    virtual void copy(const JDAQUTCTimeRange& range)
    {
      for ( ; i != in.end() && i->getTimesliceStart() <  range.getLowerLimit(); ++i) {}
      for ( ; i != in.end() && i->getTimesliceStart() <= range.getUpperLimit(); ++i) { out.put(*i); }
    }
    

  private:
    JTreeScanner<T, JDAQEvaluator> in;
    JObjectOutput<T>&              out;
    typename JTreeScanner<T, JDAQEvaluator>::iterator i;
  };


  /**
   * Copy master.
   */
  template<class JTypelist_t>
  struct JCopyMaster : 
    public std::vector< JSinglePointer<JCopyInterface> >,
    public JFileRecorder<JTYPELIST<JTriggerTypes_t, JMeta>::typelist>
  {
    /**
     * Constructor.
     *
     * \param  input_file              input file
     * \param  selection               selection
     */
    JCopyMaster(const std::string& input_file, const JROOTClassSelection& selection) :
      selection(selection)
    {
      add(JType<JTypelist_t>(), input_file);
    }

    /**
     * Copy data within given UTC time range.
     *
     * \param  range                   UTC time range
     */
    void copy(const JDAQUTCTimeRange& range)
    {
      for (iterator i = this->begin(); i != this->end(); ++i) {
        (*i)->copy(range);
      }
    }

    static int debug;
    const JROOTClassSelection selection;

  private:
    /**
     * Recursive method for adding data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    template<class JHead_t, class JTail_t>
    void add(JType< JTypeList<JHead_t, JTail_t> > type, const std::string& input_file)
    {
      add(JType<JHead_t>(), input_file);
      add(JType<JTail_t>(), input_file);
    }

    /**
     * Add data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    template<class T>
    void add(JType<T> type, const std::string& input_file)
    {
      if (selection(type)) {

        JTreeScanner<T, JDAQEvaluator>::debug = debug;

        push_back(new JCopy<T>(input_file, *this));
      }
    }

    /**
     * Termination method for adding data copier.
     *
     * \param  type                    data type
     * \param  input_file              file name
     */
    void add(JType<JNullType> type, const std::string& input_file)
    {}
  };

  /**
   * Debug.
   */
  template<class JTypelist_t>
  int JCopyMaster<JTypelist_t>::debug = 0;


  /**
   * Auxiliary data structure for file splitting.
   */
  struct JSplit {

    static const char SEPARATOR = '/';   //!< separator between optional index and total

    /**
     * Default constructor.
     */
    JSplit() :
       total(0),
       index(invalid)
    {}

    /**
     * Read split from input stream.
     *
     * \param  in         input stream
     * \param  object     split
     * \return            input stream
     */
    friend inline std::istream& operator>>(std::istream& in, JSplit& object)
    {
      using namespace std;

      string buffer;

      if (in >> buffer) {

        const string::size_type pos = buffer.find(SEPARATOR);

        if (pos != string::npos) {

          if (!(istringstream(buffer.substr(0,pos)) >> object.index)) { in.setstate(ios::badbit); }
          if (!(istringstream(buffer.substr(pos+1)) >> object.total)) { in.setstate(ios::badbit); }

        } else {

          object.index = invalid;

          istringstream(buffer) >> object.total;
        }
      }

      return in;
    }

    /**
     * Write split to output stream.
     *
     * \param  out        output stream
     * \param  object     split
     * \return            output stream
     */
    friend inline std::ostream& operator<<(std::ostream& out, const JSplit& object)
    {
      using namespace std;

      if (object.index != invalid)
        return out << object.index << SEPARATOR << object.total;
      else
        return out << object.total;
    }


    static constexpr size_t invalid = std::numeric_limits<size_t>::max();  //!< invalid value
  
    size_t total;
    size_t index;
  };


  /**
   * Compare UTC time ranges.
   *
   * \param  first                   first  UTC time range
   * \param  second                  second UTC time range
   * \param  output_file             file name
   */
  inline bool compare(const JDAQUTCTimeRange& first, const JDAQUTCTimeRange& second)
  {
    return first.getLowerLimit() < second.getLowerLimit();
  }
}


/**
 * \file
 * Auxiliary program to split DAQ data into multiple output files.
 *
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;

  typedef vector<JDAQUTCTimeRange> ranges_type;

  std::string         inputFile;
  std::string         outputFile;
  JSplit              numberOfFiles;
  ranges_type         ranges;
  JROOTClassSelection selection      = getROOTClassSelection<JDAQTypes_t>();
  int                 debug;

  try {

    JParser<> zap("Auxiliary program to split DAQ data into multiple output files.");

    zap['f'] = make_field(inputFile);
    zap['o'] = make_field(outputFile);
    zap['N'] = make_field(numberOfFiles)       = JPARSER::initialised();
    zap['r'] = make_field(ranges, 
                          "UTC time range(s)") = JPARSER::initialised();
    zap['C'] = make_field(selection,
                          "Precede name of data structure by a '+' or '-' "
                          "to add or remove data types in the output, respectively."
                          "\nROOT wildcards are accepted.") = JPARSER::initialised();
    zap['d'] = make_field(debug)               = 2;

    zap(argc, argv);
  }
  catch(const exception& error) {
    FATAL(error.what() << endl);
  }


  if ((numberOfFiles.total == 0 &&  ranges.empty()) ||
      (numberOfFiles.total != 0 && !ranges.empty()))  {
    FATAL("Invalid splitting " << numberOfFiles << ' ' << ranges.size() << "; use either option -N or -r." << endl);
  }

  const size_t pos = outputFile.find(WILDCARD);

  if (pos == string::npos) {
    FATAL("Output file name " << outputFile << " does not contain wild card '" << WILDCARD << "'" << endl);
  }

  const JDAQUTCTimeRange total  =  combine(getUTCTimeRange<JDAQSummaryslice>(inputFile),
                                           getUTCTimeRange<JDAQEvent>       (inputFile));

  if (!total.is_valid()) {
    FATAL("No (valid) summary or event data in input file " << inputFile << ' ' << total << endl);
  }

  NOTICE("Total UTC time range " << total << endl);

  if (!ranges.empty()) {

    sort(ranges.begin(), ranges.end(), compare);

  } else {

    const long double T_ns = ((long double) JDAQUTCExtended::getTick() + 1.0e9l * (long double) getTimeDifference(total.getLowerLimit(), total.getUpperLimit())) / (long double) numberOfFiles.total;

    for (long double t1 = total.getLowerLimit().getTimeNanoSecond(); t1 < (long double) total.getUpperLimit().getTimeNanoSecond(); t1 += T_ns) {
      ranges.push_back(JDAQUTCTimeRange(JDAQUTCExtended(t1), JDAQUTCExtended(t1 + T_ns)));
    }
  }


  JCopyMaster<JDAQTypes_t>::debug = debug;

  JCopyMaster<JDAQTypes_t> master(inputFile, selection);

  const int width = (int) (log10(ranges.size() + 1) + 1);

  for (size_t i = 0; i != ranges.size(); ++i) {

    if (numberOfFiles.index == numberOfFiles.invalid || numberOfFiles.index == i) {

      const string file_name = MAKE_STRING(outputFile.substr(0,pos) << FILL(width,'0') << i << FILL() << outputFile.substr(pos+1));

      STATUS("Writing " << file_name << ' ' << ranges[i] << "... " << flush);

      master.open(file_name.c_str());

      master.put(JMeta(argc, argv));           
      master.put(getTriggerParameters(inputFile));

      master.copy(ranges[i]);

      master.close();

      STATUS("OK" << endl);
    }
  }
}