Jpp  17.3.0-rc.2
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JDAQSplit.cc
Go to the documentation of this file.
1 
2 #include <string>
3 #include <iostream>
4 #include <iomanip>
5 #include <algorithm>
6 
8 #include "JDAQ/JDAQEventIO.hh"
10 #include "JDAQ/JDAQEvaluator.hh"
11 
13 
15 #include "JSupport/JTreeScanner.hh"
18 #include "JSupport/JMeta.hh"
19 #include "JSupport/JSupport.hh"
22 
23 #include "JLang/JSinglePointer.hh"
24 
25 #include "Jeep/JParser.hh"
26 #include "Jeep/JMessage.hh"
27 
28 
29 namespace {
30 
31  static const char WILDCARD = '%'; //!< wild card for output files
32 
33  using namespace JPP;
34  using namespace KM3NETDAQ;
35 
36 
37  /**
38  * Interface for copying part of data.
39  */
40  struct JCopyInterface {
41  /**
42  * Virtual destructor.
43  */
44  virtual ~JCopyInterface()
45  {}
46 
47  /**
48  * Copy data within given UTC time range.
49  *
50  * \param range UTC time range
51  */
52  virtual void copy(const JDAQUTCTimeRange& range) = 0;
53  };
54 
55 
56  /**
57  * Implementation for copying part of data.
58  */
59  template<class T>
60  struct JCopy :
61  public JCopyInterface
62  {
63  /**
64  * Copy data from input file to output.
65  *
66  * \param file_name file name
67  * \param out output
68  */
69  JCopy(const std::string& file_name, JObjectOutput<T>& out) :
70  in (file_name),
71  out(out)
72  {
73  i = in.begin();
74  }
75 
76  /**
77  * Copy data within given UTC time range.
78  *
79  * \param range UTC time range
80  */
81  virtual void copy(const JDAQUTCTimeRange& range)
82  {
83  for ( ; i != in.end() && i->getTimesliceStart() < range.getLowerLimit(); ++i) {}
84  for ( ; i != in.end() && i->getTimesliceStart() <= range.getUpperLimit(); ++i) { out.put(*i); }
85  }
86 
87 
88  private:
90  JObjectOutput<T>& out;
92  };
93 
94 
95  /**
96  * Copy master.
97  */
98  template<class JTypelist_t>
99  struct JCopyMaster :
100  public std::vector< JSinglePointer<JCopyInterface> >,
101  public JFileRecorder<JTYPELIST<JTriggerTypes_t, JMeta>::typelist>
102  {
103  /**
104  * Constructor.
105  *
106  * \param input_file input file
107  * \param selection selection
108  */
109  JCopyMaster(const std::string& input_file, const JROOTClassSelection& selection) :
110  selection(selection)
111  {
112  add(JType<JTypelist_t>(), input_file);
113  }
114 
115  /**
116  * Copy data within given UTC time range.
117  *
118  * \param range UTC time range
119  */
120  void copy(const JDAQUTCTimeRange& range)
121  {
122  for (iterator i = this->begin(); i != this->end(); ++i) {
123  (*i)->copy(range);
124  }
125  }
126 
127  static int debug;
128  const JROOTClassSelection selection;
129 
130  private:
131  /**
132  * Recursive method for adding data copier.
133  *
134  * \param type data type
135  * \param input_file file name
136  */
137  template<class JHead_t, class JTail_t>
138  void add(JType< JTypeList<JHead_t, JTail_t> > type, const std::string& input_file)
139  {
140  add(JType<JHead_t>(), input_file);
141  add(JType<JTail_t>(), input_file);
142  }
143 
144  /**
145  * Add data copier.
146  *
147  * \param type data type
148  * \param input_file file name
149  */
150  template<class T>
151  void add(JType<T> type, const std::string& input_file)
152  {
153  if (selection(type)) {
154 
156 
157  push_back(new JCopy<T>(input_file, *this));
158  }
159  }
160 
161  /**
162  * Termination method for adding data copier.
163  *
164  * \param type data type
165  * \param input_file file name
166  */
167  void add(JType<JNullType> type, const std::string& input_file)
168  {}
169  };
170 
171  /**
172  * Debug.
173  */
174  template<class JTypelist_t>
176 
177 
178  /**
179  * Auxiliary data structure for file splitting.
180  */
181  struct JSplit {
182 
183  static const char SEPARATOR = '/'; //!< separator between optional index and total
184 
185  /**
186  * Default constructor.
187  */
188  JSplit() :
189  total(invalid),
190  index(invalid)
191  {}
192 
193  /**
194  * Read split from input stream.
195  *
196  * \param in input stream
197  * \param object split
198  * \return input stream
199  */
200  friend inline std::istream& operator>>(std::istream& in, JSplit& object)
201  {
202  using namespace std;
203 
204  string buffer;
205 
206  if (in >> buffer) {
207 
208  const string::size_type pos = buffer.find(SEPARATOR);
209 
210  if (pos != string::npos) {
211 
212  if (!(istringstream(buffer.substr(0,pos)) >> object.index)) { in.setstate(ios::badbit); }
213  if (!(istringstream(buffer.substr(pos+1)) >> object.total)) { in.setstate(ios::badbit); }
214 
215  } else {
216 
217  object.index = invalid;
218 
219  istringstream(buffer) >> object.total;
220  }
221  }
222 
223  return in;
224  }
225 
226  /**
227  * Write split to output stream.
228  *
229  * \param out output stream
230  * \param object split
231  * \return output stream
232  */
233  friend inline std::ostream& operator<<(std::ostream& out, const JSplit& object)
234  {
235  using namespace std;
236 
237  if (object.index != invalid)
238  return out << object.index << SEPARATOR << object.total;
239  else
240  return out << object.total;
241  }
242 
243 
244  static constexpr size_t invalid = std::numeric_limits<size_t>::max(); //!< invalid value
245 
246  size_t total;
247  size_t index;
248  };
249 
250 
251  /**
252  * Compare UTC time ranges.
253  *
254  * \param first first UTC time range
255  * \param second second UTC time range
256  * \param output_file file name
257  */
258  inline bool compare(const JDAQUTCTimeRange& first, const JDAQUTCTimeRange& second)
259  {
260  return first.getLowerLimit() < second.getLowerLimit();
261  }
262 }
263 
264 
265 /**
266  * \file
267  * Auxiliary program to split DAQ data into multiple output files.
268  *
269  * \author mdejong
270  */
271 int main(int argc, char **argv)
272 {
273  using namespace std;
274  using namespace JPP;
275 
276  typedef vector<JDAQUTCTimeRange> ranges_type;
277 
278  std::string inputFile;
280  JSplit numberOfFiles;
281  ranges_type ranges;
282  JROOTClassSelection selection = getROOTClassSelection<JDAQTypes_t>();
283  int debug;
284 
285  try {
286 
287  JParser<> zap("Auxiliary program to split DAQ data into multiple output files.");
288 
289  zap['f'] = make_field(inputFile);
290  zap['o'] = make_field(outputFile);
291  zap['N'] = make_field(numberOfFiles) = JPARSER::initialised();
292  zap['r'] = make_field(ranges,
293  "UTC time range(s)") = JPARSER::initialised();
294  zap['C'] = make_field(selection,
295  "Precede name of data structure by a '+' or '-' "
296  "to add or remove data types in the output, respectively."
297  "\nROOT wildcards are accepted.") = JPARSER::initialised();
298  zap['d'] = make_field(debug) = 2;
299 
300  zap(argc, argv);
301  }
302  catch(const exception& error) {
303  FATAL(error.what() << endl);
304  }
305 
306 
307  if ((numberOfFiles.total == 0 && ranges.empty()) ||
308  (numberOfFiles.total != 0 && !ranges.empty())) {
309  FATAL("Invalid splitting " << numberOfFiles << ' ' << ranges.size() << "; use either option -N or -r." << endl);
310  }
311 
312  const size_t pos = outputFile.find(WILDCARD);
313 
314  if (pos == string::npos) {
315  FATAL("Output file name " << outputFile << " does not contain wild card '" << WILDCARD << "'" << endl);
316  }
317 
318  const JDAQUTCTimeRange total = combine(getUTCTimeRange<JDAQSummaryslice>(inputFile),
319  getUTCTimeRange<JDAQEvent> (inputFile));
320 
321  if (!total.is_valid()) {
322  FATAL("No (valid) summary or event data in input file " << inputFile << ' ' << total << endl);
323  }
324 
325  NOTICE("Total UTC time range " << total << endl);
326 
327  if (!ranges.empty()) {
328 
329  sort(ranges.begin(), ranges.end(), compare);
330 
331  } else {
332 
333  const long double T_ns = ((long double) JDAQUTCExtended::getTick() + 1.0e9l * (long double) getTimeDifference(total.getLowerLimit(), total.getUpperLimit())) / (long double) numberOfFiles.total;
334 
335  for (long double t1 = total.getLowerLimit().getTimeNanoSecond(); t1 < (long double) total.getUpperLimit().getTimeNanoSecond(); t1 += T_ns) {
336  ranges.push_back(JDAQUTCTimeRange(JDAQUTCExtended(t1), JDAQUTCExtended(t1 + T_ns)));
337  }
338  }
339 
340 
342 
343  JCopyMaster<JDAQTypes_t> master(inputFile, selection);
344 
345  const int width = (int) (log10(ranges.size() + 1) + 1);
346 
347  for (size_t i = 0; i != ranges.size(); ++i) {
348 
349  if (numberOfFiles.index == numberOfFiles.invalid || numberOfFiles.index == i) {
350 
351  const string file_name = MAKE_STRING(outputFile.substr(0,pos) << FILL(width,'0') << i << FILL() << outputFile.substr(pos+1));
352 
353  STATUS("Writing " << file_name << ' ' << ranges[i] << "... " << flush);
354 
355  master.open(file_name.c_str());
356 
357  master.put(JMeta(argc, argv));
358  master.put(getTriggerParameters(inputFile));
359 
360  master.copy(ranges[i]);
361 
362  master.close();
363 
364  STATUS("OK" << endl);
365  }
366  }
367 }
Auxiliary class for ROOT I/O of application specific meta data.
Definition: JMeta.hh:70
Object writing to file.
Utility class to parse command line options.
Definition: JParser.hh:1517
then echo Test string reversed by master(hit< return > to continue)." $DIR/JProcess -c "$DIR/JEcho" -rC fi if (( 1 ))
T getLowerLimit() const
Get lower limit.
Definition: JRange.hh:202
int main(int argc, char *argv[])
Definition: Main.cc:15
ROOT TTree parameter settings of various packages.
#define STATUS(A)
Definition: JMessage.hh:63
Recording of objects on file according a format that follows from the file name extension.
Auxiliary class for ROOT class selection.
static double getTick()
Get number of nano-seconds per tick.
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition: JParser.hh:83
Auxiliary class for a type holder.
Definition: JType.hh:19
string outputFile
Data structure for UTC time.
Template definition for direct access of elements in ROOT TChain.
then echo The file $DIR KM3NeT_00000001_00000000 root already please rename or remove it first
#define MAKE_STRING(A)
Make string.
Definition: JPrint.hh:127
Type list.
Definition: JTypeList.hh:22
Scanning of objects from a single file according a format that follows from the extension of each fil...
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1993
set_variable E_E log10(E_{fit}/E_{#mu})"
double getTimeDifference(const JDAQChronometer &first, const JDAQChronometer &second)
Get time difference between two chronometers.
ROOT I/O of application specific meta data.
T getUpperLimit() const
Get upper limit.
Definition: JRange.hh:213
#define NOTICE(A)
Definition: JMessage.hh:64
then awk string
Support methods.
static struct JACOUSTICS::@4 compare
Auxiliary data structure to sort transmissions.
Range of values.
Definition: JRange.hh:38
General purpose messaging.
JTOOLS::JRange< JDAQUTCExtended > JDAQUTCTimeRange
Type definition for DAQ UTC time range.
Auxiliary data structure for sequence of same character.
Definition: JManip.hh:328
JRange< T, JComparator_t > combine(const JRange< T, JComparator_t > &first, const JRange< T, JComparator_t > &second)
Combine ranges.
Definition: JRange.hh:685
#define FATAL(A)
Definition: JMessage.hh:67
z range($ZMAX-$ZMIN)< $MINIMAL_DZ." fi fi typeset -Z 4 STRING typeset -Z 2 FLOOR JPlot1D -f $
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition: JHead.hh:1756
Utility class to parse command line options.
void copy(const Head &from, JHead &to)
Copy header from from to to.
Definition: JHead.cc:162
std::ostream & operator<<(std::ostream &stream, const CLBCommonHeader &header)
Template interface of object output for single data type.
do echo n Creating graphics for string $STRING for((FLOOR=$FIRST_FLOOR;$FLOOR<=$LAST_FLOOR;FLOOR+=1))
virtual bool put(const T &object)=0
Object output.
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY JAcoustics sh $DETECTOR_ID source JAcousticsToolkit sh CHECK_EXIT_CODE typeset A EMITTERS get_tripods $WORKDIR tripod txt EMITTERS get_transmitters $WORKDIR transmitter txt EMITTERS for EMITTER in
Definition: JCanberra.sh:46
JTriggerParameters getTriggerParameters(const JMultipleFileScanner_t &file_list)
Get trigger parameters.
int debug
debug level