Jpp  15.0.1-rc.2-highQE
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JDataQuality.cc
Go to the documentation of this file.
1 #include <iostream>
2 #include <sstream>
3 #include <fstream>
4 #include <iomanip>
5 #include <vector>
6 #include <set>
7 #include <map>
8 
9 #include "TROOT.h"
10 #include "TFile.h"
11 #include "TH1D.h"
12 #include "TF1.h"
13 #include "TString.h"
14 #include "TNtuple.h"
15 #include "TRegexp.h"
16 
17 #include "JDB/JDB.hh"
18 #include "JDB/JSelector.hh"
20 #include "JDB/JRunQuality.hh"
21 #include "JDB/JRunsetups.hh"
22 #include "JDB/JDBToolkit.hh"
23 #include "JDB/JGITTags.hh"
24 
25 #include "JLang/JLangToolkit.hh"
26 #include "JTools/JRange.hh"
27 #include "JROOT/JRootClass.hh"
28 #include "JROOT/JRootPrinter.hh"
29 #include "JROOT/JRootToolkit.hh"
31 #include "JROOT/JRootDictionary.hh"
32 #include "JROOT/JManager.hh"
33 #include "JSupport/JMeta.hh"
35 
36 #include "Jeep/JeepToolkit.hh"
37 #include "Jeep/JComment.hh"
38 #include "Jeep/JParser.hh"
39 #include "Jeep/JMessage.hh"
40 
41 namespace {
42 
43  /**
44  * Auxiliary data structure for run quality evaluation.
45  *
46  * A condition is composed of a ROOT TFormula compatible expression and a validity range for the result.
47  */
48  struct JCondition {
49 
50  static const char SEPARATOR = ';';
51 
52  /**
53  * Read selection from input stream.
54  *
55  * \param in input stream
56  * \param object selection
57  * \return input stream
58  */
59  friend inline std::istream& operator>>(std::istream& in, JCondition& object)
60  {
61  std::getline(in, object.formula, SEPARATOR);
62 
63  object.range = JTOOLS::JRange<double>();
64  object.weight = 1.0;
65 
66  if ((in >> object.range) && ! (in >> object.weight)) {
67  in.clear();
68  }
69 
70  return in;
71  }
72 
73 
74  /**
75  * Write selection to output stream.
76  *
77  * \param out output stream
78  * \param object selection
79  * \return output stream
80  */
81  friend inline std::ostream& operator<<(std::ostream& out, const JCondition& object)
82  {
83  out << object.formula << SEPARATOR << object.range << ' ' << object.weight << std::endl;
84 
85  return out;
86  }
87 
88  std::string formula;
90  double weight;
91  };
92 }
93 
94 
95 /**
96  * \file
97  *
98  * Main program to evaluate quality data from data base.
99  *
100  * The evaluation of the data quality is based on the following procedure.\n
101  * For each data taking run, the script JQAQC.sh is executed which produces a set of values.
102  * The values are subsequently uploaded into the database in table "runsummarynumbers".
103  * This table is a general purpose table for any value related to some data taking run.
104  * With this application,
105  * the QA/QC data are consistently downloaded from the database;
106  * evaluated according some formula and
107  * tested against a validity range.
108  * The formula and validity range are referred to as a condition and can be specified with option -Q.
109  * Each condition has an optional weight (default is 1).
110  * The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best.
111  * There is also the possibility to specify a veto (option -V),
112  * which is similar to a condition but only the number of vetoes is counted.
113  * The overall quality and the number of vetoes can be used to accept or reject a data taking run.
114  *
115  * By default, a list of the run numbers, the overall weight and the number of vetoes is printed.
116  * If a ROOT file is specified for the output (i.e.\ filename extension is JSUPPORT::ROOT_FILE_FORMAT),
117  * a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple.
118  * If an ASCII file is specified for the output (i.e.\ filename extension is JSUPPORT::ASCII_FILE_FORMAT),
119  * a text file is written which can subsequently be used as input file.
120  *
121  * \author mdejong
122  */
123 int main(int argc, char **argv)
124 {
125  using namespace std;
126  using namespace JPP;
127 
128  typedef JRange<int> JRange_t;
129  typedef vector<JCondition> JCondition_t;
130 
131  JServer server;
132  string usr;
133  string pwd;
134  string cookie;
135  string inputFile;
136  string outputFile;
137  string detid;
138  JRange_t runs;
139  vector<string> source;
140  JCondition_t condition;
141  JCondition_t veto;
142  TRegexp regexp(".");
143  JSelector selection;
144  int debug;
145 
146  try {
147 
148  JParser<> zap("Main program to evaluate quality data from data base."\
149  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
150 
151  zap['s'] = make_field(server) = getServernames();
152  zap['u'] = make_field(usr) = "";
153  zap['!'] = make_field(pwd) = "";
154  zap['C'] = make_field(cookie) = "";
155  zap['f'] = make_field(inputFile, "Optional input file instead of database.") = "";
156  zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "quality.root";
157  zap['D'] = make_field(detid) = "";
158  zap['R'] = make_field(runs, "Run range") = JRange_t(1, JRange_t::getMaximum());
159  zap['S'] = make_field(source, "GIT versions") = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
160  zap['Q'] = make_field(condition, "User defined conditions") = JPARSER::initialised();
161  zap['V'] = make_field(veto, "User defined vetos.") = JPARSER::initialised();
162  zap['r'] = make_field(regexp, "TRegexp for selection of run setup names.") = JPARSER::initialised();
163  zap['@'] = make_field(selection) = JPARSER::initialised();
164  zap['d'] = make_field(debug, "Debug level") = 1;
165 
166  zap(argc, argv);
167  }
168  catch(const exception &error) {
169  FATAL(error.what() << endl);
170  }
171 
172 
173  double W = 0.0;
174 
175  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
176  W += i->weight;
177  }
178 
180  if (W <= 0.0) {
181  FATAL("Invalid total weight: " << W << endl);
182  }
183  }
184 
185 
186  JRunsetups setups; // run -> value
187  set<JRunQuality> buffer; // storage
188 
189 
190  if (inputFile == "") { // read data from database
191 
192  ASSERT(detid != "");
193 
194  int ID = -1; // integer representation of detector identifier
195 
196  try {
197 
198  JDB::reset(usr, pwd, cookie);
199 
200  if (is_integer(detid))
201  ID = to_value<int>(detid);
202  else
203  ID = getDetector(detid);
204 
205  if (is_integer(detid)) {
206  detid = getDetector(to_value<int>(detid));
207  }
208 
209  // run setups
210 
211  selection += getSelector<JRuns>(ID);
212 
213  NOTICE("Extracting run information from database... " << flush);
214 
215  ResultSet& rs = getResultSet(getTable<JRuns>(), selection);
216 
217  for (JRuns parameters; rs >> parameters; ) {
218  if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
219  setups.put(parameters);
220  }
221  }
222 
223  rs.Close();
224 
225  NOTICE("OK" << endl);
226 
227  // run summary data
228 
229  for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {
230 
231  typedef map<string, string> data_type;
232  typedef map<int, data_type> map_type;
233 
234  map_type zmap;
235 
236  JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
237 
238  selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);
239 
240  try {
241 
242  NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);
243 
244  ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
245 
246  for (JRunSummaryNumbers parameters; rs >> parameters; ) {
247  if (setups.has(parameters.RUN)) {
248  zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
249  }
250  }
251 
252  rs.Close();
253 
254  NOTICE("OK" << endl);
255  }
256  catch(const exception& error) { NOTICE(endl); }
257 
258  for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
259 
260  JRunQuality quality;
261 
262  quality.GIT = *git;
263  quality.detector = ID;
264  quality.run = run->first;
265 
266  for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
267  quality.put(i->first, i->second);
268  }
269 
270  buffer.insert(quality); // only insert if absent
271  }
272  }
273  }
274  catch(const exception& error) {
275  FATAL(error.what() << endl);
276  }
277 
278  } else { // read data from file
279 
280  ifstream in(inputFile.c_str());
281 
282  JComment comment;
283 
284  in >> comment;
285 
286  int nc = 0; // number of columns in file
287  int nd = getListOfDataMembers<JRunQuality>().size();
288 
289  string header;
290 
291  if (getline(in, header)) {
292 
293  istringstream is(header);
294 
295  nc = distance(istream_iterator<string>(is), istream_iterator<string>());
296  }
297 
298  JRootReader reader(in, JEquationParameters(), JDBDictionary::getInstance());
299 
300  for (JRunQuality quality; reader.getObject(quality); ) {
301 
302  buffer.insert(quality);
303 
304  if (nc > nd) { // read optional data
305 
306  double setup;
307 
308  reader.getObject(setup);
309 
310  setups[quality.run] = setup;
311  }
312  }
313 
314  in.close();
315  }
316 
317 
318  if (buffer.empty()) {
319  FATAL("No data." << endl);
320  }
321 
322 
324 
325  runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);
326 
327  TH1D h0("h0", NULL, 1000, 0.0, 1.01);
328  TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() + 0.5);
329 
330  h0.SetMinimum(0.0);
331  h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
332 
333  h1.SetMinimum(0.0);
334  h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
335 
336  const Double_t xmin = runs.getLowerLimit() - 0.5;
337  const Double_t xmax = runs.getUpperLimit() + 0.5;
338 
339  JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));
340 
341  // show validity range
342 
343  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
344  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
345  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
346  }
347 
348  for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
349  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
350  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
351  }
352 
353  ostringstream os; // N-tuple format
354 
355  os << "run";
356 
357  for (size_t i = 0; i != condition.size(); ++i) {
358  os << ":" << (char) ('a' + i);
359  }
360 
361  os << ":Q:V:R";
362 
363  TNtuple n1("n1", "quality", os.str().c_str());
364 
365 
366  // process data
367 
368  for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
369 
370  vector<Float_t> tuple(1, (Float_t) quality->run);
371 
372  double w = 0.0;
373 
374  for (size_t i = 0; i != condition.size(); ++i) {
375 
376  const JCondition& ps = condition[i];
377  const double y = getResult(ps.formula, *quality);
378 
379  DEBUG(ps.formula << ' ' << y << endl);
380 
381  tuple.push_back((Float_t) y);
382 
383  TH1D* p = H1[ps.formula];
384 
385  p->SetBinContent(p->FindBin((double) quality->run), y);
386  p->SetBinError (p->FindBin((double) quality->run), 0.0);
387 
388  if (ps.range(y)) {
389  w += ps.weight;
390  }
391 
392  h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
393  }
394 
395  const double Q = w/W;
396 
397  tuple.push_back((Float_t) Q);
398 
399  int V = 0;
400 
401  for (size_t i = 0; i != veto.size(); ++i) {
402 
403  const JCondition& ps = veto[i];
404  const double y = getResult(ps.formula, *quality);
405 
406  TH1D* p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];
407 
408  p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
409  p->SetBinError (p->FindBin((double) quality->run), 0.0);
410 
411  if (!ps.range(y)) {
412  ++V;
413  }
414  }
415 
416  tuple.push_back((Float_t) V);
417  tuple.push_back((Float_t) setups.get(quality->run));
418 
419  h0.Fill(Q);
420  n1.Fill(tuple.data());
421 
422  cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
423  }
424 
425  double w = 0.0;
426 
427  for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
428  h0.SetBinContent(i, (w += h0.GetBinContent(i)));
429  }
430 
431  // store results
432 
433  TFile out(outputFile.c_str(), "recreate");
434 
435  out << h0 << h1 << n1 << H1;
436 
437  out.Write();
438  out.Close();
439  }
440 
441 
443 
444  // store data
445 
446  ofstream out(outputFile);
447 
448  JComment comment;
449 
450  comment.add(JMeta(argc, argv));
451 
452  out << comment;
453 
454  // header
455 
456  for (const auto& i : getListOfDataMembers<JRunQuality>()) {
457  out << i->GetName() << ' ';
458  }
459 
460  out << "setup" << endl;
461 
462  JRootWriter writer(out, JEquationParameters(), JDBDictionary::getInstance());
463 
464  writer.setf(ios::fixed);
465 
466  for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
467 
468  writer.putObject(*i);
469 
470  out << ' ';
471 
472  writer.putObject(setups.get(i->run));
473 
474  out << endl;
475  }
476 
477  out.close();
478  }
479 }
Utility class to parse command line options.
Definition: JParser.hh:1500
data_type w[N+1][M+1]
Definition: JPolint.hh:741
Q(UTCMax_s-UTCMin_s)-livetime_s
int main(int argc, char *argv[])
Definition: Main.cc:15
std::vector< T >::difference_type distance(typename std::vector< T >::const_iterator first, typename PhysicsEvent::const_iterator< T > second)
Specialisation of STL distance.
do rm f tmp H1
Print objects in ASCII format using ROOT dictionary.
*fatal Wrong number of arguments esac JCookie sh typeset Z DETECTOR typeset Z SOURCE_RUN typeset Z TARGET_RUN set_variable PARAMETERS_FILE $WORKDIR parameters
Definition: diff-Tuna.sh:38
#define MAKE_CSTRING(A)
Make C-string.
Definition: JPrint.hh:151
then for HISTOGRAM in h0 h1
Definition: JMatrixNZ.sh:71
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition: JParser.hh:66
then let ID
Definition: JAcoustics.sh:31
Dynamic ROOT object management.
Auxiliary data structure for floating point format specification.
Definition: JManip.hh:446
static const char *const ASCII_FILE_FORMAT
ASCII file name extension.
string outputFile
is
Definition: JDAQCHSM.chsm:167
#define MAKE_STRING(A)
Make string.
Definition: JPrint.hh:142
std::vector< std::string > getGITTags(const TRegexp &regexp, const JGITTags_t::key_type &date)
Get selection of GIT tags.
Definition: JDB/JGITTags.hh:34
JDetectorsHelper getDetector
Function object for mapping serial number to object identifier of detector and vice versa...
Definition: JDBToolkit.cc:5
#define ASSERT(A,...)
Assert macro.
Definition: JMessage.hh:90
T & getInstance(const T &object)
Get static instance from temporary object.
Definition: JObject.hh:75
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1961
Double_t getResult(const TString &text, TObject *object=NULL)
Get result of given textual formula.
Auxiliary methods for handling file names, type names and environment.
ROOT I/O of application specific meta data.
#define NOTICE(A)
Definition: JMessage.hh:64
std::istream & getline(std::istream &in, JString &object)
Read string from input stream until end of line.
Definition: JString.hh:478
bool is_integer(const std::string &buffer)
Check if string is an integer.
Definition: JLangToolkit.hh:58
int debug
debug level
Definition: JSirene.cc:63
General purpose messaging.
#define FATAL(A)
Definition: JMessage.hh:67
z range($ZMAX-$ZMIN)< $MINIMAL_DZ." fi fi typeset -Z 4 STRING typeset -Z 2 FLOOR JPlot1D -f $
void reset(T &value)
Reset value.
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition: JHead.hh:1618
Auxiliary class to define a range between two values.
Utility class to parse command line options.
std::string getFilenameExtension(const std::string &file_name)
Get file name extension, i.e. part after last JEEP::FILENAME_SEPARATOR if any.
Definition: JeepToolkit.hh:69
then usage $script< input_file >< detector_file > fi set_variable OUTPUT_DIR set_variable SELECTOR JDAQTimesliceL1 set_variable DEBUG case set_variable DEBUG
ResultSet & getResultSet(const std::string &query)
Get result set.
Definition: JDB.hh:432
std::vector< JServer > getServernames()
Get list of names of available database servers.
Definition: JDB.hh:98
std::ostream & operator<<(std::ostream &stream, const CLBCommonHeader &header)
Selection of GIT tags.
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY source JAcoustics sh $DETECTOR_ID CHECK_EXIT_CODE typeset A TRIPODS get_tripods $WORKDIR tripod txt TRIPODS for EMITTER in
Definition: JCanberra.sh:41
static const char *const ROOT_FILE_FORMAT
ROOT file name extension.
Specifications of file name extensions.
std::vector< double > weight
Definition: JAlgorithm.hh:417