Jpp - the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JDataQuality.cc
Go to the documentation of this file.
1 #include <iostream>
2 #include <sstream>
3 #include <fstream>
4 #include <iomanip>
5 #include <vector>
6 #include <set>
7 #include <map>
8 
9 #include "TROOT.h"
10 #include "TFile.h"
11 #include "TH1D.h"
12 #include "TF1.h"
13 #include "TString.h"
14 #include "TNtuple.h"
15 #include "TRegexp.h"
16 
17 #include "JDB/JDB.hh"
18 #include "JDB/JSelector.hh"
20 #include "JDB/JRunQuality.hh"
21 #include "JDB/JRunsetups.hh"
22 #include "JDB/JDBToolkit.hh"
23 #include "JDB/JGITTags.hh"
24 
25 #include "JLang/JLangToolkit.hh"
26 #include "JTools/JRange.hh"
27 #include "JROOT/JRootPrinter.hh"
28 #include "JROOT/JRootToolkit.hh"
30 #include "JROOT/JRootDictionary.hh"
31 #include "JROOT/JManager.hh"
32 
33 #include "Jeep/JeepToolkit.hh"
34 #include "Jeep/JParser.hh"
35 #include "Jeep/JMessage.hh"
36 
37 namespace {
38 
39  /**
40  * Auxiliary data structure for run quality evaluation.
41  *
42  * A condition is composed of a ROOT TFormula compatible expression and a validity range for the result.
43  */
44  struct JCondition {
45 
46  static const char SEPARATOR = ';';
47 
48  /**
49  * Read selection from input stream.
50  *
51  * \param in input stream
52  * \param object selection
53  * \return input stream
54  */
55  friend inline std::istream& operator>>(std::istream& in, JCondition& object)
56  {
57  std::getline(in, object.formula, SEPARATOR);
58 
59  object.range = JTOOLS::JRange<double>();
60  object.weight = 1.0;
61 
62  if ((in >> object.range) && ! (in >> object.weight)) {
63  in.clear();
64  }
65 
66  return in;
67  }
68 
69 
70  /**
71  * Write selection to output stream.
72  *
73  * \param out output stream
74  * \param object selection
75  * \return output stream
76  */
77  friend inline std::ostream& operator<<(std::ostream& out, const JCondition& object)
78  {
79  out << object.formula << SEPARATOR << object.range << ' ' << object.weight << std::endl;
80 
81  return out;
82  }
83 
84  std::string formula;
86  double weight;
87  };
88 }
89 
90 
91 /**
92  * \file
93  *
94  * Main program to evaluate quality data from data base.
95  *
96  * The evaluation of the data quality is based on the following procedure.\n
97  * For each data taking run, the script JQAQC.sh is executed which produces a set of values.
98  * The values are subsequently uploaded into the database in table "runsummarynumbers".
99  * This table is a general purpose table for any value related to some data taking run.
100  * With this application,
101  * the QA/QC data are consistently downloaded from the database;
102  * evaluated according some formula and
103  * tested against a validity range.
104  * The formula and validity range are referred to as a condition and can be specified with option -Q.
105  * Each condition has an optional weight (defaut is 1).
106  * The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best.
107  * There is also the possibility to specify a veto (option -V),
108  * which is similar to a condition but only the number of vetoes is counted.
109  * The overall quality and the number of vetoes can be used to accept or reject a data taking run.
110  *
111  * By default, a list of the run numbers, the overall weight and the number of vetoes is printed.
112  * If a ROOT file is specified for the output (i.e.\ filename extension is "root"),
113  * a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple.
114  * If an ASCII file is specified for the output (i.e.\ filename extension is "txt"),
115  * a text file is written which can subsequently be used as input file.
116  *
117  * \author mdejong
118  */
119 int main(int argc, char **argv)
120 {
121  using namespace std;
122  using namespace JPP;
123 
124  typedef JRange<int> JRange_t;
125  typedef vector<JCondition> JCondition_t;
126 
127  JServer server;
128  string usr;
129  string pwd;
130  string cookie;
131  string inputFile;
132  string outputFile;
133  string detid;
134  JRange_t runs;
135  vector<string> source;
136  JCondition_t condition;
137  JCondition_t veto;
138  TRegexp regexp(".");
139  JSelector selection;
140  int debug;
141 
142  try {
143 
144  JParser<> zap("Main program to evaluate quality data from data base."\
145  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
146 
147  zap['s'] = make_field(server) = getServernames();
148  zap['u'] = make_field(usr) = "";
149  zap['!'] = make_field(pwd) = "";
150  zap['C'] = make_field(cookie) = "";
151  zap['f'] = make_field(inputFile, "Optional input file instead of database.") = "";
152  zap['o'] = make_field(outputFile, "Output file containing histograms and n-tuple or ASCII file.") = "quality.root";
153  zap['D'] = make_field(detid) = "";
154  zap['R'] = make_field(runs, "Run range") = JRange_t(1, JRange_t::getMaximum());
155  zap['S'] = make_field(source, "GIT versions") = getGITTags(TRegexp("v.*\\..*\\..*"), JGITTags_t::key_type("2019-04-12"));
156  zap['Q'] = make_field(condition, "User defined conditions") = JPARSER::initialised();
157  zap['V'] = make_field(veto, "User defined vetos.") = JPARSER::initialised();
158  zap['r'] = make_field(regexp, "TRegexp for selection of run setup names.") = JPARSER::initialised();
159  zap['@'] = make_field(selection) = JPARSER::initialised();
160  zap['d'] = make_field(debug, "Debug level") = 1;
161 
162  zap(argc, argv);
163  }
164  catch(const exception &error) {
165  FATAL(error.what() << endl);
166  }
167 
168 
169  double W = 0.0;
170 
171  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
172  W += i->weight;
173  }
174 
175  if (getFilenameExtension(outputFile) == "root") {
176  if (W <= 0.0) {
177  FATAL("Invalid total weight: " << W << endl);
178  }
179  }
180 
181 
182  JRunsetups setups; // run -> value
183  set<JRunQuality> buffer; // storage
184 
185 
186  if (inputFile == "") { // read data from database
187 
188  ASSERT(detid != "");
189 
190  int ID = -1; // integer representation of detector identifier
191 
192  try {
193 
194  JDB::reset(usr, pwd, cookie);
195 
196  if (is_integer(detid))
197  ID = to_value<int>(detid);
198  else
199  ID = getDetector(detid);
200 
201  if (is_integer(detid)) {
202  detid = getDetector(to_value<int>(detid));
203  }
204 
205  // run setups
206 
207  selection += getSelector<JRuns>(ID);
208 
209  ResultSet& rs = getResultSet(getTable<JRuns>(), selection);
210 
211  for (JRuns parameters; rs >> parameters; ) {
212  if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
213  setups.put(parameters);
214  }
215  }
216 
217  rs.Close();
218 
219  // run summary data
220 
221  for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {
222 
223  typedef map<string, string> data_type;
224  typedef map<int, data_type> map_type;
225 
226  map_type zmap;
227 
228  JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
229 
230  selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);
231 
232  try {
233 
234  ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
235 
236  for (JRunSummaryNumbers parameters; rs >> parameters; ) {
237  if (setups.has(parameters.RUN)) {
238  zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
239  }
240  }
241 
242  rs.Close();
243  }
244  catch(const exception& error) {}
245 
246  for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
247 
248  JRunQuality quality;
249 
250  quality.GIT = *git;
251  quality.detector = ID;
252  quality.run = run->first;
253 
254  for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
255  quality.put(i->first, i->second);
256  }
257 
258  buffer.insert(quality); // only insert if absent
259  }
260  }
261  }
262  catch(const exception& error) {
263  FATAL(error.what() << endl);
264  }
265 
266  } else { // read data from file
267 
268  int nd = 0; // number of data members in JRunQuality
269  int nc = 0; // number of columns in file
270 
271  {
272  TClass* rc = (TClass*) TDictionary::GetDictionary(typeid(JRunQuality));
273 
274  TIterator* i = rc->GetListOfDataMembers()->MakeIterator();
275 
276  for (const TDataMember* p; (p = (const TDataMember*) i->Next()) != NULL; ) {
277  if (JRootClass::is_class(p->GetName())) {
278  ++nd;
279  }
280  }
281  }
282 
283  ifstream in(inputFile.c_str());
284 
285  {
286  string zbuf;
287 
288  if (getline(in, zbuf)) {
289 
290  istringstream is(zbuf);
291 
292  nc = distance(istream_iterator<string>(is), istream_iterator<string>());
293 
294  in.seekg(0); // rewind file
295  }
296  }
297 
298  JRootReader reader(in, JEquationParameters(), JDBDictionary::getInstance());
299 
300  for (JRunQuality quality; reader.getObject(quality); ) {
301 
302  buffer.insert(quality);
303 
304  if (nc > nd) { // read optional data
305 
306  double setup;
307 
308  reader.getObject(setup);
309 
310  setups[quality.run] = setup;
311  }
312  }
313 
314  in.close();
315  }
316 
317 
318  if (buffer.empty()) {
319  FATAL("No data." << endl);
320  }
321 
322 
323  if (getFilenameExtension(outputFile) == "root") {
324 
325  runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);
326 
327  TH1D h0("h0", NULL, 1000, 0.0, 1.01);
328  TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() + 0.5);
329 
330  h0.SetMinimum(0.0);
331  h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
332 
333  h1.SetMinimum(0.0);
334  h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
335 
336  const Double_t xmin = runs.getLowerLimit() - 0.5;
337  const Double_t xmax = runs.getUpperLimit() + 0.5;
338 
339  JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));
340 
341  // show validity range
342 
343  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
344  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
345  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
346  }
347 
348  for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
349  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
350  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
351  }
352 
353  ostringstream os; // N-tuple format
354 
355  os << "run";
356 
357  for (size_t i = 0; i != condition.size(); ++i) {
358  os << ":" << (char) ('a' + i);
359  }
360 
361  os << ":Q:V:R";
362 
363  TNtuple n1("n1", "quality", os.str().c_str());
364 
365 
366  resetObject(&h0);
367  resetObject(&h1);
368  resetObject(&n1);
369  resetObject(&H1);
370 
371 
372  // process data
373 
374  for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
375 
376  vector<Float_t> tuple(1, (Float_t) quality->run);
377 
378  double w = 0.0;
379 
380  for (size_t i = 0; i != condition.size(); ++i) {
381 
382  const JCondition& ps = condition[i];
383  const double y = getResult(ps.formula, *quality);
384 
385  DEBUG(ps.formula << ' ' << y << endl);
386 
387  tuple.push_back((Float_t) y);
388 
389  TH1D* p = H1[ps.formula];
390 
391  p->SetBinContent(p->FindBin((double) quality->run), y);
392  p->SetBinError (p->FindBin((double) quality->run), 0.0);
393 
394  if (ps.range(y)) {
395  w += ps.weight;
396  }
397 
398  h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
399  }
400 
401  const double Q = w/W;
402 
403  tuple.push_back((Float_t) Q);
404 
405  int V = 0;
406 
407  for (size_t i = 0; i != veto.size(); ++i) {
408 
409  const JCondition& ps = veto[i];
410  const double y = getResult(ps.formula, *quality);
411 
412  TH1D* p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];
413 
414  p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
415  p->SetBinError (p->FindBin((double) quality->run), 0.0);
416 
417  if (!ps.range(y)) {
418  ++V;
419  }
420  }
421 
422  tuple.push_back((Float_t) V);
423  tuple.push_back((Float_t) setups.get(quality->run));
424 
425  h0.Fill(Q);
426  n1.Fill(tuple.data());
427 
428  cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
429  }
430 
431  double w = 0.0;
432 
433  for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
434  h0.SetBinContent(i, (w += h0.GetBinContent(i)));
435  }
436 
437  // store results
438 
439  TFile out(outputFile.c_str(), "recreate");
440 
441  out << h0 << h1 << n1 << H1;
442 
443  out.Write();
444  out.Close();
445  }
446 
447 
448  if (getFilenameExtension(outputFile) == "txt") {
449 
450  // store data
451 
452  ofstream out(outputFile);
453 
454  JRootWriter writer(out, JEquationParameters(), JDBDictionary::getInstance());
455 
456  for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
457 
458  writer.putObject(*i);
459 
460  out << ' ';
461 
462  writer.putObject(setups.get(i->run));
463 
464  out << endl;
465  }
466 
467  out.close();
468  }
469 }
Utility class to parse command line options.
Definition: JParser.hh:1500
data_type w[N+1][M+1]
Definition: JPolint.hh:741
Q(UTCMax_s-UTCMin_s)-livetime_s
int main(int argc, char *argv[])
Definition: Main.cc:15
std::vector< T >::difference_type distance(typename std::vector< T >::const_iterator first, typename PhysicsEvent::const_iterator< T > second)
Specialisation of STL distance.
std::vector< std::string > getGITTags(const TRegexp regexp, const JGITTags_t::key_type date)
Get selection of GIT tags.
Definition: JDB/JGITTags.hh:34
do rm f tmp H1
Print objects in ASCII format using ROOT dictionary.
*fatal Wrong number of arguments esac JCookie sh typeset Z DETECTOR typeset Z SOURCE_RUN typeset Z TARGET_RUN set_variable PARAMETERS_FILE $WORKDIR parameters
Definition: diff-Tuna.sh:38
#define MAKE_CSTRING(A)
Make C-string.
Definition: JPrint.hh:151
then for HISTOGRAM in h0 h1
Definition: JMatrixNZ.sh:69
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition: JParser.hh:66
then let ID
Definition: JAcoustics.sh:30
Dynamic ROOT object management.
Auxiliary data structure for floating point format specification.
Definition: JManip.hh:446
string outputFile
is
Definition: JDAQCHSM.chsm:167
bool resetObject(JManager< JKey_t, JValue_t > *object, const bool reset=false)
Reset JManager object.
Definition: JManager.hh:366
#define MAKE_STRING(A)
Make string.
Definition: JPrint.hh:142
JDetectorsHelper getDetector
Function object for mapping serial number to object identifier of detector and vice versa...
Definition: JDBToolkit.cc:5
#define ASSERT(A,...)
Assert macro.
Definition: JMessage.hh:90
T & getInstance(const T &object)
Get static instance from temporary object.
Definition: JObject.hh:75
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1961
Double_t getResult(const TString &text, TObject *object=NULL)
Get result of given textual formula.
Auxiliary methods for handling file names, type names and environment.
std::istream & getline(std::istream &in, JString &object)
Read string from input stream until end of line.
Definition: JString.hh:478
bool is_integer(const std::string &buffer)
Check if string is an integer.
Definition: JLangToolkit.hh:58
int debug
debug level
Definition: JSirene.cc:63
General purpose messaging.
#define FATAL(A)
Definition: JMessage.hh:67
z range($ZMAX-$ZMIN)< $MINIMAL_DZ." fi fi typeset -Z 4 STRING typeset -Z 2 FLOOR JPlot1D -f $
void reset(T &value)
Reset value.
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition: JHead.hh:1618
Auxiliary class to define a range between two values.
Utility class to parse command line options.
std::string getFilenameExtension(const std::string &file_name)
Get file name extension, i.e. part after last JEEP::FILENAME_SEPARATOR if any.
Definition: JeepToolkit.hh:69
then usage $script< input_file >< detector_file > fi set_variable OUTPUT_DIR set_variable SELECTOR JDAQTimesliceL1 set_variable DEBUG case set_variable DEBUG
ResultSet & getResultSet(const std::string &query)
Get result set.
Definition: JDB.hh:432
std::vector< JServer > getServernames()
Get list of names of available database servers.
Definition: JDB.hh:98
std::ostream & operator<<(std::ostream &stream, const CLBCommonHeader &header)
Selection of GIT tags.
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY source JAcoustics sh $DETECTOR_ID CHECK_EXIT_CODE typeset A TRIPODS get_tripods $WORKDIR tripod txt TRIPODS for EMITTER in
Definition: JCanberra.sh:38
std::vector< double > weight
Definition: JAlgorithm.hh:417