Jpp  15.0.3
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Functions
JDataQuality.cc File Reference

Main program to evaluate quality data from data base. More...

#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>
#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TF1.h"
#include "TString.h"
#include "TNtuple.h"
#include "TRegexp.h"
#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRunsetups.hh"
#include "JDB/JDBToolkit.hh"
#include "JDB/JGITTags.hh"
#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootClass.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JRootStreamObjectOutput.hh"
#include "JROOT/JRootDictionary.hh"
#include "JROOT/JASCIIFileStreamer.hh"
#include "JROOT/JManager.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JFilenameSupportkit.hh"
#include "Jeep/JeepToolkit.hh"
#include "Jeep/JComment.hh"
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"

Go to the source code of this file.

Functions

int main (int argc, char **argv)
 

Detailed Description

Main program to evaluate quality data from data base.

The evaluation of the data quality is based on the following procedure.
For each data taking run, the script JQAQC.sh is executed which produces a set of values. The values are subsequently uploaded into the database in table "runsummarynumbers". This table is a general purpose table for any value related to some data taking run. With this application, the QA/QC data are consistently downloaded from the database; evaluated according some formula and tested against a validity range. The underlying data structure is JDATABASE::JRunQuality.

The formula and validity range are referred to as a condition and can be specified with option -Q. Each condition has an optional weight (default is 1). The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best. There is also the possibility to specify a veto (option -V), which is similar to a condition but only the number of vetoes is counted. The overall quality and the number of vetoes can be used to accept or reject a data taking run.

By default, a list of the run numbers, the overall weight and the number of vetoes is printed. If a ROOT file is specified for the output (i.e. filename extension is JSUPPORT::ROOT_FILE_FORMAT), a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple. If an ASCII file is specified for the output (i.e. filename extension is JSUPPORT::ASCII_FILE_FORMAT), a text file is written which can subsequently be used as input file.

Author
mdejong

Definition in file JDataQuality.cc.

Function Documentation

int main ( int  argc,
char **  argv 
)

Definition at line 126 of file JDataQuality.cc.

127 {
128  using namespace std;
129  using namespace JPP;
130 
131  typedef JRange<int> JRange_t;
132  typedef vector<JCondition> JCondition_t;
133 
134  JServer server;
135  string usr;
136  string pwd;
137  string cookie;
138  string inputFile;
139  string outputFile;
140  string detid;
141  JRange_t runs;
142  vector<string> source;
143  JCondition_t condition;
144  JCondition_t veto;
145  TRegexp regexp(".");
146  JSelector selection;
147  int debug;
148 
149  try {
150 
151  JParser<> zap("Main program to evaluate quality data from data base."\
152  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
153 
154  zap['s'] = make_field(server) = getServernames();
155  zap['u'] = make_field(usr) = "";
156  zap['!'] = make_field(pwd) = "";
157  zap['C'] = make_field(cookie) = "";
158  zap['f'] = make_field(inputFile, "Optional input file instead of database.") = "";
159  zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "quality.root";
160  zap['D'] = make_field(detid) = "";
161  zap['R'] = make_field(runs, "Run range") = JRange_t(1, JRange_t::getMaximum());
162  zap['S'] = make_field(source, "GIT versions") = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
163  zap['Q'] = make_field(condition, "User defined conditions") = JPARSER::initialised();
164  zap['V'] = make_field(veto, "User defined vetos.") = JPARSER::initialised();
165  zap['r'] = make_field(regexp, "TRegexp for selection of run setup names.") = JPARSER::initialised();
166  zap['@'] = make_field(selection) = JPARSER::initialised();
167  zap['d'] = make_field(debug, "Debug level") = 1;
168 
169  zap(argc, argv);
170  }
171  catch(const exception &error) {
172  FATAL(error.what() << endl);
173  }
174 
175 
176  double W = 0.0;
177 
178  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
179  W += i->weight;
180  }
181 
183  if (W <= 0.0) {
184  FATAL("Invalid total weight: " << W << endl);
185  }
186  }
187 
188 
189  JRunsetups setups; // run -> value
190  set<JRunQuality> buffer; // storage
191 
192 
193  if (inputFile == "") { // read data from database
194 
195  ASSERT(detid != "");
196 
197  try {
198 
199  JDB::reset(usr, pwd, cookie);
200 
201  const int ID = getDetector<int> (detid);
202  detid = getDetector<string>(detid);
203 
204  // run setups
205 
206  selection += getSelector<JRuns>(ID);
207 
208  NOTICE("Extracting run information from database... " << flush);
209 
210  ResultSet& rs = getResultSet(getTable<JRuns>(), selection);
211 
212  for (JRuns parameters; rs >> parameters; ) {
213  if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
214  setups.put(parameters);
215  }
216  }
217 
218  rs.Close();
219 
220  NOTICE("OK" << endl);
221 
222  // run summary data
223 
224  for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {
225 
226  typedef map<string, string> data_type;
227  typedef map<int, data_type> map_type;
228 
229  map_type zmap;
230 
231  JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
232 
233  selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);
234 
235  try {
236 
237  NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);
238 
239  ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
240 
241  for (JRunSummaryNumbers parameters; rs >> parameters; ) {
242  if (setups.has(parameters.RUN)) {
243  zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
244  }
245  }
246 
247  rs.Close();
248 
249  NOTICE("OK" << endl);
250  }
251  catch(const exception& error) { NOTICE(endl); }
252 
253  for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
254 
255  JRunQuality quality;
256 
257  quality.GIT = *git;
258  quality.detector = ID;
259  quality.run = run->first;
260  quality.value = setups[run->first].value;
261 
262  for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
263  quality.put(i->first, i->second);
264  }
265 
266  buffer.insert(quality); // only insert if absent
267  }
268  }
269  }
270  catch(const exception& error) {
271  FATAL(error.what() << endl);
272  }
273 
274  } else { // read data from file
275 
276  JASCIIFileReader<JRunQuality> in(inputFile.c_str(), JDBDictionary::getInstance());
277 
278  JComment comment;
279 
280  in >> comment;
281 
282  for (JRunQuality quality; in >> quality; ) {
283  buffer.insert(quality);
284  }
285 
286  in.close();
287  }
288 
289 
290  if (buffer.empty()) {
291  FATAL("No data." << endl);
292  }
293 
294 
296 
297  runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);
298 
299  TH1D h0("h0", NULL, 1000, 0.0, 1.01);
300  TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() - 0.5);
301 
302  for (size_t i = 0; i != condition.size(); ++i) {
303  h1.GetXaxis()->SetBinLabel(i+1, condition[i].formula.c_str());
304  }
305 
306  h0.SetMinimum(0.0);
307  h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
308 
309  h1.SetMinimum(0.0);
310  h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
311 
312  const Double_t xmin = runs.getLowerLimit() - 0.5;
313  const Double_t xmax = runs.getUpperLimit() + 0.5;
314 
315  JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));
316 
317  // show validity range
318 
319  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
320  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
321  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
322  }
323 
324  for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
325  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
326  H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
327  }
328 
329  ostringstream os; // N-tuple format
330 
331  os << "run";
332 
333  for (size_t i = 0; i != condition.size(); ++i) {
334  os << ":" << (char) ('a' + i);
335  }
336 
337  os << ":Q:V:R";
338 
339  TNtuple n1("n1", "quality", os.str().c_str());
340 
341 
342  // process data
343 
344  for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
345 
346  vector<Float_t> tuple(1, (Float_t) quality->run);
347 
348  double w = 0.0;
349 
350  for (size_t i = 0; i != condition.size(); ++i) {
351 
352  const JCondition& ps = condition[i];
353  const double y = getResult(ps.formula, *quality);
354 
355  DEBUG(ps.formula << ' ' << y << endl);
356 
357  tuple.push_back((Float_t) y);
358 
359  TH1D* p = H1[ps.formula];
360 
361  p->SetBinContent(p->FindBin((double) quality->run), y);
362  p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
363 
364  if (ps.range(y)) {
365  w += ps.weight;
366  }
367 
368  h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
369  }
370 
371  const double Q = w/W;
372 
373  tuple.push_back((Float_t) Q);
374 
375  int V = 0;
376 
377  for (size_t i = 0; i != veto.size(); ++i) {
378 
379  const JCondition& ps = veto[i];
380  const double y = getResult(ps.formula, *quality);
381 
382  DEBUG(ps.formula << ' ' << y << endl);
383 
384  TH1D* p = NULL;
385 
386  p = H1[ps.formula];
387 
388  p->SetBinContent(p->FindBin((double) quality->run), y);
389  p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
390 
391  p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];
392 
393  p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
394  p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
395 
396  if (!ps.range(y)) {
397  ++V;
398  }
399  }
400 
401  tuple.push_back((Float_t) V);
402  tuple.push_back((Float_t) setups.get(quality->run));
403 
404  h0.Fill(Q);
405  n1.Fill(tuple.data());
406 
407  cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
408  }
409 
410  double w = 0.0;
411 
412  for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
413  h0.SetBinContent(i, (w += h0.GetBinContent(i)));
414  }
415 
416  // store results
417 
418  TFile out(outputFile.c_str(), "recreate");
419 
420  out << h0 << h1 << n1 << H1;
421 
422  out.Write();
423  out.Close();
424  }
425 
426 
428 
429  // store data
430 
431  JASCIIFileWriter<JRunQuality> out(outputFile.c_str(), JDBDictionary::getInstance());
432 
433  out.setf(ios::fixed);
434 
435  JComment comment;
436 
437  comment.add(JMeta(argc, argv));
438 
439  out << comment;
440 
441  for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
442  out << *i << endl;
443  }
444 
445  out.close();
446  }
447 }
Utility class to parse command line options.
Definition: JParser.hh:1500
data_type w[N+1][M+1]
Definition: JPolint.hh:741
Q(UTCMax_s-UTCMin_s)-livetime_s
*fatal Wrong number of arguments esac JCookie sh typeset Z DETECTOR typeset Z SOURCE_RUN typeset Z TARGET_RUN set_variable PARAMETERS_FILE $WORKDIR parameters
Definition: diff-Tuna.sh:38
#define MAKE_CSTRING(A)
Make C-string.
Definition: JPrint.hh:151
then for HISTOGRAM in h0 h1
Definition: JMatrixNZ.sh:71
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition: JParser.hh:66
then let ID
Definition: JAcoustics.sh:31
Auxiliary data structure for floating point format specification.
Definition: JManip.hh:446
static const char *const ASCII_FILE_FORMAT
file name extension ASCII format
V(JDAQEvent-JTriggerReprocessor)*1.0/(JDAQEvent+1.0e-10)
string outputFile
#define MAKE_STRING(A)
Make string.
Definition: JPrint.hh:142
std::vector< std::string > getGITTags(const TRegexp &regexp, const JGITTags_t::key_type &date)
Get selection of GIT tags.
Definition: JDB/JGITTags.hh:34
#define ASSERT(A,...)
Assert macro.
Definition: JMessage.hh:90
T & getInstance(const T &object)
Get static instance from temporary object.
Definition: JObject.hh:75
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1961
Double_t getResult(const TString &text, TObject *object=NULL)
Get result of given textual formula.
#define NOTICE(A)
Definition: JMessage.hh:64
int debug
debug level
Definition: JSirene.cc:63
#define FATAL(A)
Definition: JMessage.hh:67
void reset(T &value)
Reset value.
std::string getFilenameExtension(const std::string &file_name)
Get file name extension, i.e. part after last JEEP::FILENAME_SEPARATOR if any.
Definition: JeepToolkit.hh:69
then usage $script< input_file >< detector_file > fi set_variable OUTPUT_DIR set_variable SELECTOR JDAQTimesliceL1 set_variable DEBUG case set_variable DEBUG
ResultSet & getResultSet(const std::string &query)
Get result set.
Definition: JDB.hh:432
std::vector< JServer > getServernames()
Get list of names of available database servers.
Definition: JDB.hh:98
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY source JAcoustics sh $DETECTOR_ID CHECK_EXIT_CODE typeset A TRIPODS get_tripods $WORKDIR tripod txt TRIPODS for EMITTER in
Definition: JCanberra.sh:42
static const char *const ROOT_FILE_FORMAT
file name extension ROOT format