Jpp test-rotations-new
the software that should make you happy
Loading...
Searching...
No Matches
JDataQuality.cc
Go to the documentation of this file.
1#include <iostream>
2#include <sstream>
3#include <fstream>
4#include <iomanip>
5#include <vector>
6#include <set>
7#include <map>
8
9#include "TROOT.h"
10#include "TFile.h"
11#include "TH1D.h"
12#include "TF1.h"
13#include "TString.h"
14#include "TNtuple.h"
15#include "TRegexp.h"
16
17#include "JDB/JDB.hh"
18#include "JDB/JSelector.hh"
20#include "JDB/JRunQuality.hh"
21#include "JDB/JRunsetups.hh"
22#include "JDB/JDBToolkit.hh"
23
24#include "JLang/JLangToolkit.hh"
25#include "JTools/JRange.hh"
26#include "JROOT/JRootClass.hh"
27#include "JROOT/JRootPrinter.hh"
31#include "JROOT/JManager.hh"
32#include "JSupport/JMeta.hh"
34
36
37#include "Jeep/JeepToolkit.hh"
38#include "Jeep/JComment.hh"
39#include "Jeep/JParser.hh"
40#include "Jeep/JMessage.hh"
41
42namespace {
43
44 /**
45 * Auxiliary data structure for run quality evaluation.
46 *
47 * A condition is composed of a ROOT TFormula compatible expression and a validity range for the result.
48 */
49 struct JCondition {
50
51 static const char SEPARATOR = ';';
52
53 /**
54 * Read selection from input stream.
55 *
56 * \param in input stream
57 * \param object selection
58 * \return input stream
59 */
60 friend inline std::istream& operator>>(std::istream& in, JCondition& object)
61 {
62 std::getline(in, object.formula, SEPARATOR);
63
64 object.range = JTOOLS::JRange<double>();
65 object.weight = 1.0;
66
67 if ((in >> object.range) && ! (in >> object.weight)) {
68 in.clear();
69 }
70
71 return in;
72 }
73
74
75 /**
76 * Write selection to output stream.
77 *
78 * \param out output stream
79 * \param object selection
80 * \return output stream
81 */
82 friend inline std::ostream& operator<<(std::ostream& out, const JCondition& object)
83 {
84 out << object.formula << SEPARATOR << object.range << ' ' << object.weight << std::endl;
85
86 return out;
87 }
88
89 std::string formula;
91 double weight;
92 };
93}
94
95
96/**
97 * \file
98 *
99 * Main program to evaluate quality data from data base.
100 *
101 * The evaluation of the data quality is based on the following procedure.\n
102 * For each data taking run, the script JQAQC.sh is executed which produces a set of values.
103 * The values are subsequently uploaded into the database in table "runsummarynumbers".
104 * This table is a general purpose table for any value related to some data taking run.
105 * With this application,
106 * the QA/QC data are consistently downloaded from the database;
107 * evaluated according some formula and
108 * tested against a validity range.
109 * The underlying data structure is JDATABASE::JRunQuality.
110 *
111 * The formula and validity range are referred to as a condition and can be specified with option -Q.
112 * Each condition has an optional weight (default is 1).
113 * The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best.
114 * There is also the possibility to specify a veto (option -V),
115 * which is similar to a condition but only the number of vetoes is counted.
116 * The overall quality and the number of vetoes can be used to accept or reject a data taking run.
117 *
118 * By default, a list of the run numbers, the overall weight and the number of vetoes is printed.
119 * If a ROOT file is specified for the output (i.e.\ filename extension is JSUPPORT::ROOT_FILE_FORMAT),
120 * a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple.
121 * If an ASCII file is specified for the output (i.e.\ filename extension is JSUPPORT::ASCII_FILE_FORMAT),
122 * a text file is written which can subsequently be used as input file.
123 *
124 * \author mdejong
125 */
126int main(int argc, char **argv)
127{
128 using namespace std;
129 using namespace JPP;
130
131 typedef JRange<int> JRange_t;
132 typedef vector<JCondition> JCondition_t;
133
134 JServer server;
135 string usr;
136 string pwd;
137 string cookie;
138 string inputFile;
139 string outputFile;
140 string detid;
141 JRange_t runs;
142 vector<string> source;
143 JCondition_t condition;
144 JCondition_t veto;
145 TRegexp regexp(".");
146 JSelector selection;
147 int debug;
148
149 try {
150
151 JParser<> zap("Main program to evaluate quality data from data base."\
152 "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
153
154 zap['s'] = make_field(server) = getServernames();
155 zap['u'] = make_field(usr) = "";
156 zap['!'] = make_field(pwd) = "";
157 zap['C'] = make_field(cookie) = "";
158 zap['f'] = make_field(inputFile, "Optional input file instead of database.") = "";
159 zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "quality.root";
160 zap['D'] = make_field(detid) = "";
161 zap['R'] = make_field(runs, "Run range") = JRange_t(1, JRange_t::getMaximum());
162 zap['S'] = make_field(source, "GIT versions") = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
163 zap['Q'] = make_field(condition, "User defined conditions") = JPARSER::initialised();
164 zap['V'] = make_field(veto, "User defined vetos.") = JPARSER::initialised();
165 zap['r'] = make_field(regexp, "TRegexp for selection of run setup names.") = JPARSER::initialised();
166 zap['@'] = make_field(selection) = JPARSER::initialised();
167 zap['d'] = make_field(debug, "Debug level") = 1;
168
169 zap(argc, argv);
170 }
171 catch(const exception &error) {
172 FATAL(error.what() << endl);
173 }
174
175
176 double W = 0.0;
177
178 for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
179 W += i->weight;
180 }
181
182 if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {
183 if (W <= 0.0) {
184 FATAL("Invalid total weight: " << W << endl);
185 }
186 }
187
188
189 JRunsetups setups; // run -> value
190 set<JRunQuality> buffer; // storage
191
192
193 if (inputFile == "") { // read data from database
194
195 ASSERT(detid != "");
196
197 try {
198
199 JDB::reset(usr, pwd, cookie);
200
201 const int ID = getDetector<int> (detid);
202 detid = getDetector<string>(detid);
203
204 // run setups
205
206 selection += getSelector<JRuns>(ID);
207
208 NOTICE("Extracting run information from database... " << flush);
209
210 ResultSet& rs = getResultSet(getTable<JRuns>(), selection);
211
212 for (JRuns parameters; rs >> parameters; ) {
213 if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
214 setups.put(parameters);
215 }
216 }
217
218 rs.Close();
219
220 NOTICE("OK" << endl);
221
222 // run summary data
223
224 for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {
225
227 typedef map<int, data_type> map_type;
228
229 map_type zmap;
230
231 JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
232
233 selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);
234
235 try {
236
237 NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);
238
239 ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
240
241 for (JRunSummaryNumbers parameters; rs >> parameters; ) {
242 if (setups.has(parameters.RUN)) {
243 zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
244 }
245 }
246
247 rs.Close();
248
249 NOTICE("OK" << endl);
250 }
251 catch(const exception& error) { NOTICE(endl); }
252
253 for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
254
255 JRunQuality quality;
256
257 quality.GIT = *git;
258 quality.detector = ID;
259 quality.run = run->first;
260 quality.name = setups[run->first].name;
261 quality.value = setups[run->first].value;
262
263 for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
264 quality.put(i->first, i->second);
265 }
266
267 quality.name = replace(quality.name, ' ', '_');
268
269 buffer.insert(quality); // only insert if absent
270 }
271 }
272 }
273 catch(const exception& error) {
274 FATAL(error.what() << endl);
275 }
276
277 } else { // read data from file
278
279 if (detid != "" && !is_integer(detid)) {
280 WARNING("Detector identifier \"" << detid << "\" discarded." << endl);
281 }
282
283 const int ID = (is_integer(detid) ? to_value<int>(detid) : -1);
284
285 JASCIIFileReader<JRunQuality> in(inputFile.c_str(), JDBDictionary::getInstance());
286
287 JComment comment;
288
289 in >> comment;
290
291 for (JRunQuality quality; in >> quality; ) {
292 if (ID == -1 || ID == quality.detector) {
293 if (runs(quality.run)) {
294 buffer.insert(quality);
295 }
296 }
297 }
298
299 in.close();
300 }
301
302
303 if (buffer.empty()) {
304 FATAL("No data." << endl);
305 }
306
307 if (debug >= debug_t) {
308
309 JRootWriter writer(cout, JEquationParameters(), JDBDictionary::getInstance());
310
311 for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
312 writer.put(*quality);
313 }
314 }
315
316
317 if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {
318
319 runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);
320
321 TH1D h0("h0", NULL, 1000, 0.0, 1.01);
322 TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() - 0.5);
323
324 for (size_t i = 0; i != condition.size(); ++i) {
325 h1.GetXaxis()->SetBinLabel(i+1, condition[i].formula.c_str());
326 }
327
328 h0.SetMinimum(0.0);
329 h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit() + 1));
330
331 h1.SetMinimum(0.0);
332 h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit() + 1));
333
334 const Double_t xmin = runs.getLowerLimit() - 0.5;
335 const Double_t xmax = runs.getUpperLimit() + 0.5;
336
337 JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));
338
339 // show validity range
340
341 for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
342 H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
343 H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
344 }
345
346 for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
347 H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
348 H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
349 }
350
351 ostringstream os; // N-tuple format
352
353 os << "run";
354
355 for (size_t i = 0; i != condition.size(); ++i) {
356 os << ":" << (char) ('a' + i);
357 }
358
359 os << ":Q:V:R";
360
361 TNtuple n1("n1", "quality", os.str().c_str());
362
363
364 // process data
365
366 for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
367
368 vector<Float_t> tuple(1, (Float_t) quality->run);
369
370 double w = 0.0;
371
372 for (size_t i = 0; i != condition.size(); ++i) {
373
374 const JCondition& ps = condition[i];
375 const double y = getResult(ps.formula, *quality);
376
377 DEBUG(ps.formula << ' ' << y << endl);
378
379 tuple.push_back((Float_t) y);
380
381 TH1D* p = H1[ps.formula];
382
383 p->SetBinContent(p->FindBin((double) quality->run), y);
384 p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
385
386 if (ps.range(y)) {
387 w += ps.weight;
388 }
389
390 h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
391 }
392
393 const double Q = w/W;
394
395 tuple.push_back((Float_t) Q);
396
397 int V = 0;
398
399 for (size_t i = 0; i != veto.size(); ++i) {
400
401 const JCondition& ps = veto[i];
402 const double y = getResult(ps.formula, *quality);
403
404 DEBUG(ps.formula << ' ' << y << endl);
405
406 TH1D* p = NULL;
407
408 p = H1[ps.formula];
409
410 p->SetBinContent(p->FindBin((double) quality->run), y);
411 p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
412
413 p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];
414
415 p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
416 p->SetBinError (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());
417
418 if (!ps.range(y)) {
419 ++V;
420 }
421 }
422
423 tuple.push_back((Float_t) V);
424 tuple.push_back((Float_t) setups.get(quality->run));
425
426 h0.Fill(Q);
427 n1.Fill(tuple.data());
428
429 cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
430 }
431
432 double w = 0.0;
433
434 for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
435 h0.SetBinContent(i, (w += h0.GetBinContent(i)));
436 }
437
438 // store results
439
440 TFile out(outputFile.c_str(), "recreate");
441
442 out << h0 << h1 << n1 << H1;
443
444 out.Write();
445 out.Close();
446 }
447
448
449 if (getFilenameExtension(outputFile) == ASCII_FILE_FORMAT) {
450
451 // store data
452
453 JASCIIFileWriter<JRunQuality> out(outputFile.c_str(), JDBDictionary::getInstance());
454
455 out.setf(ios::fixed);
456
457 JComment comment;
458
459 comment.add(JMeta(argc, argv));
460
461 out << comment;
462
463 for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
464 out << *i << endl;
465 }
466
467 out.close();
468 }
469}
string outputFile
Selection of GIT tags.
int main(int argc, char **argv)
Specifications of file name extensions.
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition JHead.hh:1832
Dynamic ROOT object management.
General purpose messaging.
#define DEBUG(A)
Message macros.
Definition JMessage.hh:62
#define ASSERT(A,...)
Assert macro.
Definition JMessage.hh:90
#define NOTICE(A)
Definition JMessage.hh:64
#define FATAL(A)
Definition JMessage.hh:67
int debug
debug level
Definition JSirene.cc:72
#define WARNING(A)
Definition JMessage.hh:65
ROOT I/O of application specific meta data.
Utility class to parse command line options.
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition JParser.hh:2142
TString replace(const TString &target, const TRegexp &regexp, const T &replacement)
Replace regular expression in input by given replacement.
#define MAKE_CSTRING(A)
Make C-string.
Definition JPrint.hh:72
#define MAKE_STRING(A)
Make string.
Definition JPrint.hh:63
Auxiliary class to define a range between two values.
Print objects in ASCII format using ROOT dictionary.
Auxiliary methods for handling file names, type names and environment.
Object reading from ASCII file.
Object(s) writing to ASCII file.
virtual void close()
Close file.
virtual void close()
Close file.
Simple data structure to support I/O of equations (see class JLANG::JEquation).
Utility class to parse command line options.
Definition JParser.hh:1698
Implementation for ASCII output of objects with ROOT dictionary.
JRootWriter & put(const T &object)
Write object according equation format.
Range of values.
Definition JRange.hh:42
std::ostream & operator<<(std::ostream &stream, const CLBCommonHeader &header)
This name space includes all other name spaces (except KM3NETDAQ, KM3NET and ANTARES).
Auxiliary data structure for floating point format specification.
Definition JManip.hh:448
Data structure for measured coincidence rates of all pairs of PMTs in optical module.
Definition JFitK40.hh:103
Auxiliary data structure for data quality.
double value
setup identifier (see JRunsetups)
void put(const std::string &key, const std::string &value)
Put value at given key.
std::string name
setup name (see JRunsetups)
std::string GIT
GIT version used to write QA/QC data.
int detector
detector identifier
Auxiliary class for run setup evaluation.
Definition JRunsetups.hh:36
void put(const int run, const std::string setup)
Put run parameters.
bool has(const int run) const
Check if run setup is vailable.
float get(const int run) const
Get run setup value.
Wrapper class for server name.
Definition JDB.hh:54
Template definition for getting table specific selector.
Auxiliary class for comment.
Definition JComment.hh:43
JComment & add(const std::string &comment)
Add comment.
Definition JComment.hh:100
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition JParser.hh:68
Auxiliary class for ROOT I/O of application specific meta data.
Definition JMeta.hh:72