Jpp
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JDataQuality.cc
Go to the documentation of this file.
1 #include <iostream>
2 #include <sstream>
3 #include <fstream>
4 #include <iomanip>
5 #include <set>
6 #include <map>
7 
8 #include "TROOT.h"
9 #include "TFile.h"
10 #include "TH1D.h"
11 #include "TF1.h"
12 #include "TString.h"
13 #include "TNtuple.h"
14 #include "TRegexp.h"
15 
16 #include "JDB/JDB.hh"
17 #include "JDB/JSelector.hh"
19 #include "JDB/JRunQuality.hh"
20 #include "JDB/JRunsetups.hh"
21 #include "JDB/JDBToolkit.hh"
22 
23 #include "JLang/JLangToolkit.hh"
24 #include "JTools/JRange.hh"
25 #include "JROOT/JRootPrinter.hh"
26 #include "JROOT/JRootToolkit.hh"
27 #include "JGizmo/JManager.hh"
28 
29 #include "Jeep/JGITTags.hh"
30 #include "Jeep/JParser.hh"
31 #include "Jeep/JMessage.hh"
32 
33 namespace {
34 
35  /**
36  * Auxiliary data structure for run quality evaluation.
37  *
38  * A condition is composed of a ROOT TFormula compatible exoression and a range.
39  */
40  struct JCondition {
41 
42  static const char SEPARATOR = ';';
43 
44  /**
45  * Read selection from input stream.
46  *
47  * \param in input stream
48  * \param object selection
49  * \return input stream
50  */
51  friend inline std::istream& operator>>(std::istream& in, JCondition& object)
52  {
53  std::getline(in, object.formula, SEPARATOR);
54 
55  object.range = JTOOLS::JRange<double>();
56  object.weight = 1.0;
57 
58  if ((in >> object.range) && ! (in >> object.weight)) {
59  in.clear();
60  }
61 
62  return in;
63  }
64 
65 
66  /**
67  * Write selection to output stream.
68  *
69  * \param out output stream
70  * \param object selection
71  * \return output stream
72  */
73  friend inline std::ostream& operator<<(std::ostream& out, const JCondition& object)
74  {
75  out << object.formula << SEPARATOR << object.range << ' ' << object.weight << std::endl;
76 
77  return out;
78  }
79 
80  std::string formula;
82  double weight;
83  };
84 }
85 
86 
87 /**
88  * \file
89  *
90  * Example program to plot quality data from data base.
91  * \author mdejong
92  */
93 int main(int argc, char **argv)
94 {
95  using namespace std;
96  using namespace JPP;
97 
98  typedef JRange<int> JRange_t;
99  typedef vector<JCondition> JCondition_t;
100 
101  string usr;
102  string pwd;
103  string cookie;
104  string inputFile;
105  string outputFile;
106  string detid;
107  JRange_t runs;
108  vector<string> source;
109  JCondition_t condition;
110  JCondition_t veto;
111  TRegexp regexp(".");
112  JSelector selection;
113  int debug;
114 
115  try {
116 
117  const JGITTags_t tags = getGITTags();
118  const JGITTags_t::key_type date ("2019-04-12");
119  const TRegexp version("v.*\\..*\\..*");
120  vector<string> buffer;
121 
122  for (JGITTags_t::const_reverse_iterator i = tags.rbegin(); i.base() != tags.lower_bound(date); ++i) {
123  if (TString(i->second.c_str()).Contains(version)) {
124  buffer.push_back(getGITVersion(i->second));
125  }
126  }
127 
128  JParser<> zap("Example program to plot quality data from data base."\
129  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
130 
131  zap['u'] = make_field(usr) = "";
132  zap['!'] = make_field(pwd) = "";
133  zap['C'] = make_field(cookie) = "";
134  zap['f'] = make_field(inputFile, "Optional input file instead of database.") = "";
135  zap['o'] = make_field(outputFile, "Output file containing histograms and n-tuple.") = "quality.root";
136  zap['D'] = make_field(detid) = "";
137  zap['R'] = make_field(runs, "Run range") = JRange_t(1, JRange_t::getMaximum());
138  zap['S'] = make_field(source, "GIT versions") = buffer;
139  zap['Q'] = make_field(condition, "User defined conditions");
140  zap['V'] = make_field(veto, "User defined vetos.") = JPARSER::initialised();
141  zap['r'] = make_field(regexp, "TRegexp for selection of run setup names.") = JPARSER::initialised();
142  zap['@'] = make_field(selection) = JPARSER::initialised();
143  zap['d'] = make_field(debug, "Debug level") = 1;
144 
145  zap(argc, argv);
146  }
147  catch(const exception &error) {
148  FATAL(error.what() << endl);
149  }
150 
151 
152  double W = 0.0;
153 
154  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
155  W += i->weight;
156  }
157 
158  if (W <= 0.0) {
159  FATAL("Invalid total weight: " << W << endl);
160  }
161 
162  JRunsetups setup;
163 
164  set<JRunQuality> buffer;
165 
166  if (inputFile == "") {
167 
168  ASSERT(detid != "");
169 
170  int ID = -1; // integer representation of detectir identifier
171 
172  try {
173 
174  JDB::reset(usr, pwd, cookie);
175 
176  if (is_integer(detid))
177  ID = to_value<int>(detid);
178  else
179  ID = getDetector(detid);
180 
181  if (is_integer(detid)) {
182  detid = getDetector(to_value<int>(detid));
183  }
184 
185  selection += getSelector<JRuns>(ID);
186 
187  ResultSet& rs = getResultSet(getTable<JRuns>(), selection);
188 
189  for (JRuns parameters; rs >> parameters; ) {
190  if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
191  setup.put(parameters);
192  }
193  }
194 
195  rs.Close();
196 
197  for (vector<string>::const_iterator i = source.begin(); i != source.end(); ++i) {
198 
199  typedef map<string, string> data_type;
200  typedef map<int, data_type> map_type;
201 
202  map_type zmap;
203 
204  JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
205 
206  selector.add(&JRunSummaryNumbers::SOURCE_NAME, *i);
207 
208  try {
209 
210  ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
211 
212  for (JRunSummaryNumbers parameters; rs >> parameters; ) {
213  if (setup.has(parameters.RUN)) {
214  zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
215  }
216  }
217  }
218  catch(const exception& error) {}
219 
220  for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
221 
222  JRunQuality quality;
223 
224  quality.GIT = *i;
225  quality.detector = ID;
226  quality.run = run->first;
227 
228  for (data_type::const_iterator p = run->second.begin(); p != run->second.end(); ++p) {
229  quality.put(p->first, p->second);
230  }
231 
232  buffer.insert(quality);
233  }
234  }
235  }
236  catch(const exception& error) {
237  FATAL(error.what() << endl);
238  }
239 
240  } else {
241 
242  ifstream in(inputFile.c_str());
243 
244  for (JRunQuality quality; in >> quality; ) {
245  buffer.insert(quality);
246  }
247 
248  in.close();
249  }
250 
251  if (buffer.empty()) {
252  FATAL("No data." << endl);
253  }
254 
255  if (debug >= debug_t) {
256  for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
257  cout << i->run << ' ' << i->GIT << endl;
258  }
259  }
260 
261 
262  runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);
263 
264  TH1D h0("h0", NULL, 1000, 0.0, 1.01);
265  TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() + 0.5);
266 
267  h0.SetMinimum(0.0);
268  h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
269 
270  h1.SetMinimum(0.0);
271  h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));
272 
273  JManager<TString, TH1D> zmap(new TH1D("%", NULL,
274  runs.getLength() + 1,
275  runs.getLowerLimit() - 0.5,
276  runs.getUpperLimit() + 0.5));
277 
278  // show range
279 
280  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
281 
282  const string buffer = i->formula;
283 
284  zmap[buffer]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(buffer << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), runs.getLowerLimit(), runs.getUpperLimit()));
285  zmap[buffer]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(buffer << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), runs.getLowerLimit(), runs.getUpperLimit()));
286  }
287 
288  for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
289 
290  const string buffer = MAKE_STRING("VETO[" << i->formula << "]");
291 
292  zmap[buffer]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(buffer << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), runs.getLowerLimit(), runs.getUpperLimit()));
293  zmap[buffer]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING(buffer << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), runs.getLowerLimit(), runs.getUpperLimit()));
294  }
295 
296  ostringstream os;
297 
298  os << "run";
299 
300  for (size_t i = 0; i != condition.size(); ++i) {
301  os << ":" << (char) ('a' + i);
302  }
303 
304  os << ":Q:V:R";
305 
306  TNtuple n1("n1", "quality", os.str().c_str());
307 
308 
309  resetObject(&h0);
310  resetObject(&h1);
311  resetObject(&n1);
312  resetObject(&zmap);
313 
314  // process data
315 
316  for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
317 
318  vector<Float_t> tuple;
319 
320  tuple.push_back((Float_t) quality->run);
321 
322  double w = 0.0;
323 
324  for (size_t i = 0; i != condition.size(); ++i) {
325 
326  const JCondition& ps = condition[i];
327  const double y = getResult(ps.formula, *quality);
328 
329  tuple.push_back((Float_t) y);
330 
331  TH1D* p = zmap[ps.formula];
332 
333  p->SetBinContent(p->FindBin((double) quality->run), y);
334  p->SetBinError (p->FindBin((double) quality->run), 0.0);
335 
336  if (ps.range(y)) {
337  w += ps.weight;
338  }
339 
340  h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
341  }
342 
343  const double Q = w/W;
344 
345  tuple.push_back((Float_t) Q);
346 
347  int V = 0;
348 
349  for (size_t i = 0; i != veto.size(); ++i) {
350 
351  const JCondition& ps = veto[i];
352  const double y = getResult(ps.formula, *quality);
353 
354  TH1D* p = zmap[MAKE_STRING("VETO[" << ps.formula << "]")];
355 
356  p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
357  p->SetBinError (p->FindBin((double) quality->run), 0.0);
358 
359  if (!ps.range(y)) {
360  ++V;
361  }
362  }
363 
364  tuple.push_back((Float_t) V);
365  tuple.push_back((Float_t) setup.get(quality->run));
366 
367  h0.Fill(Q);
368  n1.Fill(tuple.data());
369 
370  cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
371  }
372 
373  double w = 0.0;
374 
375  for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
376  h0.SetBinContent(i, (w += h0.GetBinContent(i)));
377  }
378 
379  // output
380 
381  TFile out(outputFile.c_str(), "recreate");
382 
383  out << h0 << h1 << n1 << zmap;
384 
385  out.Write();
386  out.Close();
387 }
Utility class to parse command line options.
Definition: JParser.hh:1493
data_type w[N+1][M+1]
Definition: JPolint.hh:708
debug
Definition: JMessage.hh:29
Print objects in ASCII format using ROOT dictionary.
esac print_variable DETECTOR INPUT_FILE OUTPUT_FILE CDF for TYPE in
Definition: JSirene.sh:45
*fatal Wrong number of arguments esac JCookie sh typeset Z DETECTOR typeset Z SOURCE_RUN typeset Z TARGET_RUN set_variable PARAMETERS_FILE $WORKDIR parameters
Definition: diff-Tuna.sh:38
#define MAKE_CSTRING(A)
Make C-string.
Definition: JPrint.hh:708
then for HISTOGRAM in h0 h1
Definition: JMatrixNZ.sh:69
Empty structure for specification of parser element that is initialised (i.e. does not require input)...
Definition: JParser.hh:63
Dynamic ROOT object management.
Auxiliary data structure for floating point format specification.
Definition: JPrint.hh:481
string outputFile
#define MAKE_STRING(A)
Make string.
Definition: JPrint.hh:699
std::map< JDate<'-'>, std::string > JGITTags_t
Type definition of dated GIT tags.
Definition: JGITTags.hh:25
Dates GIT tags.
#define ASSERT(A,...)
Assert macro.
Definition: JMessage.hh:90
std::string getGITVersion(const std::string &tag)
Get GIT version for given GIT tag.
Definition: JGITTags.hh:34
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1954
Double_t getResult(const TString &text, TObject *object=NULL)
Get result of given textual formula.
then usage $script[detector file[variant[identifier]]] fi case set_variable ID
Definition: JDetector.sh:30
std::istream & getline(std::istream &in, JString &object)
Read string from input stream until end of line.
Definition: JString.hh:478
bool is_integer(const std::string &buffer)
Check if string is an integer.
Definition: JLangToolkit.hh:58
int debug
debug level
Definition: JSirene.cc:61
General purpose messaging.
#define FATAL(A)
Definition: JMessage.hh:67
JRange< Double_t > JRange_t
Definition: JFitToT.hh:34
static const JDetectorsHelper & getDetector
Function object for mapping serial number and object identifier of detectors.
Definition: JDBToolkit.hh:131
void reset(T &value)
Reset value.
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition: JHead.hh:1278
const JGITTags_t & getGITTags()
Get GIT dated tags.
Definition: JGITTags.hh:54
Auxiliary class to define a range between two values.
Utility class to parse command line options.
ResultSet & getResultSet(const std::string &query)
Get result set.
Definition: JDB.hh:269
std::ostream & operator<<(std::ostream &stream, const CLBCommonHeader &header)
bool resetObject(JManager< JKey_t, JValue_t > *object, const bool reset=false)
Reset JManager object.
Definition: JManager.hh:365
version
Definition: JCalibratePMT.sh:7
std::vector< double > weight
Definition: JAlgorithm.hh:428
int main(int argc, char *argv[])
Definition: Main.cpp:15