Jpp - the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JDataMonitor.cc
Go to the documentation of this file.
1 #include <string>
2 #include <iostream>
3 #include <sstream>
4 #include <iomanip>
5 #include <vector>
6 #include <set>
7 #include <map>
8 #include <algorithm>
9 
10 #include "TROOT.h"
11 #include "TFile.h"
12 #include "TH1D.h"
13 #include "TTimeStamp.h"
14 
15 #include "JDB/JDB.hh"
16 #include "JDB/JSelector.hh"
18 #include "JDB/JRunQuality.hh"
19 #include "JDB/JRuns.hh"
20 #include "JDB/JDBToolkit.hh"
21 #include "JDB/JDBSupportkit.hh"
22 #include "JDB/JGITTags.hh"
23 
24 #include "JLang/JVectorize.hh"
25 #include "JLang/JLangToolkit.hh"
26 #include "JTools/JRange.hh"
27 #include "JROOT/JRootPrinter.hh"
28 #include "JROOT/JRootToolkit.hh"
29 #include "JROOT/JMathSupportkit.hh"
30 #include "JROOT/JManager.hh"
31 #include "JGizmo/JGizmoToolkit.hh"
32 
33 #include "Jeep/JParser.hh"
34 #include "Jeep/JMessage.hh"
35 
36 
37 /**
38  * Read time stamp from input stream.
39  *
40  * Format is <tt>YYYY-MM-DD HH:MM:SS"</tt>
41  *
42  * \param in input stream
43  * \param object time stamp
44  * \return input stream
45  */
46 inline std::istream& operator>>(std::istream& in, TTimeStamp& object)
47 {
48  using namespace std;
49  using namespace JPP;
50 
51  string buffer;
52 
53  if (in >> buffer) {
54 
55  UInt_t year;
56  UInt_t month;
57  UInt_t day;
58 
59  istringstream is;
60 
61  is.str(replace(buffer, '-', ' '));
62 
63  if (is >> year >> month >> day) {
64 
65  UInt_t hour = 0;
66  UInt_t min = 0;
67  UInt_t sec = 0;
68 
69  if (in >> buffer) {
70 
71  is.clear();
72  is.str(replace(buffer, ':', ' '));
73 
74  if (is >> hour >> min >> sec) {
75 
76  object = TTimeStamp(year, month, day, hour, min, sec);
77 
78  return in;
79  }
80  }
81  }
82  }
83 
84  in.setstate(ios::failbit);
85 
86  return in;
87 }
88 
89 
90 /**
91  * Write time stamp to output stream.
92  *
93  * \param out output stream
94  * \param object time stamp
95  * \return output stream
96  */
97 inline std::ostream& operator>>(std::ostream& out, const TTimeStamp& object)
98 {
99  return out << object.AsString();
100 }
101 
102 
103 /**
104  * \file
105  *
106  * Auxiliary program to plot data from data base.
107  * \author mdejong
108  */
109 int main(int argc, char **argv)
110 {
111  using namespace std;
112  using namespace JPP;
113 
114  typedef JRange<int> JRange_t;
115 
116  string usr;
117  string pwd;
118  string cookie;
119  string outputFile;
120  string detid;
121  JRange_t runs;
122  vector<string> source;
123  JRange<TTimeStamp> UTC;
124  double Tmin_s;
125  int debug;
126 
127  try {
128 
129  JParser<> zap("Example program to plot quality data from data base.");
130 
131  zap['u'] = make_field(usr) = "";
132  zap['!'] = make_field(pwd) = "";
133  zap['C'] = make_field(cookie) = "";
134  zap['o'] = make_field(outputFile) = "monitor.root";
135  zap['D'] = make_field(detid, "detector identifier");
136  zap['R'] = make_field(runs, "run range") = JRange_t(1, JRange_t::getMaximum());
137  zap['S'] = make_field(source, "GIT versions") = getGITTags(TRegexp("v.*\\..*\\..*"), JGITTags_t::key_type("2019-04-12"));
138  zap['U'] = make_field(UTC, "UTC time range" ) = JRange<TTimeStamp>();
139  zap['T'] = make_field(Tmin_s, "minimal run duration [s]") = 60;
140  zap['d'] = make_field(debug) = 1;
141 
142  zap(argc, argv);
143  }
144  catch(const exception &error) {
145  FATAL(error.what() << endl);
146  }
147 
148 
149  int ID = -1; // integer representation of detector identifier
150 
151  vector<JRuns> data;
153  vector<JRunQuality> buffer;
154 
155  try {
156 
157  JDB::reset(usr, pwd, cookie);
158 
159  if (is_integer(detid))
160  ID = to_value<int>(detid);
161  else
162  ID = getDetector(detid);
163 
164  // runs
165 
166  try {
167 
168  ResultSet& rs = getResultSet(getTable<JRuns>(), getSelector<JRuns>(ID));
169 
170  for (JRuns parameters; rs >> parameters; ) {
171 
172  parameters.DETID = ID;
173 
174  if (UTC(parameters.getRunStartTime())) {
175  data.push_back(parameters);
176  }
177  }
178 
179  rs.Close();
180  }
181  catch(const exception& error) {}
182 
183  // data quality
184 
185  if (is_integer(detid)) {
186  detid = getDetector(to_value<int>(detid));
187  }
188 
189  for (vector<string>::const_iterator i = source.begin(); i != source.end(); ++i) {
190 
191  typedef map<string, string> data_type;
192  typedef map<int, data_type> map_type;
193 
194  map_type zmap;
195 
196  JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());
197 
198  selector.add(&JRunSummaryNumbers::SOURCE_NAME, *i);
199 
200  try {
201 
202  ResultSet& rs = getResultSet(getTable<JRunSummaryNumbers>(), selector);
203 
204  for (JRunSummaryNumbers parameters; rs >> parameters; ) {
205  zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
206  }
207  }
208  catch(const exception& error) {}
209 
210  for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
211 
212  JRunQuality quality;
213 
214  quality.GIT = *i;
215  quality.detector = ID;
216  quality.run = run->first;
217 
218  for (data_type::const_iterator p = run->second.begin(); p != run->second.end(); ++p) {
219  quality.put(p->first, p->second);
220  }
221 
222  qaqc.insert(JRun_t(ID, quality.run));
223 
224  if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
225  UTC(TTimeStamp(quality.UTCMax_s, 0))) {
226 
227  vector<JRunQuality>::iterator p = lower_bound(buffer.begin(), buffer.end(), quality);
228 
229  if (p == buffer.end() || p->run != quality.run) {
230  buffer.insert(p, quality);
231  }
232  }
233  }
234  }
235  }
236  catch(const exception& error) {
237  FATAL(error.what() << endl);
238  }
239 
240  if (buffer.empty()) {
241  FATAL("No valid QA/QC data for detector " << detid << endl);
242  }
243 
244 
245  // bins
246 
247  vector<double> X;
248 
249  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
250  X.push_back(quality->UTCMin_s);
251  X.push_back(quality->UTCMax_s);
252  }
253 
254  sort(X.begin(), X.end());
255 
256  struct Xmin {
257  Xmin(double xmin) :
258  xmin(xmin)
259  {}
260 
261  bool operator()(const double x1, const double x2)
262  {
263  return x2 - x1 <= xmin;
264  }
265 
266  double xmin;
267  };
268 
269  X.push_back(data.rbegin()->getRunStartTime()); // start of last run
270 
271  X.erase(unique(X.begin(), X.end(), Xmin(Tmin_s)), X.end());
272 
273  TH1D h0("livetime_s", NULL, X.size() - 1, X.data());
274  TH1D h1("QAQC", NULL, X.size() - 1, X.data());
275 
276  for (vector<JRuns>::const_iterator i = data.begin(); i != data.end(); ++i) {
277 
278  const JRun_t run(i->DETID,i->RUN);
279 
280  if (debug >= debug_t) {
281 
282  cout << "Run "
283  << setw(8) << i->RUN << ' '
284  << TTimeStamp((time_t) i->UNIXSTARTTIME/1000).AsString("c") << ' ';
285 
286  vector<JRunQuality>::const_iterator quality = lower_bound(buffer.begin(), buffer.end(), run);
287 
288  if (quality != buffer.end() && quality->run == i->RUN) {
289  cout << "[" << TTimeStamp((time_t) quality->UTCMin_s).AsString("c") << "," << TTimeStamp((time_t) quality->UTCMax_s).AsString("c") << "]";
290  } else {
291  cout << (qaqc.count(run) == 0 ? "missing" : "invalid") << " QA/QC data";
292  }
293 
294  cout << endl;
295  }
296 
297  double W = 1.0;
298 
299  if (! binary_search(buffer.begin(), buffer.end(), run)) {
300  W = (qaqc.count(run) == 0 ? -1.0 : 0.0);
301  }
302 
303  h1.Fill(i->getRunStartTime() + Tmin_s, W);
304  }
305 
306  JManager<string, TH1D> H1(new TH1D("%", NULL, X.size() - 1, X.data()));
307 
308  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
309 
310  const double x = 0.5 * (quality->UTCMin_s + quality->UTCMax_s);
311 
312  h0.Fill(x, 100.0 * quality->livetime_s / (quality->UTCMax_s - quality->UTCMin_s));
313 
314  H1["JDAQEvent"] -> Fill(x, quality->JDAQEvent);
315  H1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower);
316  H1["JTrigger3DMuon"] -> Fill(x, quality->JTrigger3DMuon);
317  H1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower);
318  }
319 
320 
321  Double_t W[2] = { 0.0 };
322 
323  W[0] = *X.rbegin() - *X.begin();
324 
325  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
326  W[1] += quality->livetime_s;
327  }
328 
329  NOTICE("Average data taking efficiency " << FIXED(5,1) << 100.0*W[1]/W[0] << " %." << endl);
330 
331 
332  for (TH1* p : { &h0, &h1 }) {
333  p->GetXaxis()->SetTimeDisplay(1);
334  p->GetXaxis()->SetTimeFormat(TIMESTAMP);
335  p->Sumw2(false);
336  }
337 
338  for (JManager<string, TH1D>::iterator p = H1.begin(); p != H1.end(); ++p) {
339 
340  Double_t W = 0.0;
341 
342  for (Int_t i = 1; i <= p->second->GetXaxis()->GetNbins(); ++i) {
343  p->second->SetBinContent(i, (W += p->second->GetBinContent(i)));
344  }
345 
346  p->second->GetXaxis()->SetTimeDisplay(1);
347  p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
348  p->second->Sumw2(false);
349  }
350 
351  TFile out(outputFile.c_str(), "recreate");
352 
353  out << h0 << h1 << H1;
354 
355  out.Write();
356  out.Close();
357 }
Utility class to parse command line options.
Definition: JParser.hh:1500
debug
Definition: JMessage.hh:29
int main(int argc, char *argv[])
Definition: Main.cc:15
TString replace(const TString &target, const TRegexp &regexp, const T &replacement)
Replace regular expression in input by given replacement.
Definition: JPrintResult.cc:63
std::vector< std::string > getGITTags(const TRegexp regexp, const JGITTags_t::key_type date)
Get selection of GIT tags.
Definition: JDB/JGITTags.hh:34
do rm f tmp H1
Print objects in ASCII format using ROOT dictionary.
*fatal Wrong number of arguments esac JCookie sh typeset Z DETECTOR typeset Z SOURCE_RUN typeset Z TARGET_RUN set_variable PARAMETERS_FILE $WORKDIR parameters
Definition: diff-Tuna.sh:38
then for HISTOGRAM in h0 h1
Definition: JMatrixNZ.sh:69
then let ID
Definition: JAcoustics.sh:30
Dynamic ROOT object management.
Auxiliary data structure for floating point format specification.
Definition: JManip.hh:446
string outputFile
is
Definition: JDAQCHSM.chsm:167
JDetectorsHelper getDetector
Function object for mapping serial number to object identifier of detector and vice versa...
Definition: JDBToolkit.cc:5
#define make_field(A,...)
macro to convert parameter to JParserTemplateElement object
Definition: JParser.hh:1961
#define NOTICE(A)
Definition: JMessage.hh:64
bool is_integer(const std::string &buffer)
Check if string is an integer.
Definition: JLangToolkit.hh:58
int debug
debug level
Definition: JSirene.cc:63
Auxiliary methods to convert data members or return values of member methods of a set of objects to a...
General purpose messaging.
#define FATAL(A)
Definition: JMessage.hh:67
void reset(T &value)
Reset value.
std::istream & operator>>(std::istream &in, JAANET::JHead &header)
Read header from input.
Definition: JHead.hh:1618
Auxiliary class to define a range between two values.
Utility class to parse command line options.
ResultSet & getResultSet(const std::string &query)
Get result set.
Definition: JDB.hh:269
Selection of GIT tags.
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY source JAcoustics sh $DETECTOR_ID CHECK_EXIT_CODE typeset A TRIPODS get_tripods $WORKDIR tripod txt TRIPODS for EMITTER in
Definition: JCanberra.sh:38
static const char *const TIMESTAMP
Time stamp of earliest UTC time.
int qaqc
QA/QC file descriptor.