JDataMonitor.cc

Go to the documentation of this file.

#include <string>
#include <iostream>
#include <sstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>
#include <algorithm>
 
#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TTimeStamp.h"
 
#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRuns.hh"
#include "JDB/JDBToolkit.hh"
#include "JDB/JDBSupportkit.hh"
 
#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JMathSupportkit.hh"
#include "JROOT/JASCIIFileStreamer.hh"
#include "JROOT/JManager.hh"
#include "JGizmo/JGizmoToolkit.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JFilenameSupportkit.hh"
 
#include "JDataQuality/JGITTags.hh"
 
#include "Jeep/JeepToolkit.hh"
#include "Jeep/JComment.hh"
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"
 
 
/**
 * Read time stamp from input stream.
 *
 * Format is <tt>YYYY-MM-DD HH:MM:SS"</tt>
 *
 * \param  in         input stream
 * \param  object     time stamp
 * \return            input stream
 */
inline std::istream& operator>>(std::istream& in, TTimeStamp& object)
{
  using namespace std;
  using namespace JPP;
 
  string buffer;
 
  if (in >> buffer) {
 
    UInt_t year;
    UInt_t month;
    UInt_t day;
 
    istringstream is;
 
    is.str(replace(buffer, '-', ' '));
 
    if (is >> year >> month >> day) {
 
      UInt_t hour = 0;
      UInt_t min  = 0;
      UInt_t sec  = 0;
 
      if (in >> buffer) {
 
        is.clear();
        is.str(replace(buffer, ':', ' '));
 
        if (is >> hour >> min >> sec) {
 
          object = TTimeStamp(year, month, day, hour, min, sec);
 
          return in;
        }
      }
    }
  }
 
  in.setstate(ios::failbit);
 
  return in;
}
 
 
/**
 * Write time stamp to output stream.
 *
 * \param  out        output stream
 * \param  object     time stamp
 * \return            output stream
 */
inline std::ostream& operator>>(std::ostream& out, const TTimeStamp& object)
{
  return out << object.AsString();
}
 
 
/**
 * \file
 *
 * Auxiliary program to plot data from data base or input file.
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;
 
  typedef JRange<TTimeStamp>  JRange_t;
 
  JServer         server;
  string          usr;
  string          pwd;
  string          cookie;
  string          inputFile;
  string          outputFile;
  string          detid;
  JRange_t        UTC;
  vector<string>  source;
  double          Tmin_s;
  int             debug;
 
  try {
 
    JParser<> zap("Example program to plot quality data from data base or input file.");
    
    zap['s'] = make_field(server)        = getServernames();
    zap['u'] = make_field(usr)           = "";
    zap['!'] = make_field(pwd)           = "";
    zap['C'] = make_field(cookie)        = "";
    zap['f'] = make_field(inputFile,  "Optional input file instead of database.")       = "";
    zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "monitor.root";
    zap['D'] = make_field(detid,      "detector identifier");
    zap['S'] = make_field(source,     "GIT versions")               = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
    zap['U'] = make_field(UTC,        "UTC time range" )            = JRange<TTimeStamp>();
    zap['T'] = make_field(Tmin_s,     "minimal run duration [s]")   = 60;
    zap['d'] = make_field(debug)         = 1;
 
    zap(argc, argv);
  }
  catch(const exception &error) {
    FATAL(error.what() << endl);
  }
 
 
  set<JRuns>       runs;
  set<JRunQuality> buffer;
 
  try {
 
    JDB::reset(usr, pwd, cookie);
 
    const int ID = getDetector<int   >(detid);
    detid        = getDetector<string>(detid);
 
    // runs
 
    NOTICE("Extracting run information from database... " << flush);
      
    ResultSet& rs  = getResultSet(getTable<JRuns>(), getSelector<JRuns>(ID));
 
    for (JRuns parameters; rs >> parameters; ) {
 
      parameters.DETID = ID;
 
      if (UTC(parameters.getRunStartTime())) {
        runs.insert(parameters);
      }
    }
 
    rs.Close();
 
    NOTICE("OK" << endl);
 
    if (runs.empty()) {
      FATAL("No runs for detector " << detid << endl);
    }
 
    NOTICE("Run range " << runs.begin()->RUN << ' ' << runs.rbegin()->RUN << endl);
 
    if (inputFile == "") {                    // read data from database
 
      // run summary data
 
      for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {
 
        typedef map<string, string>    data_type;
        typedef map<int, data_type>    map_type;
 
        map_type zmap;
 
        JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.begin()->RUN, runs.rbegin()->RUN);
 
        selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);
 
        try {
 
          NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);
 
          ResultSet& rs  = getResultSet(getTable<JRunSummaryNumbers>(), selector);
 
          for (JRunSummaryNumbers parameters; rs >> parameters; ) {
            zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
          }
        
          rs.Close();
 
          NOTICE("OK" << endl);
        }
        catch(const exception& error) { NOTICE(endl); }
 
        for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {
 
          JRunQuality quality;
 
          quality.GIT      = *git;
          quality.detector = ID;
          quality.run      = run->first;
 
          for (data_type::const_iterator p = run->second.begin(); p != run->second.end(); ++p) {
            quality.put(p->first, p->second);
          }
 
          if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
              UTC(TTimeStamp(quality.UTCMax_s, 0))) {
            buffer.insert(quality);
          }
        }
      }
 
    } else {                                  // read data from file
 
      JASCIIFileReader<JRunQuality> in(inputFile.c_str(), JDBDictionary::getInstance());
 
      JComment comment;
 
      in >> comment;
 
      for (JRunQuality quality; in >> quality; ) {
        if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
            UTC(TTimeStamp(quality.UTCMax_s, 0))) {
          buffer.insert(quality);
        }
      }
 
      in.close();
    }
  }
  catch(const exception& error) {
    FATAL(error.what() << endl);
  }
 
 
  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {
 
    // bins
 
    vector<double> X;
 
    X.push_back(runs. begin()->getRunStartTime());    // start of first run
 
    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
      X.push_back(quality->UTCMin_s);
      X.push_back(quality->UTCMax_s);
    }
 
    X.push_back(runs.rbegin()->getRunStartTime());    // start of last run
 
    sort(X.begin(), X.end());
 
    struct Xmin {
      Xmin(double xmin) : 
        xmin(xmin)
      {}
 
      bool operator()(const double x1, const double x2) 
      {
        return x2 - x1 <= xmin;
      }
 
      double xmin;
    };
 
    X.erase(unique(X.begin(), X.end(), Xmin(Tmin_s)), X.end());
 
    TH1D h0("livetime_s", NULL, X.size() - 1, X.data());
    TH1D h1("QAQC",       NULL, X.size() - 1, X.data());
 
    for (set<JRuns>::const_iterator i = runs.begin(); i != runs.end(); ++i) {
 
      set<JRunQuality>::const_iterator quality = buffer.find(JRunQuality(i->DETID, i->RUN));
 
      if (debug >= debug_t) {
 
        cout << "Run " 
             << setw(8) << i->RUN                                        << ' ' 
             << TTimeStamp((time_t) i->UNIXSTARTTIME/1000).AsString("c") << ' ';
 
        if (quality != buffer.end())
          cout << "[" << TTimeStamp((time_t) quality->UTCMin_s).AsString("c") << "," << TTimeStamp((time_t) quality->UTCMax_s).AsString("c") << "]";
        else
          cout << "missing QA/QC data";
 
        cout << endl;
      }
 
      h1.Fill(i->getRunStartTime() + Tmin_s, (quality != buffer.end() ? 1.0 : -1.0));
    }
 
    JManager<string, TH1D> H1(new TH1D("H[%]", NULL, X.size() - 1, X.data()));  
    JManager<string, TH1D> R1(new TH1D("R[%]", NULL, X.size() - 1, X.data()));  
 
    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
 
      const double x = 0.5 * (quality->UTCMin_s + quality->UTCMax_s);
 
      h0.Fill(x, 100.0 * quality->livetime_s / (quality->UTCMax_s - quality->UTCMin_s));
 
      H1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent);
      H1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower);
      H1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon);
      H1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower);
 
      if (quality->livetime_s > 0.0) {
        R1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent        / quality->livetime_s);
        R1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower / quality->livetime_s);
        R1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon   / quality->livetime_s);
        R1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower / quality->livetime_s);
      }
    }
 
 
    Double_t W[2] = { 0.0 };
 
    W[0] = *X.rbegin() - *X.begin();
 
    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
      W[1] += quality->livetime_s;
    }
 
    NOTICE("Average data taking efficiency  " << FIXED(5,1) << 100.0*W[1]/W[0] << " %." << endl);
 
 
    for (TH1* p : { &h0, &h1 }) {
      p->GetXaxis()->SetTimeDisplay(1);
      p->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->Sumw2(false);
    }
 
    for (JManager<string, TH1D>::iterator p = H1.begin(); p != H1.end(); ++p) {
 
      Double_t W = 0.0;
    
      for (Int_t i = 1; i <= p->second->GetXaxis()->GetNbins(); ++i) {
        p->second->SetBinContent(i, (W += p->second->GetBinContent(i)));
      }
    
      p->second->GetXaxis()->SetTimeDisplay(1);
      p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->second->Sumw2(false);
    }
 
    for (JManager<string, TH1D>::iterator p = R1.begin(); p != R1.end(); ++p) {
 
      p->second->GetXaxis()->SetTimeDisplay(1);
      p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->second->Sumw2(false);
    }
 
    TFile out(outputFile.c_str(), "recreate");
 
    out << h0 << h1 << H1 << R1;
 
    out.Write();
    out.Close();
  }
 
 
  if (getFilenameExtension(outputFile) == ASCII_FILE_FORMAT) {
 
    // store data
 
    JASCIIFileWriter<JRunQuality> out(outputFile.c_str(), JDBDictionary::getInstance());
 
    out.setf(ios::fixed);
 
    JComment comment;
 
    comment.add(JMeta(argc, argv));
 
    out << comment;
 
    for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
      out << *i << endl;
    }
 
    out.close();
  }
}