JDataMonitor.cc

Go to the documentation of this file.

#include <string>
#include <iostream>
#include <sstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>
#include <algorithm>

#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TTimeStamp.h"

#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRuns.hh"
#include "JDB/JDBToolkit.hh"
#include "JDB/JDBSupportkit.hh"
#include "JDB/JGITTags.hh"

#include "JLang/JVectorize.hh"
#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JMathSupportkit.hh"
#include "JROOT/JManager.hh"
#include "JGizmo/JGizmoToolkit.hh"

#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"


/**
 * Read time stamp from input stream.
 *
 * Format is <tt>YYYY-MM-DD HH:MM:SS"</tt>
 *
 * \param  in         input stream
 * \param  object     time stamp
 * \return            input stream
 */
inline std::istream& operator>>(std::istream& in, TTimeStamp& object)
{
  using namespace std;
  using namespace JPP;

  string buffer;

  if (in >> buffer) {

    UInt_t year;
    UInt_t month;
    UInt_t day;

    istringstream is;

    is.str(replace(buffer, '-', ' '));

    if (is >> year >> month >> day) {

      UInt_t hour = 0;
      UInt_t min  = 0;
      UInt_t sec  = 0;

      if (in >> buffer) {

        is.clear();
        is.str(replace(buffer, ':', ' '));

        if (is >> hour >> min >> sec) {

          object = TTimeStamp(year, month, day, hour, min, sec);

          return in;
        }
      }
    }
  }

  in.setstate(ios::failbit);

  return in;
}


/**
 * Write time stamp to output stream.
 *
 * \param  out        output stream
 * \param  object     time stamp
 * \return            output stream
 */
inline std::ostream& operator>>(std::ostream& out, const TTimeStamp& object)
{
  return out << object.AsString();
}


/**
 * \file
 *
 * Auxiliary program to plot data from data base.
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;
 
  typedef JRange<int>  JRange_t;

  JServer            server;
  string             usr;
  string             pwd;
  string             cookie;
  string             outputFile;
  string             detid;
  JRange_t           runs;
  vector<string>     source;
  JRange<TTimeStamp> UTC;
  double             Tmin_s;
  int                debug;

  try {

    JParser<> zap("Example program to plot quality data from data base.");
    
    zap['s'] = make_field(server)        = getServernames();
    zap['u'] = make_field(usr)           = "";
    zap['!'] = make_field(pwd)           = "";
    zap['C'] = make_field(cookie)        = "";
    zap['o'] = make_field(outputFile)    = "monitor.root";
    zap['D'] = make_field(detid,      "detector identifier");
    zap['R'] = make_field(runs,       "run range")                  = JRange_t(1, JRange_t::getMaximum());
    zap['S'] = make_field(source,     "GIT versions")               = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
    zap['U'] = make_field(UTC,        "UTC time range" )            = JRange<TTimeStamp>();
    zap['T'] = make_field(Tmin_s,     "minimal run duration [s]")   = 60;
    zap['d'] = make_field(debug)         = 1;

    zap(argc, argv);
  }
  catch(const exception &error) {
    FATAL(error.what() << endl);
  }


  int ID = -1;        // integer representation of detector identifier

  vector<JRuns>       data;
  set   <JRun_t>      qaqc;
  vector<JRunQuality> buffer;

  try {

    JDB::reset(usr, pwd, cookie);

    if (is_integer(detid))
      ID = to_value<int>(detid);
    else
      ID = getDetector(detid);

    // runs

    try {

      NOTICE("Extracting run information from database... " << flush);
      
      ResultSet& rs  = getResultSet(getTable<JRuns>(), getSelector<JRuns>(ID));

      for (JRuns parameters; rs >> parameters; ) {

        parameters.DETID = ID;

        if (UTC(parameters.getRunStartTime())) {
          data.push_back(parameters);
        }
      }

      rs.Close();

      NOTICE("OK" << endl);
    }
    catch(const exception& error) {}

    // data quality

    if (is_integer(detid)) {
      detid = getDetector(to_value<int>(detid));
    }

    for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {

      typedef map<string, string>    data_type;
      typedef map<int, data_type>    map_type;

      map_type zmap;

      JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());

      selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);

      try {

        NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);

        ResultSet& rs  = getResultSet(getTable<JRunSummaryNumbers>(), selector);

        for (JRunSummaryNumbers parameters; rs >> parameters; ) {
          zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
        }
        
        rs.Close();

        NOTICE("OK" << endl);
      }
      catch(const exception& error) { NOTICE(endl); }

      for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {

        JRunQuality quality;

        quality.GIT      = *git;
        quality.detector = ID;
        quality.run      = run->first;

        for (data_type::const_iterator p = run->second.begin(); p != run->second.end(); ++p) {
          quality.put(p->first, p->second);
        }

        qaqc.insert(JRun_t(ID, quality.run));

        if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
            UTC(TTimeStamp(quality.UTCMax_s, 0))) {

          vector<JRunQuality>::iterator p = lower_bound(buffer.begin(), buffer.end(), quality);

          if (p == buffer.end() || p->run != quality.run) {
            buffer.insert(p, quality);
          }
        }
      }
    }
  }
  catch(const exception& error) {
    FATAL(error.what() << endl);
  }

  if (buffer.empty()) {
    FATAL("No valid QA/QC data for detector " << detid << endl);
  }


  // bins

  vector<double> X;

  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
    X.push_back(quality->UTCMin_s);
    X.push_back(quality->UTCMax_s);
  }

  sort(X.begin(), X.end());

  struct Xmin {
    Xmin(double xmin) : 
      xmin(xmin)
    {}

    bool operator()(const double x1, const double x2) 
    {
      return x2 - x1 <= xmin;
    }

    double xmin;
  };

  X.push_back(data.rbegin()->getRunStartTime());    // start of last run

  X.erase(unique(X.begin(), X.end(), Xmin(Tmin_s)), X.end());

  TH1D h0("livetime_s", NULL, X.size() - 1, X.data());
  TH1D h1("QAQC",       NULL, X.size() - 1, X.data());

  for (vector<JRuns>::const_iterator i = data.begin(); i != data.end(); ++i) {

    const JRun_t run(i->DETID,i->RUN);

    if (debug >= debug_t) {

      cout << "Run " 
           << setw(8) << i->RUN                                        << ' ' 
           << TTimeStamp((time_t) i->UNIXSTARTTIME/1000).AsString("c") << ' ';

      vector<JRunQuality>::const_iterator quality = lower_bound(buffer.begin(), buffer.end(), run);
          
      if (quality != buffer.end() && quality->run == i->RUN) {
        cout << "[" << TTimeStamp((time_t) quality->UTCMin_s).AsString("c") << "," << TTimeStamp((time_t) quality->UTCMax_s).AsString("c") << "]";
      } else {
        cout << (qaqc.count(run) == 0 ? "missing" : "invalid") << " QA/QC data";
      }

      cout << endl;
    }

    double W = 1.0;

    if (! binary_search(buffer.begin(), buffer.end(), run)) {
      W = (qaqc.count(run) == 0 ? -1.0 : 0.0);
    }

    h1.Fill(i->getRunStartTime() + Tmin_s, W);
  }

  JManager<string, TH1D> H1(new TH1D("H[%]", NULL, X.size() - 1, X.data()));  
  JManager<string, TH1D> R1(new TH1D("R[%]", NULL, X.size() - 1, X.data()));  

  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {

    const double x = 0.5 * (quality->UTCMin_s + quality->UTCMax_s);

    h0.Fill(x, 100.0 * quality->livetime_s / (quality->UTCMax_s - quality->UTCMin_s));

    H1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent);
    H1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower);
    H1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon);
    H1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower);

    if (quality->livetime_s > 0.0) {
      R1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent        / quality->livetime_s);
      R1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower / quality->livetime_s);
      R1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon   / quality->livetime_s);
      R1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower / quality->livetime_s);
    }
  }


  Double_t W[2] = { 0.0 };

  W[0] = *X.rbegin() - *X.begin();

  for (vector<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
    W[1] += quality->livetime_s;
  }

  NOTICE("Average data taking efficiency  " << FIXED(5,1) << 100.0*W[1]/W[0] << " %." << endl);


  for (TH1* p : { &h0, &h1 }) {
    p->GetXaxis()->SetTimeDisplay(1);
    p->GetXaxis()->SetTimeFormat(TIMESTAMP);
    p->Sumw2(false);
  }

  for (JManager<string, TH1D>::iterator p = H1.begin(); p != H1.end(); ++p) {

    Double_t W = 0.0;
    
    for (Int_t i = 1; i <= p->second->GetXaxis()->GetNbins(); ++i) {
      p->second->SetBinContent(i, (W += p->second->GetBinContent(i)));
    }
    
    p->second->GetXaxis()->SetTimeDisplay(1);
    p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
    p->second->Sumw2(false);
  }

  for (JManager<string, TH1D>::iterator p = R1.begin(); p != R1.end(); ++p) {

    p->second->GetXaxis()->SetTimeDisplay(1);
    p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
    p->second->Sumw2(false);
  }

  TFile out(outputFile.c_str(), "recreate");

  out << h0 << h1 << H1 << R1;

  out.Write();
  out.Close();
}