JDataMonitor.cc

Go to the documentation of this file.

#include <string>
#include <iostream>
#include <sstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>
#include <algorithm>

#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TTimeStamp.h"

#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRuns.hh"
#include "JDB/JDBToolkit.hh"
#include "JDB/JDBSupportkit.hh"

#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JMathSupportkit.hh"
#include "JROOT/JASCIIFileStreamer.hh"
#include "JROOT/JManager.hh"
#include "JGizmo/JGizmoToolkit.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JFilenameSupportkit.hh"

#include "JDataQuality/JGITTags.hh"

#include "Jeep/JeepToolkit.hh"
#include "Jeep/JComment.hh"
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"


/**
 * Read time stamp from input stream.
 *
 * Format is <tt>YYYY-MM-DD HH:MM:SS"</tt>
 *
 * \param  in         input stream
 * \param  object     time stamp
 * \return            input stream
 */
inline std::istream& operator>>(std::istream& in, TTimeStamp& object)
{
  using namespace std;
  using namespace JPP;

  string buffer;

  if (in >> buffer) {

    UInt_t year;
    UInt_t month;
    UInt_t day;

    istringstream is;

    is.str(replace(buffer, '-', ' '));

    if (is >> year >> month >> day) {

      UInt_t hour = 0;
      UInt_t min  = 0;
      UInt_t sec  = 0;

      if (in >> buffer) {

        is.clear();
        is.str(replace(buffer, ':', ' '));

        if (is >> hour >> min >> sec) {

          object = TTimeStamp(year, month, day, hour, min, sec);

          return in;
        }
      }
    }
  }

  in.setstate(ios::failbit);

  return in;
}


/**
 * Write time stamp to output stream.
 *
 * \param  out        output stream
 * \param  object     time stamp
 * \return            output stream
 */
inline std::ostream& operator>>(std::ostream& out, const TTimeStamp& object)
{
  return out << object.AsString();
}


/**
 * \file
 *
 * Auxiliary program to plot data from data base.
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;
 
  typedef JRange<TTimeStamp>  JRange_t;

  JServer         server;
  string          usr;
  string          pwd;
  string          cookie;
  string          inputFile;
  string          outputFile;
  string          detid;
  JRange_t        UTC;
  vector<string>  source;
  double          Tmin_s;
  int             debug;

  try {

    JParser<> zap("Example program to plot quality data from data base.");
    
    zap['s'] = make_field(server)        = getServernames();
    zap['u'] = make_field(usr)           = "";
    zap['!'] = make_field(pwd)           = "";
    zap['C'] = make_field(cookie)        = "";
    zap['f'] = make_field(inputFile,  "Optional input file instead of database.")       = "";
    zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "monitor.root";
    zap['D'] = make_field(detid,      "detector identifier");
    zap['S'] = make_field(source,     "GIT versions")               = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
    zap['U'] = make_field(UTC,        "UTC time range" )            = JRange<TTimeStamp>();
    zap['T'] = make_field(Tmin_s,     "minimal run duration [s]")   = 60;
    zap['d'] = make_field(debug)         = 1;

    zap(argc, argv);
  }
  catch(const exception &error) {
    FATAL(error.what() << endl);
  }


  set<JRuns>       runs;
  set<JRunQuality> buffer;

  try {

    JDB::reset(usr, pwd, cookie);

    const int ID = getDetector<int   >(detid);
    detid        = getDetector<string>(detid);

    // runs

    NOTICE("Extracting run information from database... " << flush);
      
    ResultSet& rs  = getResultSet(getTable<JRuns>(), getSelector<JRuns>(ID));

    for (JRuns parameters; rs >> parameters; ) {

      parameters.DETID = ID;

      if (UTC(parameters.getRunStartTime())) {
        runs.insert(parameters);
      }
    }

    rs.Close();

    NOTICE("OK" << endl);

    if (runs.empty()) {
      FATAL("No runs for detector " << detid << endl);
    }

    NOTICE("Run range " << runs.begin()->RUN << ' ' << runs.rbegin()->RUN << endl);

    if (inputFile == "") {                    // read data from database

      // run summary data

      for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {

        typedef map<string, string>    data_type;
        typedef map<int, data_type>    map_type;

        map_type zmap;

        JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.begin()->RUN, runs.rbegin()->RUN);

        selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);

        try {

          NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);

          ResultSet& rs  = getResultSet(getTable<JRunSummaryNumbers>(), selector);

          for (JRunSummaryNumbers parameters; rs >> parameters; ) {
            zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
          }
        
          rs.Close();

          NOTICE("OK" << endl);
        }
        catch(const exception& error) { NOTICE(endl); }

        for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {

          JRunQuality quality;

          quality.GIT      = *git;
          quality.detector = ID;
          quality.run      = run->first;

          for (data_type::const_iterator p = run->second.begin(); p != run->second.end(); ++p) {
            quality.put(p->first, p->second);
          }

          if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
              UTC(TTimeStamp(quality.UTCMax_s, 0))) {
            buffer.insert(quality);
          }
        }
      }

    } else {                                  // read data from file

      JASCIIFileReader<JRunQuality> in(inputFile.c_str(), JDBDictionary::getInstance());

      JComment comment;

      in >> comment;

      for (JRunQuality quality; in >> quality; ) {
        if (UTC(TTimeStamp(quality.UTCMin_s, 0)) &&
            UTC(TTimeStamp(quality.UTCMax_s, 0))) {
          buffer.insert(quality);
        }
      }

      in.close();
    }
  }
  catch(const exception& error) {
    FATAL(error.what() << endl);
  }


  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {

    // bins

    vector<double> X;

    X.push_back(runs. begin()->getRunStartTime());    // start of first run

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
      X.push_back(quality->UTCMin_s);
      X.push_back(quality->UTCMax_s);
    }

    X.push_back(runs.rbegin()->getRunStartTime());    // start of last run

    sort(X.begin(), X.end());

    struct Xmin {
      Xmin(double xmin) : 
        xmin(xmin)
      {}

      bool operator()(const double x1, const double x2) 
      {
        return x2 - x1 <= xmin;
      }

      double xmin;
    };

    X.erase(unique(X.begin(), X.end(), Xmin(Tmin_s)), X.end());

    TH1D h0("livetime_s", NULL, X.size() - 1, X.data());
    TH1D h1("QAQC",       NULL, X.size() - 1, X.data());

    for (set<JRuns>::const_iterator i = runs.begin(); i != runs.end(); ++i) {

      set<JRunQuality>::const_iterator quality = buffer.find(JRunQuality(i->DETID, i->RUN));

      if (debug >= debug_t) {

        cout << "Run " 
             << setw(8) << i->RUN                                        << ' ' 
             << TTimeStamp((time_t) i->UNIXSTARTTIME/1000).AsString("c") << ' ';

        if (quality != buffer.end())
          cout << "[" << TTimeStamp((time_t) quality->UTCMin_s).AsString("c") << "," << TTimeStamp((time_t) quality->UTCMax_s).AsString("c") << "]";
        else
          cout << "missing QA/QC data";

        cout << endl;
      }

      h1.Fill(i->getRunStartTime() + Tmin_s, (quality != buffer.end() ? 1.0 : -1.0));
    }

    JManager<string, TH1D> H1(new TH1D("H[%]", NULL, X.size() - 1, X.data()));  
    JManager<string, TH1D> R1(new TH1D("R[%]", NULL, X.size() - 1, X.data()));  

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {

      const double x = 0.5 * (quality->UTCMin_s + quality->UTCMax_s);

      h0.Fill(x, 100.0 * quality->livetime_s / (quality->UTCMax_s - quality->UTCMin_s));

      H1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent);
      H1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower);
      H1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon);
      H1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower);

      if (quality->livetime_s > 0.0) {
        R1["JDAQEvent"]        -> Fill(x, quality->JDAQEvent        / quality->livetime_s);
        R1["JTrigger3DShower"] -> Fill(x, quality->JTrigger3DShower / quality->livetime_s);
        R1["JTrigger3DMuon"]   -> Fill(x, quality->JTrigger3DMuon   / quality->livetime_s);
        R1["JTriggerMXShower"] -> Fill(x, quality->JTriggerMXShower / quality->livetime_s);
      }
    }


    Double_t W[2] = { 0.0 };

    W[0] = *X.rbegin() - *X.begin();

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
      W[1] += quality->livetime_s;
    }

    NOTICE("Average data taking efficiency  " << FIXED(5,1) << 100.0*W[1]/W[0] << " %." << endl);


    for (TH1* p : { &h0, &h1 }) {
      p->GetXaxis()->SetTimeDisplay(1);
      p->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->Sumw2(false);
    }

    for (JManager<string, TH1D>::iterator p = H1.begin(); p != H1.end(); ++p) {

      Double_t W = 0.0;
    
      for (Int_t i = 1; i <= p->second->GetXaxis()->GetNbins(); ++i) {
        p->second->SetBinContent(i, (W += p->second->GetBinContent(i)));
      }
    
      p->second->GetXaxis()->SetTimeDisplay(1);
      p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->second->Sumw2(false);
    }

    for (JManager<string, TH1D>::iterator p = R1.begin(); p != R1.end(); ++p) {

      p->second->GetXaxis()->SetTimeDisplay(1);
      p->second->GetXaxis()->SetTimeFormat(TIMESTAMP);
      p->second->Sumw2(false);
    }

    TFile out(outputFile.c_str(), "recreate");

    out << h0 << h1 << H1 << R1;

    out.Write();
    out.Close();
  }


  if (getFilenameExtension(outputFile) == ASCII_FILE_FORMAT) {

    // store data

    JASCIIFileWriter<JRunQuality> out(outputFile.c_str(), JDBDictionary::getInstance());

    out.setf(ios::fixed);

    JComment comment;

    comment.add(JMeta(argc, argv));

    out << comment;

    for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
      out << *i << endl;
    }

    out.close();
  }
}