JDataQuality.cc

Go to the documentation of this file.

#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>

#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TF1.h"
#include "TString.h"
#include "TNtuple.h"
#include "TRegexp.h"

#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRunsetups.hh"
#include "JDB/JDBToolkit.hh"

#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootClass.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JRootStreamObjectOutput.hh"
#include "JROOT/JRootDictionary.hh"
#include "JROOT/JASCIIFileStreamer.hh"
#include "JROOT/JManager.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JFilenameSupportkit.hh"

#include "JDataQuality/JGITTags.hh"

#include "Jeep/JeepToolkit.hh"
#include "Jeep/JComment.hh"
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"

namespace {

  /**
   * Auxiliary data structure for run quality evaluation.
   *
   * A condition is composed of a ROOT TFormula compatible expression and a validity range for the result.
   */
  struct JCondition {

    static const char SEPARATOR = ';';

    /**
     * Read selection from input stream.
     *
     * \param  in           input stream
     * \param  object       selection
     * \return              input stream
     */
    friend inline std::istream& operator>>(std::istream& in, JCondition& object)
    {
      std::getline(in, object.formula, SEPARATOR);

      object.range  = JTOOLS::JRange<double>();
      object.weight = 1.0;

      if ((in >> object.range) && ! (in >> object.weight)) {
        in.clear();
      }

      return in;
    }


    /**
     * Write selection to output stream.
     *
     * \param  out          output stream
     * \param  object       selection
     * \return              output stream
     */
    friend inline std::ostream& operator<<(std::ostream& out, const JCondition& object)
    {
      out << object.formula << SEPARATOR << object.range << ' ' << object.weight << std::endl;

      return out;
    }

    std::string            formula;
    JTOOLS::JRange<double> range;
    double                 weight;
  };
}


/**
 * \file
 *
 * Main program to evaluate quality data from data base.
 *
 * The evaluation of the data quality is based on the following procedure.\n
 * For each data taking run, the script JQAQC.sh is executed which produces a set of values. 
 * The values are subsequently uploaded into the database in table "runsummarynumbers". 
 * This table is a general purpose table for any value related to some data taking run.
 * With this application,
 * the QA/QC data are consistently downloaded from the database;
 * evaluated according some formula and
 * tested against a validity range.
 * The underlying data structure is JDATABASE::JRunQuality.
 *
 * The formula and validity range are referred to as a condition and can be specified with option -Q.
 * Each condition has an optional weight (default is 1).
 * The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best.
 * There is also the possibility to specify a veto (option -V),
 * which is similar to a condition but only the number of vetoes is counted.
 * The overall quality and the number of vetoes can be used to accept or reject a data taking run.
 *
 * By default, a list of the run numbers, the overall weight and the number of vetoes is printed.
 * If a ROOT file is specified for the output (i.e.\ filename extension is JSUPPORT::ROOT_FILE_FORMAT),
 * a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple.
 * If an ASCII file is specified for the output (i.e.\ filename extension is JSUPPORT::ASCII_FILE_FORMAT),
 * a text file is written which can subsequently be used as input file.
 *
 * \author mdejong
 */
int main(int argc, char **argv)
{
  using namespace std;
  using namespace JPP;

  typedef JRange<int>           JRange_t; 
  typedef vector<JCondition>    JCondition_t;

  JServer         server;
  string          usr;
  string          pwd;
  string          cookie;
  string          inputFile;
  string          outputFile;
  string          detid;
  JRange_t        runs;
  vector<string>  source;
  JCondition_t    condition;
  JCondition_t    veto;
  TRegexp         regexp(".");
  JSelector       selection;
  int             debug;

  try {

    JParser<> zap("Main program to evaluate quality data from data base."\
                  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
    
    zap['s'] = make_field(server)        = getServernames();
    zap['u'] = make_field(usr)           = "";
    zap['!'] = make_field(pwd)           = "";
    zap['C'] = make_field(cookie)        = "";
    zap['f'] = make_field(inputFile,  "Optional input file instead of database.")       = "";
    zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "quality.root";
    zap['D'] = make_field(detid)         = "";
    zap['R'] = make_field(runs,       "Run range")                                      = JRange_t(1, JRange_t::getMaximum());
    zap['S'] = make_field(source,     "GIT versions")                                   = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
    zap['Q'] = make_field(condition,  "User defined conditions")                        = JPARSER::initialised();
    zap['V'] = make_field(veto,       "User defined vetos.")                            = JPARSER::initialised();
    zap['r'] = make_field(regexp,     "TRegexp for selection of run setup names.")      = JPARSER::initialised();
    zap['@'] = make_field(selection)     = JPARSER::initialised();
    zap['d'] = make_field(debug,      "Debug level")                                    = 1;

    zap(argc, argv);
  }
  catch(const exception &error) {
    FATAL(error.what() << endl);
  }


  double W = 0.0;
  
  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
    W += i->weight;
  }

  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {
    if (W <= 0.0) {
      FATAL("Invalid total weight: " << W << endl);
    }
  }


  JRunsetups       setups;                  // run -> value
  set<JRunQuality> buffer;                  // storage


  if (inputFile == "") {                    // read data from database

    ASSERT(detid != "");

    try {

      JDB::reset(usr, pwd, cookie);
      
      const int ID = getDetector<int>   (detid);
      detid        = getDetector<string>(detid);

      // run setups

      selection += getSelector<JRuns>(ID);

      NOTICE("Extracting run information from database... " << flush);

      ResultSet& rs  = getResultSet(getTable<JRuns>(), selection);

      for (JRuns parameters; rs >> parameters; ) {
        if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
          setups.put(parameters);
        }
      }

      rs.Close();

      NOTICE("OK" << endl);

      // run summary data

      for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {

        typedef map<string, string>    data_type;
        typedef map<int, data_type>    map_type;

        map_type zmap;

        JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());

        selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);

        try {

          NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);

          ResultSet& rs  = getResultSet(getTable<JRunSummaryNumbers>(), selector);

          for (JRunSummaryNumbers parameters; rs >> parameters; ) {
            if (setups.has(parameters.RUN)) {
              zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
            }
          }

          rs.Close();

          NOTICE("OK" << endl);
        }
        catch(const exception& error) { NOTICE(endl); }

        for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {

          JRunQuality quality;
          
          quality.GIT      = *git;
          quality.detector = ID;
          quality.run      = run->first;
          quality.name     = setups[run->first].name;
          quality.value    = setups[run->first].value;

          for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
            quality.put(i->first, i->second);
          }

          quality.name     = replace(quality.name, ' ', '_');

          buffer.insert(quality);           // only insert if absent
        }
      }
    }
    catch(const exception& error) {
      FATAL(error.what() << endl);
    }

  } else {                                  // read data from file

    if (detid != "" && !is_integer(detid)) {
      WARNING("Detector identifier \"" << detid << "\" discarded." << endl);
    }

    const int ID = (is_integer(detid) ? to_value<int>(detid) : -1);

    JASCIIFileReader<JRunQuality> in(inputFile.c_str(), JDBDictionary::getInstance());

    JComment comment;

    in >> comment;

    for (JRunQuality quality; in >> quality; ) {
      if (ID == -1 || ID == quality.detector) {
        if (runs(quality.run)) {
          buffer.insert(quality);
        }
      }
    }

    in.close();
  }


  if (buffer.empty()) {
    FATAL("No data." << endl);
  }

  if (debug >= debug_t) {

    JRootWriter writer(cout, JEquationParameters(), JDBDictionary::getInstance());

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {
      writer.put(*quality);
    }
  } 


  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {

    runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);

    TH1D h0("h0", NULL, 1000, 0.0, 1.01);
    TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() - 0.5);

    for (size_t i = 0; i != condition.size(); ++i) {
      h1.GetXaxis()->SetBinLabel(i+1, condition[i].formula.c_str());
    }

    h0.SetMinimum(0.0);
    h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit() + 1));

    h1.SetMinimum(0.0);
    h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit() + 1));

    const Double_t xmin = runs.getLowerLimit() - 0.5;
    const Double_t xmax = runs.getUpperLimit() + 0.5;

    JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));

    // show validity range

    for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
    }

    for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
    }

    ostringstream os;                       // N-tuple format

    os << "run";

    for (size_t i = 0; i != condition.size(); ++i) {
      os << ":" << (char) ('a' + i);
    }

    os << ":Q:V:R";

    TNtuple n1("n1", "quality", os.str().c_str());


    // process data

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {

      vector<Float_t> tuple(1, (Float_t) quality->run);

      double w = 0.0;

      for (size_t i = 0; i != condition.size(); ++i) {

        const JCondition& ps = condition[i];
        const double      y  = getResult(ps.formula, *quality);

        DEBUG(ps.formula << ' ' << y << endl);

        tuple.push_back((Float_t) y);

        TH1D* p = H1[ps.formula];

        p->SetBinContent(p->FindBin((double) quality->run), y);
        p->SetBinError  (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());

        if (ps.range(y)) {
          w += ps.weight;
        }

        h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
      }

      const double Q = w/W;

      tuple.push_back((Float_t) Q);

      int V = 0;

      for (size_t i = 0; i != veto.size(); ++i) {

        const JCondition& ps = veto[i];
        const double      y  = getResult(ps.formula, *quality);
        
        DEBUG(ps.formula << ' ' << y << endl);

        TH1D* p = NULL;

        p = H1[ps.formula];

        p->SetBinContent(p->FindBin((double) quality->run), y);
        p->SetBinError  (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());

        p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];

        p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
        p->SetBinError  (p->FindBin((double) quality->run), numeric_limits<double>::epsilon());

        if (!ps.range(y)) {
          ++V;
        }
      }

      tuple.push_back((Float_t) V);
      tuple.push_back((Float_t) setups.get(quality->run));

      h0.Fill(Q);
      n1.Fill(tuple.data());

      cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
    }

    double w = 0.0;

    for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
      h0.SetBinContent(i, (w += h0.GetBinContent(i)));
    }

    // store results

    TFile out(outputFile.c_str(), "recreate");

    out << h0 << h1 << n1 << H1;
    
    out.Write();
    out.Close();
  }


  if (getFilenameExtension(outputFile) == ASCII_FILE_FORMAT) {

    // store data

    JASCIIFileWriter<JRunQuality> out(outputFile.c_str(), JDBDictionary::getInstance());

    out.setf(ios::fixed);

    JComment comment;

    comment.add(JMeta(argc, argv));

    out << comment;

    for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {
      out << *i << endl;
    }

    out.close();
  }
}