JDataQuality.cc File Reference

Main program to evaluate quality data from data base. More...

#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <vector>
#include <set>
#include <map>
#include "TROOT.h"
#include "TFile.h"
#include "TH1D.h"
#include "TF1.h"
#include "TString.h"
#include "TNtuple.h"
#include "TRegexp.h"
#include "JDB/JDB.hh"
#include "JDB/JSelector.hh"
#include "JDB/JSelectorSupportkit.hh"
#include "JDB/JRunQuality.hh"
#include "JDB/JRunsetups.hh"
#include "JDB/JDBToolkit.hh"
#include "JDB/JGITTags.hh"
#include "JLang/JLangToolkit.hh"
#include "JTools/JRange.hh"
#include "JROOT/JRootClass.hh"
#include "JROOT/JRootPrinter.hh"
#include "JROOT/JRootToolkit.hh"
#include "JROOT/JRootStreamObjectOutput.hh"
#include "JROOT/JRootDictionary.hh"
#include "JROOT/JManager.hh"
#include "JSupport/JMeta.hh"
#include "JSupport/JFilenameSupportkit.hh"
#include "Jeep/JeepToolkit.hh"
#include "Jeep/JComment.hh"
#include "Jeep/JParser.hh"
#include "Jeep/JMessage.hh"

Go to the source code of this file.

Functions
int	main (int argc, char **argv)

Detailed Description

Main program to evaluate quality data from data base.

The evaluation of the data quality is based on the following procedure.
For each data taking run, the script JQAQC.sh is executed which produces a set of values. The values are subsequently uploaded into the database in table "runsummarynumbers". This table is a general purpose table for any value related to some data taking run. With this application, the QA/QC data are consistently downloaded from the database; evaluated according some formula and tested against a validity range. The formula and validity range are referred to as a condition and can be specified with option -Q. Each condition has an optional weight (default is 1). The normalised total weight is used to define an overall quality, ranging from 0 = worst to 1 = best. There is also the possibility to specify a veto (option -V), which is similar to a condition but only the number of vetoes is counted. The overall quality and the number of vetoes can be used to accept or reject a data taking run.

By default, a list of the run numbers, the overall weight and the number of vetoes is printed. If a ROOT file is specified for the output (i.e. filename extension is JSUPPORT::ROOT_FILE_FORMAT), a file is written which contains a histogram for each condition and each veto; some summary histograms and a ROOT n-tuple. If an ASCII file is specified for the output (i.e. filename extension is JSUPPORT::ASCII_FILE_FORMAT), a text file is written which can subsequently be used as input file.

Author

mdejong

Definition in file JDataQuality.cc.

Function Documentation

int main	(	int	argc,
		char **	argv
	)

Definition at line 123 of file JDataQuality.cc.

{
  using namespace std;
  using namespace JPP;

  typedef JRange<int>           JRange_t; 
  typedef vector<JCondition>    JCondition_t;

  JServer         server;
  string          usr;
  string          pwd;
  string          cookie;
  string          inputFile;
  string          outputFile;
  string          detid;
  JRange_t        runs;
  vector<string>  source;
  JCondition_t    condition;
  JCondition_t    veto;
  TRegexp         regexp(".");
  JSelector       selection;
  int             debug;

  try {

    JParser<> zap("Main program to evaluate quality data from data base."\
                  "\nThe condition and veto can be any TFormula compatible expression involving QA/QC parameters (see e.g. JQAQC.sh -h).");
    
    zap['s'] = make_field(server)        = getServernames();
    zap['u'] = make_field(usr)           = "";
    zap['!'] = make_field(pwd)           = "";
    zap['C'] = make_field(cookie)        = "";
    zap['f'] = make_field(inputFile,  "Optional input file instead of database.")       = "";
    zap['o'] = make_field(outputFile, "ROOT file with histograms and n-tuple or ASCII file with QA/QC data.") = "quality.root";
    zap['D'] = make_field(detid)         = "";
    zap['R'] = make_field(runs,       "Run range")                                      = JRange_t(1, JRange_t::getMaximum());
    zap['S'] = make_field(source,     "GIT versions")                                   = getGITTags(TRegexp("v[0-9]*\\.[0-9]*\\.[0-9]*$"), JGITTags_t::key_type("2019-04-12"));
    zap['Q'] = make_field(condition,  "User defined conditions")                        = JPARSER::initialised();
    zap['V'] = make_field(veto,       "User defined vetos.")                            = JPARSER::initialised();
    zap['r'] = make_field(regexp,     "TRegexp for selection of run setup names.")      = JPARSER::initialised();
    zap['@'] = make_field(selection)     = JPARSER::initialised();
    zap['d'] = make_field(debug,      "Debug level")                                    = 1;

    zap(argc, argv);
  }
  catch(const exception &error) {
    FATAL(error.what() << endl);
  }


  double W = 0.0;
  
  for (JCondition_t::const_iterator i = condition.begin(); i != condition.end(); ++i) {
    W += i->weight;
  }

  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {
    if (W <= 0.0) {
      FATAL("Invalid total weight: " << W << endl);
    }
  }


  JRunsetups       setups;                  // run -> value
  set<JRunQuality> buffer;                  // storage


  if (inputFile == "") {                    // read data from database

    ASSERT(detid != "");

    int ID = -1;                            // integer representation of detector identifier

    try {

      JDB::reset(usr, pwd, cookie);

      if (is_integer(detid))
        ID = to_value<int>(detid);
      else
        ID = getDetector(detid);

      if (is_integer(detid)) {
        detid = getDetector(to_value<int>(detid));
      }

      // run setups

      selection += getSelector<JRuns>(ID);

      NOTICE("Extracting run information from database... " << flush);

      ResultSet& rs  = getResultSet(getTable<JRuns>(), selection);

      for (JRuns parameters; rs >> parameters; ) {
        if (TString(parameters.RUNSETUPNAME.c_str()).Contains(regexp)) {
          setups.put(parameters);
        }
      }

      rs.Close();

      NOTICE("OK" << endl);

      // run summary data

      for (vector<string>::const_iterator git = source.begin(); git != source.end(); ++git) {

        typedef map<string, string>    data_type;
        typedef map<int, data_type>    map_type;

        map_type zmap;

        JSelector selector = getSelector<JRunSummaryNumbers>(detid, runs.getLowerLimit(), runs.getUpperLimit());

        selector.add(&JRunSummaryNumbers::SOURCE_NAME, *git);

        try {

          NOTICE("Extracting run summmary information with source " << *git << " from database... " << flush);

          ResultSet& rs  = getResultSet(getTable<JRunSummaryNumbers>(), selector);

          for (JRunSummaryNumbers parameters; rs >> parameters; ) {
            if (setups.has(parameters.RUN)) {
              zmap[parameters.RUN].insert(make_pair(parameters.PARAMETER_NAME, parameters.DATA_VALUE));
            }
          }

          rs.Close();

          NOTICE("OK" << endl);
        }
        catch(const exception& error) { NOTICE(endl); }

        for (map_type::const_iterator run = zmap.begin(); run != zmap.end(); ++run) {

          JRunQuality quality;
          
          quality.GIT      = *git;
          quality.detector = ID;
          quality.run      = run->first;

          for (data_type::const_iterator i = run->second.begin(); i != run->second.end(); ++i) {
            quality.put(i->first, i->second);
          }

          buffer.insert(quality);           // only insert if absent
        }
      }
    }
    catch(const exception& error) {
      FATAL(error.what() << endl);
    }

  } else {                                  // read data from file

    ifstream in(inputFile.c_str());

    JComment comment;

    in >> comment;

    int nc = 0;                             // number of columns in file
    int nd = getListOfDataMembers<JRunQuality>().size();

    string header;

    if (getline(in, header)) {

      istringstream is(header);
    
      nc = distance(istream_iterator<string>(is), istream_iterator<string>());
    }

    JRootReader reader(in, JEquationParameters(), JDBDictionary::getInstance());

    for (JRunQuality quality; reader.getObject(quality); ) {

      buffer.insert(quality);

      if (nc > nd) {                        // read optional data

        double setup;

        reader.getObject(setup);

        setups[quality.run] = setup;
      }
    }

    in.close();
  }


  if (buffer.empty()) {
    FATAL("No data." << endl);
  }


  if (getFilenameExtension(outputFile) == ROOT_FILE_FORMAT) {

    runs = JRange_t(buffer.begin()->run, buffer.rbegin()->run);

    TH1D h0("h0", NULL, 1000, 0.0, 1.01);
    TH1D h1("h1", NULL, condition.size(), -0.5, condition.size() + 0.5);

    h0.SetMinimum(0.0);
    h0.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));

    h1.SetMinimum(0.0);
    h1.SetMaximum(1.1 * (runs.getUpperLimit() - runs.getLowerLimit()));

    const Double_t xmin = runs.getLowerLimit() - 0.5;
    const Double_t xmax = runs.getUpperLimit() + 0.5;

    JManager<TString, TH1D> H1(new TH1D("%", NULL, runs.getLength() + 1, xmin, xmax));

    // show validity range

    for (JCondition_t::const_iterator i = condition.begin(); i != condition.end();++i) {
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("C " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
    }

    for (JCondition_t::const_iterator i = veto.begin(); i != veto.end();++i) {
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":upper"), MAKE_CSTRING(i->range.getUpperLimit()), xmin, xmax));
      H1[i->formula]->GetListOfFunctions()->Add(new TF1(MAKE_CSTRING("V " << i->formula << ":lower"), MAKE_CSTRING(i->range.getLowerLimit()), xmin, xmax));
    }

    ostringstream os;                       // N-tuple format

    os << "run";

    for (size_t i = 0; i != condition.size(); ++i) {
      os << ":" << (char) ('a' + i);
    }

    os << ":Q:V:R";

    TNtuple n1("n1", "quality", os.str().c_str());


    // process data

    for (set<JRunQuality>::const_iterator quality = buffer.begin(); quality != buffer.end(); ++quality) {

      vector<Float_t> tuple(1, (Float_t) quality->run);

      double w = 0.0;

      for (size_t i = 0; i != condition.size(); ++i) {

        const JCondition& ps = condition[i];
        const double      y  = getResult(ps.formula, *quality);

        DEBUG(ps.formula << ' ' << y << endl);

        tuple.push_back((Float_t) y);

        TH1D* p = H1[ps.formula];

        p->SetBinContent(p->FindBin((double) quality->run), y);
        p->SetBinError  (p->FindBin((double) quality->run), 0.0);

        if (ps.range(y)) {
          w += ps.weight;
        }

        h1.AddBinContent(i + 1, ps.range(y) ? 1.0 : 0.0);
      }

      const double Q = w/W;

      tuple.push_back((Float_t) Q);

      int V = 0;

      for (size_t i = 0; i != veto.size(); ++i) {

        const JCondition& ps = veto[i];
        const double      y  = getResult(ps.formula, *quality);

        TH1D* p = H1[MAKE_STRING("VETO[" << ps.formula << "]")];

        p->SetBinContent(p->FindBin((double) quality->run), ps.range(y) ? 0.0 : 1.0);
        p->SetBinError  (p->FindBin((double) quality->run), 0.0);

        if (!ps.range(y)) {
          ++V;
        }
      }

      tuple.push_back((Float_t) V);
      tuple.push_back((Float_t) setups.get(quality->run));

      h0.Fill(Q);
      n1.Fill(tuple.data());

      cout << setw(8) << quality->run << ' ' << FIXED(5,3) << Q << ' ' << setw(2) << V << endl;
    }

    double w = 0.0;

    for (Int_t i = 0; i <= h0.GetXaxis()->GetNbins(); ++i) {
      h0.SetBinContent(i, (w += h0.GetBinContent(i)));
    }

    // store results

    TFile out(outputFile.c_str(), "recreate");

    out << h0 << h1 << n1 << H1;
    
    out.Write();
    out.Close();
  }


  if (getFilenameExtension(outputFile) == ASCII_FILE_FORMAT) {

    // store data

    ofstream out(outputFile);

    JComment comment;

    comment.add(JMeta(argc, argv));

    out << comment;

    // header

    for (const auto& i : getListOfDataMembers<JRunQuality>()) {
      out << i->GetName() << ' ';
    }

    out << "setup" << endl;

    JRootWriter writer(out, JEquationParameters(), JDBDictionary::getInstance());

    for (set<JRunQuality>::const_iterator i = buffer.begin(); i != buffer.end(); ++i) {

      writer.putObject(*i);

      out << ' ';

      writer.putObject(setups.get(i->run));

      out << endl;
    }

    out.close();
  }
}