JQAQC.sh

Go to the documentation of this file.

#!/bin/zsh
#
# \author mdejong
#
script=${0##*/}
version=1.0
doxygen=JDATABASE::JRunQuality

if [ -z $JPP_DIR ]; then
    echo "Variable JPP_DIR undefined."
    exit
fi

source $JPP_DIR/setenv.sh $JPP_DIR >& /dev/null

# Auxiliary script to produce QA/QC data.
#
# The output data are uploaded to the database with a tool that is part of the km3pipe package.
# The locally produced data as well as the uploaded data can be viewed with applications/scripts JDataMonitor[.sh] and JDataQuality[.sh].
#
# Note that if the list of values is extended, the include file JRunQuality.hh should accordingly be updated.

set_variable:  DEBUG              QAQC_DEBUG    0
set_variable:  VERSION            QAQC_VERSION  process_B
set_variable:  WORKDIR            QAQC_WORKDIR  ${TMPDIR:-/tmp}/.$$

TRIGGER_PARAMETERS=(
    trigger3DMuon.enabled
    trigger3DShower.enabled
    triggerMXShower.enabled
    triggerNB.enabled
d    writeL0.prescale
    writeL1.prescale
    writeL2.prescale
    writeSN.prescale)

TRIGGERS=(`JPrintTriggerBits`)

DAQ=(
    JDAQTimeslice
    JDAQTimesliceL0
    JDAQTimesliceL1
    JDAQTimesliceL2
    JDAQTimesliceSN
    JDAQSummaryslice
    JDAQEvent)

typeset -a PARAMETERS

PARAMETERS=(
    GIT JPP UUID 
    detector run livetime_s UTCMin_s UTCMax_s
    ${TRIGGER_PARAMETERS[*]}
    ${DAQ[*]}
    JTriggerReprocessor
    ${TRIGGERS[*]}
    in_sync out_sync DAQ WR HRV FIFO
    PMTs MEAN_Rate_Hz RMS_Rate_Hz
    Acoustics AHRS
    in_usync out_usync
    event_duration)

if do_usage $*; then
    usage \
        "$script <archive> (input file)+ <QA/QC file>"                                    \
        "\nThe archive should correspond to the GIT repository \"auxiliary_data/jpp/\""   \
        "\nAuxiliary script to produce the following QA/QC data:\n"                       \
        ${PARAMETERS//\.*}                                                                \
        "\nSee also: $JPP_PAGES/ Search \"JRunQuality\""
fi

if (( $# < 2 )); then
    fatal "Wrong number of arguments."
fi

set_variable   ARCHIVE            ${argv[1]}
set_array      INPUT_FILES        ${argv[2,-2]}
set_variable   QAQC_FILE          ${argv[-1]:a}

mkdir -p ${WORKDIR}
pushd    ${WORKDIR}

set_variable   ACOUSTICS_DEBUG    ${DEBUG}
set_variable   COMPASS_DEBUG      ${DEBUG}
set_variable   DAQX               daq.detx                               # detector used during data taking
set_variable   DETECTOR           detector.datx                          # fixed nomenclature

JCookie.sh
CHECK_EXIT_CODE

if [[ ! -d $ARCHIVE ]]; then
    fatal "No archive $ARCHIVE"
fi

if [[ ! -s $QAQC_FILE ]]; then
    echo ${PARAMETERS//\.*/}                                             > $QAQC_FILE
fi

for INPUT_FILE in ${INPUT_FILES[*]}; do

    # determine values

    JPrintMeta -v |& awk '/version/ { print $2 }'                        | read     GIT_VERSION
    CHECK_EXIT_CODE

    JPrintMeta -f ${INPUT_FILE} -k GIT                                   | read     DAQ_VERSION

    echo ${(s/./)DAQ_VERSION}                                            | read     DAQ_MAJOR DAQ_MINOR DAQ_PATCH

    if [[ -z $DAQ_VERSION ]]; then
        JPrintMeta -f ${INPUT_FILE} -k SVN                               | read     DAQ_VERSION
    fi

    getUUID -f ${INPUT_FILE} -d 0                                        | read     UUID
    CHECK_EXIT_CODE

    JPrintDAQHeader -f ${INPUT_FILE} -d 0                                | read -A  DAQ_HEADER
    CHECK_EXIT_CODE

    set_variable  DETECTOR_ID    ${DAQ_HEADER[1]}
    set_variable  RUN            ${DAQ_HEADER[2]}

    JDetectorDB -D ${DETECTOR_ID} -r ${RUN} -o ${DAQX} -d ${DEBUG}       # install detector file as-is for JTriggerReprocessor
    CHECK_EXIT_CODE

    if [[ "$DAQ_MAJOR" != [0-9][0-9]* ]] || (( $DAQ_MAJOR <= 12 )); then # Jpp-version dependence
        JEditDetector \
            -a ${DAQX}                          \
            -P "-1 -1 set UDP_COUNTER_DISABLE"  \
            -P "-1 -1 set UDP_TRAILER_DISABLE"  \
            -o ${DAQX}                          \
            -d ${DEBUG}
    fi

    JAHRSCalibration.sh     ${DETECTOR_ID}                               # install AHRS calibration file
    CHECK_EXIT_CODE

    getLivetime     -f ${INPUT_FILE} -d 0                                | read -A  LIVETIME_S
    CHECK_EXIT_CODE

    getUTC          -f ${INPUT_FILE} -d 0                                | read -A  UTC_S
    CHECK_EXIT_CODE

    {{ JTriggerMonitor -f ${INPUT_FILE} -d ${DEBUG} -Q 3 3>&1 1>&4 }     | read -A  TRIGGER_MONITOR } 4>&1
    CHECK_EXIT_CODE

    set_variable  OUTPUT_FILE   KM3NeT_${(l:8::0::0:)DETECTOR_ID}_${(l:8::0::0:)RUN}_reprocessor.root

    JTriggerReprocessor \
        -a ${DAQX}                              \
        -f ${INPUT_FILE}                        \
        -o ${OUTPUT_FILE}                       \
        -U                                      \
        -C -\.\*                                \
        -d ${DEBUG}                                  
    CHECK_EXIT_CODE

    JPrintTree -f ${OUTPUT_FILE} |& awk '/JDAQEvent/ { print $3 }'       | read     TRIGGER_REPROCESSOR

    {{ JTurbot \
        -f ${INPUT_FILE}                \
        -a ${DAQX}                      \
        -o /dev/null                    \
        -d ${DEBUG}                     \
        -Q 3 3>&1 1>&4 }                                                 | read -A  TURBOT } 4>&1
    CHECK_EXIT_CODE

    {{ JSummaryMonitor -f ${INPUT_FILE} -d ${DEBUG} -Q 3 3>&1 1>&4 }     | read -A  SUMMARY_MONITOR } 4>&1
    CHECK_EXIT_CODE

    archive-get-calibration.sh   ${ARCHIVE}  ${DETECTOR_ID}  ${VERSION}  # detector and auxiliary calibrations
    CHECK_EXIT_CODE

    JAcousticsEventBuilder.sh   ${DETECTOR}     ${RUN}

    if (( $? == 0 )); then

        set_variable  OUTPUT_FILE    KM3NeT_${(l:8::0::0:)DETECTOR_ID}_${(l:8::0::0:)RUN}_event.root

        JPrintTree -f ${OUTPUT_FILE} |& awk '/JACOUSTICS::JEvent/ { print $3 }'  | read -A  NUMBER_OF_ACOUSTICS_EVENTS  
    else
        NUMBER_OF_ACOUSTICS_EVENTS=(0)
    fi

    JAHRS.sh                    ${DETECTOR_ID}  ${RUN}

    if (( $? == 0 )); then

        set_variable  OUTPUT_FILE    KM3NeT_${(l:8::0::0:)DETECTOR_ID}_${(l:8::0::0:)RUN}_ahrs.root

        {{ JAHRSMonitor \
            -a ${DETECTOR}                      \
            -f ${OUTPUT_FILE}                   \
            -c ahrs_calibration.txt             \
            -q 0.2                              \
            -d ${DEBUG}                         \
            -Q 3 3>&1 1>&4 }                                             | read -A  NUMBER_OF_AHRS_EVENTS } 4>&1
    else
        NUMBER_OF_AHRS_EVENTS=(0)
    fi

    set_variable  OUTPUT_FILE    KM3NeT_${(l:8::0::0:)DETECTOR_ID}_${(l:8::0::0:)RUN}_trigger.root

    JTriggerProcessor \
        -a ${DETECTOR}                                    \
        -f ${INPUT_FILE}                                  \
        -C JDAQTimesliceSN                                \
        -@ "trigger3DMuon.enabled         =  1"           \
        -@ "trigger3DMuon.numberOfHits    =  3"           \
        -@ "trigger3DMuon.numberOfModules =  3"           \
        -@ "trigger3DMuon.gridAngle_deg   = 10"           \
        -@ "trigger3DMuon.TMaxExtra_ns    = 500.0e3"      \
        -@ "TMaxEvent_ns    = 500.0e3"                    \
        -@ "TMaxLocal_ns    = 20.0"                       \
        -@ "L2.numberOfHits = 4"                          \
        -@ "L2.TMaxLocal_ns = 20.0"                       \
        -@ "L2.ctMin        = 0.0"                        \
        -o ${OUTPUT_FILE}                                 \
        -d ${DEBUG}
    CHECK_EXIT_CODE

    {{ JSquid \
        -a ${DETECTOR}                                        \
        -f ${OUTPUT_FILE}                                     \
        -o /dev/null                                          \
        -d ${DEBUG}                                           \
        -Q 3 3>&1 1>&4 }                                                 | read -A  SQUID } 4>&1

    {{ JMermaid \
        -a ${DETECTOR}                                        \
        -f ${INPUT_FILE}                                      \
        -o /dev/null                                          \
        -d ${DEBUG}                                           \
        -Q 3 3>&1 1>&4 }                                                 | read -A  MERMAID } 4>&1

    # print values

    printf " %12s"   ${GIT_VERSION}                                      >> $QAQC_FILE
    printf " %12s"   ${DAQ_VERSION:-\?}                                  >> $QAQC_FILE
    printf " %s"     ${UUID:-\?}                                         >> $QAQC_FILE
    printf " %8d"    ${DAQ_HEADER[1,2]}                                  >> $QAQC_FILE
    printf " %9.1f"  ${LIVETIME_S[1]}                                    >> $QAQC_FILE
    printf " %12.1f" ${UTC_S[*]}                                         >> $QAQC_FILE

    for KEY in ${TRIGGER_PARAMETERS[*]}; do

        JPrintTriggerParameters -f ${INPUT_FILE} -k $KEY -d 0                   | read VALUE

        printf " %4d" ${VALUE:-0}                                        >> $QAQC_FILE
    done

    for KEY in ${DAQ[*]}; do

        JPrintTree -f ${INPUT_FILE} |& eval awk \'/${KEY}/ \{ print \$3 \}\'    | read VALUE

        printf " %8d" ${VALUE:-0}                                        >> $QAQC_FILE
    done

    printf " %8d"   ${TRIGGER_REPROCESSOR:-0}                            >> $QAQC_FILE
    printf " %8d"   ${TRIGGER_MONITOR[*]}                                >> $QAQC_FILE
    printf " %4d"   ${TURBOT[1,2]}                                       >> $QAQC_FILE
    printf " %7.5f" ${SUMMARY_MONITOR[1,4]}                              >> $QAQC_FILE
    printf " %8.1f" ${SUMMARY_MONITOR[5]}                                >> $QAQC_FILE
    printf " %7.0f" ${SUMMARY_MONITOR[6,7]}                              >> $QAQC_FILE
    printf " %4d"   ${NUMBER_OF_ACOUSTICS_EVENTS[1]}                     >> $QAQC_FILE
    printf " %8.3f" ${NUMBER_OF_AHRS_EVENTS[1]}                          >> $QAQC_FILE
    printf " %4d"   ${SQUID[1,2]}                                        >> $QAQC_FILE
    printf " %8d"   ${MERMAID[1]}                                        >> $QAQC_FILE
    printf "\n"                                                          >> $QAQC_FILE
done

popd

rm -rf ${WORKDIR}