8 # ------------------------------------------------------------------------------------------
10 # Utility script to evaluate gSeaGen reweightings
12 # ------------------------------------------------------------------------------------------
15 if [ -z $JPP_DIR ];
then
16 echo "Variable JPP_DIR undefined."
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 source $JPP_DIR/software/scripts/qlib.sh
28 set_variable+ BATCH_PROCESSING REWEIGHTING_BATCH_PROCESSING 1
35 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHTING_TEST_TYPE1D_PARAMETERS
"0.05 UWCHI2/NDF"
37 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHTING_TEST_TYPE2D_PARAMETERS
"0.05 UWCHI2/NDF"
41 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHTING_N_TRIGGERED_HITS_RANGE
"0 $((2**31))"
42 set_variable: CONTAINMENT_VOLUME REWEIGHTING_CONTAINMENT_VOLUME
"0.0 0.0 $((2**31)) -$((2**31)) $((2**31))"
43 set_variable: COSTH_RANGE REWEIGHTING_COSTH_RANGE
"-1.0 1.0"
44 set_variable: ENERGY_RANGE REWEIGHTING_ENERGY_RANGE
"0.0 $((2.0**31))"
55 usage "$script <detector file> <input file>+ <reweighting file> <oscillation probability table> [default oscillation parameter file [oscillation parameter grid file [working directory]]]\n" \
56 "Note that if more than one input file is given all optional parameters need to be specified."
79 # Recursive function to process parameter files
81 # \param prefix prefix
82 # \param label neutrino label
83 # \param formula reweighting formula
84 # \param parameters_file parameter file
86 process_parameters() {
91 local parameters_file=$4
97 if [[ ${line:0:1} == \# ]];
then continue;
fi
99 pars=( $(
echo $line | cut -
d' ' -
f1-) )
103 for ((
i=1;
i<=${#pars}; ++
i ));
do
104 local parstr=
"${parstr} p$((i-1))=${pars[$i]};"
107 local reweighting_input=$(printf
"\t%s=\"${formula};${parstr}\"" ${(z)labels})
110 process_parameters
"${prefix}${reweighting_input}" "$@"
112 printf
'%b\n' "${prefix}${reweighting_input}"
114 done < ${parameters_file}
118 # Function to write job identifier to file
120 # \param output_file output file
126 if (( ${+BATCH_PROCESSING} ));
then
128 if [[ -z $EXEC_FILE ]];
then fatal
"write_jobID(): Variable EXEC_FILE undefined";
fi
130 if ( is_CCNikhef );
then
131 local jobid=$(qstat -
u $USER -
f |
grep -B1
"${EXEC_FILE:t:r}.${USER}" |
grep -oP
"Job Id: [0-9]+" | tail -1 | cut -
d' ' -
f3)
133 local jobid=$(ps |
grep "${EXEC_FILE:t:r}" | tail -1 | tr -s
' ' | cut -
d' ' -f2)
136 echo ${jobid} >> ${output_file}
141 # Set neutrino labels
153 # Initialise local batch processor
155 if [[ ${BATCH_PROCESSING} == 1 ]];
then
157 if ( is_CCNikhef );
then
159 echo "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${QUEUE_NAME}..."
161 if [[ ${SCRATCHDIR} ==
"/tmp" ]];
then
162 fatal
"Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
167 fatal
"No batch-processing option available yet for CC-Lyon."
171 echo "Submitting reweighting and histogram comparison jobs to local batch queue: ${QUEUE_NAME}..."
173 $JPP_DIR/examples/scripts/mkqueue.sh ${QUEUE_NAME} ${QUEUE_SIZE}
"start"
185 echo "----------------------------"
187 echo "----------------------------"
189 echo "Remove the above regular files? (y/n)"
191 read REMOVE_MC_MERGE_FILES
193 if [[ ${REMOVE_MC_MERGE_FILES} ==
"y" || ${REMOVE_MC_MERGE_FILES} ==
"Y" ||
194 ${REMOVE_MC_MERGE_FILES} ==
"yes" || ${REMOVE_MC_MERGE_FILES} ==
"Yes" || ${REMOVE_MC_MERGE_FILES} ==
"YES" ]];
then
203 # Divide into DAQ- and MC-files
205 echo "Dividing into DAQ- and MC-files..."
210 rm -
f ${INPUT_MC_FILELIST} ${INPUT_DAQ_FILELIST}
211 touch ${INPUT_MC_FILELIST} ${INPUT_DAQ_FILELIST}
215 if [[
"$(JPrintHeader -f ${INPUT_FILE} -k simul --)" !=
" 0" ]];
then # MC-
file
231 if (( $N_MC_FILES == 0 ));
then
236 # Initialize job ID file
249 $JPP_DIR/examples/JAAnet/JSortEvt \
250 -
f ${INPUT_MC_FILELIST} \
252 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
255 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
261 if (( N_DAQ_FILES > 0 ));
then
268 ${SCRATCHDIR}/split_events.job\%.sh 1
275 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/
null
296 \${JPP_DIR}/examples/JAAnet/JSplitEvt \
298 -
o "${INPUT_REFERENCE_FILE} ${INPUT_REWEIGHTING_FILE}" \
303 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
305 ${SUBMIT_JOB+
rm -
f ${EXEC_FILE}}
310 set_array INPUT_REWEIGHTING_FILES \
317 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR}}}
325 # Process reference files and create histogram comparison steering file
329 set_variable REFERENCE_POSTFIT_FILE_SCALED ${REFERENCE_POSTFIT_FILE:
r}.scaled.root
331 if ( ! reuse_file ${REFERENCE_POSTFIT_FILE} );
then
339 ${SCRATCHDIR}/${REFERENCE_POSTFIT_FILE:t:
r}.job\%.sh 1
346 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/
null
351 -
f "${INPUT_REFERENCE_FILES[*]}" \\
352 -
o ${REFERENCE_POSTFIT_FILE} \\
356 -@atmospheric=
"${(k)NULABELS}" \\
357 -%energyRange=
"${ENERGY_RANGE}" \\
358 -%coszenithRange=
"${COSTH_RANGE}" \\
359 -%containmentVolume=
"${CONTAINMENT_VOLUME}" \\
360 -%triggeredHitsRange=
"${N_TRIGGERED_HITS_RANGE}" \\
362 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
365 -
f ${REFERENCE_POSTFIT_FILE}:. \\
366 -
o ${REFERENCE_POSTFIT_FILE_SCALED} \\
368 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
370 mv ${REFERENCE_POSTFIT_FILE_SCALED} ${REFERENCE_POSTFIT_FILE}
373 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
375 set_array NAMES \$(JPrintName -
f ${REFERENCE_POSTFIT_FILE}:.)
377 for NAME
in \${NAMES[@]};
do
381 -
f ${REFERENCE_POSTFIT_FILE}:^\${NAME}\$ \
382 -
F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
385 if [[ \${NAME} !=
"META" && \${NAME} !=
"TObject" ]];
then
388 warning
"Cannot perform comparison test for histogram \${NAME} with >2 dimensions; skip."
390 echo "\${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
392 echo "\${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
403 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} $PREREQUISITES[@]}}
411 # Process MC-files with standard weights
414 set_variable DEFAULT_COMPARISON_FILE ${DEFAULT_POSTFIT_FILE:
r}.comparison.root
416 if ( ! reuse_file ${DEFAULT_COMPARISON_FILE} );
then
418 echo "Submitting default histogram comparison jobs"
421 ${SCRATCHDIR}/${DEFAULT_COMPARISON_FILE:t:
r}.job\%.sh 1
428 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/
null
432 unset DEFAULT_POSTFIT_FILES
434 for ((
i=1;
i<=${#INPUT_REWEIGHTING_FILES}; ++
i ));
do
438 set_variable POSTFIT_FILE_SCALED ${POSTFIT_FILE:
r}.scaled.root
440 rm -
f ${POSTFIT_FILE} ${COMPARISON_FILE}
447 -
o ${POSTFIT_FILE} \\
451 -@atmospheric=
"${(k)NULABELS}" \\
452 -%energyRange=
"${ENERGY_RANGE}" \\
453 -%coszenithRange=
"${COSTH_RANGE}" \\
454 -%containmentVolume=
"${CONTAINMENT_VOLUME}" \\
455 -%triggeredHitsRange=
"${N_TRIGGERED_HITS_RANGE}" \\
457 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE}}
460 -
f ${POSTFIT_FILE}:. \\
461 -
o ${POSTFIT_FILE_SCALED} \\
463 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
465 mv ${POSTFIT_FILE_SCALED} ${POSTFIT_FILE}
468 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
471 set_array DEFAULT_POSTFIT_FILES \
472 ${DEFAULT_POSTFIT_FILES[@]} ${POSTFIT_FILE}
477 hadd -
v 0 -
f ${DEFAULT_POSTFIT_FILE} ${DEFAULT_POSTFIT_FILES[@]}
482 -
a ${REFERENCE_POSTFIT_FILE} \\
483 -b ${DEFAULT_POSTFIT_FILE} \\
484 -s ${STEERING_FILE} \\
485 -
o ${DEFAULT_COMPARISON_FILE} \\
486 -t ${DEFAULT_COMPARISON_FILE:
r}.txt \\
487 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
490 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
498 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${REFERENCE_POSTFIT_JOBID}}}
508 # Parse reweighting inputs
515 for TYPE in ${(z)TYPES};
do
517 ${LABELS[@]} ${NULABELS[$TYPE]}
521 ${NUTYPES[@]} ${TYPES[@]}
523 set_array PARINPUTS \
524 ${PARINPUTS[@]}
"${LABELS}" "${FORMULA}" "${PARAMETER_FILE}"
528 set_variable PARINPUTS_FILE ${SCRATCHDIR}/parametersets.${RANDOM}.txt
530 process_parameters
' ' ${PARINPUTS[@]} > ${PARINPUTS_FILE}
533 # Compile oscillation parameters files
535 if [[ ! -z $OSCPARAMETERS_GRID_FILE ]];
then
537 $JPP_DIR/examples/JOscProb/JWriteOscParametersGrid \
539 -
o ${
WORKDIR}/oscillation_parameters.\%.txt \
547 ls ${
WORKDIR}/oscillation_parameters.*.txt >> ${OSCPARAMETERS_FILE_LIST}
550 # Process MC-files with redefined weights
563 echo "Submitting reweighting job ${n}"
565 IFS=$
'\t' read -
r -
A PARINPUTS <<<
"${LINE}"
567 while read -
u 4 OSCPARAMETERS_FILE;
do
570 set_variable REWEIGHTED_COMPARISON_FILE ${REWEIGHTED_POSTFIT_FILE:
r}.comparison.root
573 ${SCRATCHDIR}/reweighting.job\%.sh ${JOB_SIZE}
575 if (( ${+START_JOB} ));
then
582 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/
null
586 for ((
j=1;
j<=${#INPUT_REWEIGHTING_FILES}; ++
j ));
do
591 set_variable POSTFIT_FILE_SCALED ${POSTFIT_FILE:
r}.scaled.root
593 set_array POSTFIT_FILES \
594 ${POSTFIT_FILES[@]} ${POSTFIT_FILE}
598 set_array PRIMARIES \\
599 \$(JPrintHeader -
f ${
INPUT_FILE} -
k primary -- | cut -
d'=' -f2)
601 set_array PRIMARIES \\
608 if [[
"${NUTYPES[@]}" =~
"\${PRIMARY}" ]];
then
614 if (( \${+DO_REWEIGHTING} ));
then
618 JEvtReweightGSeaGen \\
620 -
o ${REWEIGHTED_FILE} \\
622 -@ ${OSCPARAMETERS_FILE} \\
623 $(printf
' -#%s \\\n' ${PARINPUTS[@]})
624 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
627 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
632 -
f ${REWEIGHTED_FILE} \\
633 -
o ${POSTFIT_FILE} \\
636 -%energyRange=
"${ENERGY_RANGE}" \\
637 -%coszenithRange=
"${COSTH_RANGE}" \\
638 -%containmentVolume=
"${CONTAINMENT_VOLUME}" \\
639 -%triggeredHitsRange=
"${N_TRIGGERED_HITS_RANGE}" \\
641 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
644 -
f ${POSTFIT_FILE}:. \\
645 -
o ${POSTFIT_FILE_SCALED} \\
647 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
649 mv ${POSTFIT_FILE_SCALED} ${POSTFIT_FILE}
652 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
654 else # This
file does not correspond to any of the given primaries
657 ln -fv ${
INPUT_FILE:
r}.postfit.root ${POSTFIT_FILE}
660 rm -
f ${REWEIGHTED_FILE}
666 hadd -
v 0 -
f ${REWEIGHTED_POSTFIT_FILE} ${POSTFIT_FILES[@]}
671 -
a ${REFERENCE_POSTFIT_FILE} \\
672 -b ${REWEIGHTED_POSTFIT_FILE} \\
673 -s ${STEERING_FILE} \\
674 -
o ${REWEIGHTED_COMPARISON_FILE} \\
675 -t ${REWEIGHTED_COMPARISON_FILE:
r}.txt \\
676 -
d ${
DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
679 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
681 ${SUBMIT_JOB+#
rm -
f ${EXEC_FILE}}
684 if (( ${+SUBMIT_JOB} ));
then
689 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${
PREREQUISITES[@]}}}
696 echo "${REWEIGHTED_COMPARISON_FILE:r}.txt\t${OSCPARAMETERS_FILE}\t${PARINPUTS[*]}" >> ${META_FILE}
700 done 4< ${OSCPARAMETERS_FILE_LIST}
702 done 3< ${PARINPUTS_FILE}
704 rm -
f ${PARINPUTS_FILE}
then rm f $STEERING_FILE touch $STEERING_FILE set_array PREREQUISITES(cat $JOBID_FILE) configure_job $
then fatal No hydrophone data file $HYDROPHONE_TXT fi sort gr k
then usage $script< detector file >< inputfiles > n Note that if more than one input file is specified
std::istream & read(std::istream &in, JTestSummary &summary, const char delimiter= ' ')
Read test summary.
then usage $script[< detector identifier >< run range >]< QA/QCfile > nExample script to produce data quality plots nWhen a detector identifier and run range are data are downloaded from the database nand subsequently stored in the given QA QC file
&set_variable OSCPROBTABLE
do set_variable N_EVENTS(JPrintTree-f ${MC_MERGE_FILE}-@"type = Evt"-k number_of_entries) set_variable N1 $((N_EVENTS/2)) set_variable N2 $((N_EVENTS-N1)) set_variable INPUT_REFERENCE_FILE $
then JLigier sh continue fi cat
o $QUALITY_ROOT d $DEBUG!CHECK_EXIT_CODE JPlot1D f
then usage $script< input file > nFor possible input files
&set_variable PARAMETER_FILE
do set_variable MC_MERGE_FILE
&set_variable DEFAULT_OSCPARAMETERS_FILE
then fatal Wrong number of arguments fi JConvertDetectorFormat a o
const JPolynome f1(1.0, 2.0, 3.0)
Function.
then timer_start(JRandomTimesliceWriter-a ${DETECTOR}-o ${PIPE}-n ${NUMBER_OF_SLICES}-P ${PMT_FILE}-d ${DEBUG}-S ${SEED}-B"$BACKGROUND_HZ"-N"$RECYCLING"--!&&for((i=0;$i!=10;++i));do;echo 'EOF' > $PIPE;usleep 100000;done &&rm-f $PIPE)&JTimesliceReprocessor-a $
then set_array INPUT_REFERENCE_FILES(cat ${INPUT_DAQ_FILELIST}) set_array INPUT_REWEIGHTING_FILES $
do set_variable BINFRAC(JPrintResult-f ${REFERENCE_POSTFIT_FILE}:^\${NAME}\$-F"GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))"-O"%f"--) if[[\$
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR if do_usage *then usage $script[(input file)+] fi set_variable DEBUG set_variable WORKDIR TMPDIR
do grep(chi2\[.\]\|gradient\|string:\)" $INPUT_FILE > $WORKDIR/stage.txt awk 'BEGIN
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable DEBUG set_variable WORKDIR
then fatal No MC input files specified fi set_variable JOBID_FILE
&set_variable OUTPUT_FILE
then fatal The output file must have the wildcard in the name
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR source $JPP_DIR software scripts qlib sh set_variable IFS0
then usage $script< input file >[option[primary[working directory]]] nWhere option can be LINE
part1 root set_variable INPUT_REWEIGHTING_FILE
then JCookie sh JDataQuality D $DETECTOR_ID R
then JMuonMCEvt f $INPUT_FILE o $INTERMEDIATE_FILE d
then fatal The output file must have the wildcard in the e g root fi eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY JAcoustics sh $DETECTOR_ID source JAcousticsToolkit sh CHECK_EXIT_CODE typeset A EMITTERS get_tripods $WORKDIR tripod txt EMITTERS get_transmitters $WORKDIR transmitter txt EMITTERS for EMITTER in
&set_variable OSCPARAMETERS_GRID_FILE
then fatal Wrong number of arguments fi set_variable ARCHIVE $argv[1] set_variable VERSION $argv[2] set_variable DIR $argv[3] source JAcousticsToolkit sh set_variable DETECTOR $DIR $ACOUSTICS_DETECTOR if[[!-f $DETECTOR]]
then echo Submitting reweighting and histogram comparison jobs to nikhef stbc batch queue
do sput $STRING $FLOOR sget MODULE INPUT_FILES
double f3(const double x, const double y, const double z)
3D function.
fi set_array MC_MERGE_FILES(find ${WORKDIR}-maxdepth 1-name"*.merge.root") if(($
source $JPP_DIR setenv csh $JPP_DIR &dev null eval JShellParser o a A
then elif[["$OPTION"=="count"]]
esac $JPP_BIN JLogger sh $LOGGER until pgrep JGetMessage</dev/null > dev null
set_variable REWEIGHTING_INPUTS_FILE
#define DEBUG(A)
Message macros.