8 # ------------------------------------------------------------------------------------------
10 # Utility script to evaluate gSeaGen reweightings
12 # ------------------------------------------------------------------------------------------
15 if [ -z $JPP_DIR ]; then
16 echo "Variable JPP_DIR undefined."
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 source $JPP_DIR/software/scripts/qlib.sh
24 set_variable IFS0 ${IFS}
26 set_variable: DEBUG REWEIGHTING_DEBUG 2
28 set_variable+ BATCH_PROCESSING REWEIGHTING_BATCH_PROCESSING 1
29 set_variable+ BATCH_LOGGING REWEIGHTING_BATCH_LOGGING 1
30 set_variable: BATCH_QUEUE_SIZE REWEIGHTING_BATCH_QUEUE_SIZE 3
31 set_variable: BATCH_QUEUE_NAME REWEIGHTING_BATCH_QUEUE_NAME ${TMPDIR:-/tmp}/queue
32 set_variable: BATCH_JOB_SIZE REWEIGHTING_BATCH_JOB_SIZE 1
34 set_variable: TEST_TYPE1D REWEIGHTING_TEST_TYPE1D 0
35 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHTING_TEST_TYPE1D_PARAMETERS "0.05 UWCHI2/NDF"
36 set_variable: TEST_TYPE2D REWEIGHTING_TEST_TYPE2D 0
37 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHTING_TEST_TYPE2D_PARAMETERS "0.05 UWCHI2/NDF"
39 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHTING_N_TRIGGERED_HITS_RANGE "0 $((2**31))"
40 set_variable: CONTAINMENT_VOLUME REWEIGHTING_CONTAINMENT_VOLUME "0.0 0.0 $((2**31)) -$((2**31)) $((2**31))"
41 set_variable: COSTH_RANGE REWEIGHTING_COSTH_RANGE "-1.0 1.0"
42 set_variable: ENERGY_RANGE REWEIGHTING_ENERGY_RANGE "0.0 $((2.0**31))"
44 set_variable: RECOTYPE REWEIGHTING_RECOTYPE "track"
45 set_variable: LIVETIME REWEIGHTING_LIVETIME $(( 5 * 365.25 * 24. * 3600. )) # Default livetime of 5 years
47 set_variable: POSTFIT_APP REWEIGHTING_POSTFIT_APP JAAPostfit
49 set_variable DEFAULT_OSCPARAMETERS_FILE ${JPP_DATA}/oscillation_parameters_NO.txt
50 set_variable OSCPARAMETERS_GRID_FILE ""
51 set_variable WORKDIR ${TMPDIR:-/tmp}
55 usage "$script <detector file> <input file>+ <weight factor file list> [default oscillation parameter file [oscillation parameter grid file [working directory]]]\n" \
56 "Note that if more than one input file is given all optional parameters need to be specified."
60 7) set_variable WORKDIR $7;&
61 6) set_variable OSCPARAMETERS_GRID_FILE $6;&
62 5) set_variable DEFAULT_OSCPARAMETERS_FILE $5;&
63 4) set_variable OSCPROBTABLE $4;
64 set_variable WEIGHT_FACTOR_FILELIST $3;
65 set_array INPUT_FILES $2;
66 set_variable DETX_FILE $1;;
67 *) set_variable DETX_FILE ${argv[1]}
68 set_array INPUT_FILES ${argv[2,-6]}
69 set_variable REWEIGHTING_INPUTS_FILE ${argv[-5]}
70 set_variable OSCPROBTABLE ${argv[-4]}
71 set_variable DEFAULT_OSCPARAMETERS_FILE ${argv[-3]}
72 set_variable OSCPARAMETERS_GRID_FILE ${argv[-2]}
73 set_variable WORKDIR ${argv[-1]}
77 # Function to write job identifier to file
79 # \param output_file output file
85 if (( ${+BATCH_PROCESSING} )); then
87 if [[ -z $EXEC_FILE ]]; then fatal "write_jobID(): Variable EXEC_FILE undefined"; fi
89 if ( is_CCNikhef ); then
90 local jobid=$(qstat -u $USER -f | grep -B1 "${EXEC_FILE:t:r}.${USER}" | grep -oP "Job Id: [0-9]+" | tail -1 | cut -d' ' -f3)
92 local jobid=$(ps | grep "${EXEC_FILE:t:r}" | tail -1 | tr -s ' ' | cut -d' ' -f2)
95 echo ${jobid} >> ${output_file}
100 # Set neutrino labels
112 # Initialise local batch processor
114 if [[ ${BATCH_PROCESSING} == 1 ]]; then
116 if ( is_CCNikhef ); then
118 notice "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${BATCH_QUEUE_NAME}..."
120 if [[ ${WORKDIR} == "/tmp" ]]; then
121 fatal "Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
124 elif ( is_CCLyon ); then
126 fatal "No batch-processing option available yet for CC-Lyon."
130 notice "Submitting reweighting and histogram comparison jobs to local batch queue: ${BATCH_QUEUE_NAME}..."
132 $JPP_DIR/examples/scripts/mkqueue.sh ${BATCH_QUEUE_NAME} ${BATCH_QUEUE_SIZE} "start"
137 # Initialize job ID file
139 set_variable JOBID_FILE ${WORKDIR}/reweighting.jobIDs.txt
144 # Compile oscillation parameters files
146 if [[ -s $OSCPARAMETERS_GRID_FILE ]]; then
148 $JPP_DIR/examples/JOscProb/JWriteOscParametersGrid \
149 -@ ${OSCPARAMETERS_GRID_FILE} \
150 -o ${WORKDIR}/oscillation_parameters.\%.txt \
157 set_variable OSCPARAMETERS_FILE_LIST ${WORKDIR}/oscillation_parameters.files
159 echo ${DEFAULT_OSCPARAMETERS_FILE} > ${OSCPARAMETERS_FILE_LIST}
160 ls ${WORKDIR}/oscillation_parameters.*.txt >> ${OSCPARAMETERS_FILE_LIST}
165 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*merge.root")
167 if (( ${#MC_MERGE_FILES} != 0 )); then
169 echo "----------------------------"
170 printf '%s\n' ${MC_MERGE_FILES[@]}
171 echo "----------------------------"
173 echo "Remove the above regular files? (y/n)"
175 read REMOVE_MC_MERGE_FILES
177 if [[ ${REMOVE_MC_MERGE_FILES} == "y" || ${REMOVE_MC_MERGE_FILES} == "Y" ||
178 ${REMOVE_MC_MERGE_FILES} == "yes" || ${REMOVE_MC_MERGE_FILES} == "Yes" || ${REMOVE_MC_MERGE_FILES} == "YES" ]]; then
179 rm -f ${MC_MERGE_FILES[@]}
184 # Divide into DAQ- and MC-files
186 notice "Dividing into DAQ- and MC-files..."
188 set_variable INPUT_MC_FILELIST ${WORKDIR}/MC.files
189 set_variable INPUT_DAQ_FILELIST ${WORKDIR}/DAQ.files
191 rm -f ${INPUT_MC_FILELIST} ${INPUT_DAQ_FILELIST}
192 touch ${INPUT_MC_FILELIST} ${INPUT_DAQ_FILELIST}
194 for INPUT_FILE in ${INPUT_FILES[@]}; do
196 if [[ "$(JPrintHeader -f ${INPUT_FILE} -k simul --)" != " 0" ]]; then # MC-file
197 set_variable OUTPUT_FILE ${INPUT_MC_FILELIST}
199 set_variable OUTPUT_FILE ${INPUT_DAQ_FILELIST}
202 if [[ ${INPUT_FILE:e} == "files" ]]; then
203 cat ${INPUT_FILE} >> ${OUTPUT_FILE}
205 echo ${INPUT_FILE} >> ${OUTPUT_FILE}
209 set_variable N_MC_FILES $(cat $INPUT_MC_FILELIST | wc -l)
210 set_variable N_DAQ_FILES $(cat $INPUT_DAQ_FILELIST | wc -l)
212 if (( $N_MC_FILES == 0 )); then
213 fatal "No MC input files specified!"
219 if (( ${#MC_MERGE_FILES} == 0 )); then
223 $JPP_DIR/examples/JAAnet/JSortEvt \
224 -f ${INPUT_MC_FILELIST} \
225 -o ${WORKDIR}/\%.merge.root \
226 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
231 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
233 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*merge.root")
237 if (( N_DAQ_FILES > 0 )); then
239 set_array INPUT_REFERENCE_FILES $(cat ${INPUT_DAQ_FILELIST})
240 set_array INPUT_REWEIGHTING_FILES ${MC_MERGE_FILES[@]}
244 ${WORKDIR}/split_events.job\%.sh 1
251 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/null
254 for MC_MERGE_FILE in ${MC_MERGE_FILES[@]}; do
256 set_variable N_EVENTS \
257 $(JPrintTree -f ${MC_MERGE_FILE} -@ "type = Evt" -k number_of_entries)
258 set_variable N1 $(( N_EVENTS / 2 ))
259 set_variable N2 $(( N_EVENTS - N1 ))
261 set_variable INPUT_REFERENCE_FILE ${MC_MERGE_FILE:r}.part1.root
262 set_variable INPUT_REWEIGHTING_FILE ${MC_MERGE_FILE:r}.part2.root
268 if [[ ! -f ${INPUT_REFERENCE_FILE} || ! -f ${INPUT_REWEIGHTING_FILE} ]]; then
270 rm -f ${INPUT_REFERENCE_FILE} ${INPUT_REWEIGHTING_FILE}
272 \${JPP_DIR}/examples/JAAnet/JSplitEvt \
273 -f ${MC_MERGE_FILE} \
274 -o "${INPUT_REFERENCE_FILE} ${INPUT_REWEIGHTING_FILE}" \
279 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
281 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
284 set_array INPUT_REFERENCE_FILES \
285 ${INPUT_REFERENCE_FILES[@]} ${INPUT_REFERENCE_FILE}
286 set_array INPUT_REWEIGHTING_FILES \
287 ${INPUT_REWEIGHTING_FILES[@]} ${INPUT_REWEIGHTING_FILE}
293 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${BATCH_QUEUE_NAME} ${WORKDIR}}}
295 write_jobID ${JOBID_FILE}
301 # Process reference files and create histogram comparison steering file
303 set_variable STEERING_FILE ${WORKDIR}/steeringfile.txt
304 set_variable REFERENCE_POSTFIT_FILE ${WORKDIR}/reference.postfit.root
305 set_variable REFERENCE_POSTFIT_FILE_SCALED ${REFERENCE_POSTFIT_FILE:r}.scaled.root
307 if ( ! reuse_file ${REFERENCE_POSTFIT_FILE} ); then
312 set_array PREREQUISITES $(cat $JOBID_FILE)
315 ${WORKDIR}/${REFERENCE_POSTFIT_FILE:t:r}.job\%.sh 1
322 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/null
327 -f "${INPUT_REFERENCE_FILES[*]}" \\
328 -o ${REFERENCE_POSTFIT_FILE} \\
330 -P ${OSCPROBTABLE} \\
331 -# ${DEFAULT_OSCPARAMETERS_FILE} \\
332 -@ "${(k)NULABELS}; 3;" \\
333 -%energyRange="${ENERGY_RANGE}" \\
334 -%coszenithRange="${COSTH_RANGE}" \\
335 -%containmentVolume="${CONTAINMENT_VOLUME}" \\
336 -%triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
338 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
343 -f ${REFERENCE_POSTFIT_FILE}:. \\
344 -o ${REFERENCE_POSTFIT_FILE_SCALED} \\
346 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
350 mv ${REFERENCE_POSTFIT_FILE_SCALED} ${REFERENCE_POSTFIT_FILE}
353 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
355 set_array NAMES \$(JPrintName -f ${REFERENCE_POSTFIT_FILE}:.)
357 for NAME in \${NAMES[@]}; do
359 set_variable BINFRAC \
361 -f ${REFERENCE_POSTFIT_FILE}:^\${NAME}\$ \
362 -F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
365 if [[ \${NAME} != "META" && \${NAME} != "TObject" ]]; then
367 if (( BINFRAC > 1.0 )); then
368 warning "Cannot perform comparison test for histogram \${NAME} with >2 dimensions; skip."
369 elif (( BINFRAC < 1.0 )); then
370 echo "\${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
372 echo "\${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
383 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${BATCH_QUEUE_NAME} ${WORKDIR} $PREREQUISITES[@]}}
385 write_jobID ${JOBID_FILE}
391 # Process MC-files with standard weights
393 set_variable DEFAULT_POSTFIT_FILE ${WORKDIR}/default.postfit.root
394 set_variable DEFAULT_COMPARISON_FILE ${DEFAULT_POSTFIT_FILE:r}.comparison.root
396 if ( ! reuse_file ${DEFAULT_COMPARISON_FILE} ); then
398 notice "Submitting default histogram comparison jobs"
401 ${WORKDIR}/${DEFAULT_COMPARISON_FILE:t:r}.job\%.sh 1
408 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/null
412 unset DEFAULT_POSTFIT_FILES
414 for (( i=1; i<=${#INPUT_REWEIGHTING_FILES}; ++i )); do
416 set_variable INPUT_FILE ${INPUT_REWEIGHTING_FILES[$i]}
417 set_variable POSTFIT_FILE ${INPUT_FILE:r}.postfit.root
418 set_variable POSTFIT_FILE_SCALED ${POSTFIT_FILE:r}.scaled.root
420 rm -f ${POSTFIT_FILE} ${COMPARISON_FILE}
427 -o ${POSTFIT_FILE} \\
429 -P ${OSCPROBTABLE} \\
430 -# ${DEFAULT_OSCPARAMETERS_FILE} \\
431 -@ "${(k)NULABELS}; 3;" \\
432 -%energyRange="${ENERGY_RANGE}" \\
433 -%coszenithRange="${COSTH_RANGE}" \\
434 -%containmentVolume="${CONTAINMENT_VOLUME}" \\
435 -%triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
437 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE}}
442 -f ${POSTFIT_FILE}:. \\
443 -o ${POSTFIT_FILE_SCALED} \\
445 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
449 mv ${POSTFIT_FILE_SCALED} ${POSTFIT_FILE}
452 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
455 set_array DEFAULT_POSTFIT_FILES \
456 ${DEFAULT_POSTFIT_FILES[@]} ${POSTFIT_FILE}
461 hadd -v 0 -f ${DEFAULT_POSTFIT_FILE} ${DEFAULT_POSTFIT_FILES[@]}
466 -a ${REFERENCE_POSTFIT_FILE} \\
467 -b ${DEFAULT_POSTFIT_FILE} \\
468 -s ${STEERING_FILE} \\
469 -o ${DEFAULT_COMPARISON_FILE} \\
470 -t ${DEFAULT_COMPARISON_FILE:r}.txt \\
471 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
476 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
484 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${BATCH_QUEUE_NAME} ${WORKDIR} ${REFERENCE_POSTFIT_JOBID}}}
486 write_jobID ${JOBID_FILE}
492 # Process MC-files with redefined weights
494 set_array PREREQUISITES $(cat $JOBID_FILE)
496 set_variable META_FILE ${WORKDIR}/comparisonResults.meta.txt
497 set_variable JOBID_FILE ${WORKDIR}/reweighting.jobIDs.txt
499 rm -f ${META_FILE} ${JOBID_FILE}
503 while read -u 3 WEIGHT_FACTOR_FILE; do
505 notice "Submitting reweighting job ${n}"
507 while read -u 4 OSCPARAMETERS_FILE; do
509 set_variable REWEIGHTED_POSTFIT_FILE ${WORKDIR}/reweighting.${n}.postfit.root
510 set_variable REWEIGHTED_COMPARISON_FILE ${REWEIGHTED_POSTFIT_FILE:r}.comparison.root
513 ${WORKDIR}/reweighting.job\%.sh ${BATCH_JOB_SIZE}
515 if (( ${+START_JOB} )); then
522 source ${JPP_DIR}/setenv.sh ${JPP_DIR} > /dev/null
526 for (( j=1; j<=${#INPUT_REWEIGHTING_FILES}; ++j )); do
528 set_variable INPUT_FILE ${INPUT_REWEIGHTING_FILES[$j]}
529 set_variable REWEIGHTED_FILE ${INPUT_FILE:r}.reweighting.${n}.root
530 set_variable POSTFIT_FILE ${REWEIGHTED_FILE:r}.postfit.root
531 set_variable POSTFIT_FILE_SCALED ${POSTFIT_FILE:r}.scaled.root
533 set_array POSTFIT_FILES \
534 ${POSTFIT_FILES[@]} ${POSTFIT_FILE}
538 set_array PRIMARIES \\
539 \$(JPrintHeader -f ${INPUT_FILE} -k primary -- | cut -d'=' -f2)
541 set_array PRIMARIES \\
543 \$(JPrintHeader -f ${INPUT_FILE} -k flux -- | cut -d' ' -f3)
547 for PRIMARY in \${PRIMARIES[@]}; do
548 if [[ "${NUTYPES[@]}" =~ "\${PRIMARY}" ]]; then
549 set_variable DO_REWEIGHTING
554 if (( \${+DO_REWEIGHTING} )); then
560 -o ${REWEIGHTED_FILE} \\
561 -P ${OSCPROBTABLE} \\
562 -@ ${OSCPARAMETERS_FILE} \\
563 -% ${WEIGHT_FACTOR_FILE} \\
564 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
569 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
574 -f ${REWEIGHTED_FILE} \\
575 -o ${POSTFIT_FILE} \\
577 -P ${OSCPROBTABLE} \\
578 -%energyRange="${ENERGY_RANGE}" \\
579 -%coszenithRange="${COSTH_RANGE}" \\
580 -%containmentVolume="${CONTAINMENT_VOLUME}" \\
581 -%triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
583 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
588 -f ${POSTFIT_FILE}:. \\
589 -o ${POSTFIT_FILE_SCALED} \\
591 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
595 mv ${POSTFIT_FILE_SCALED} ${POSTFIT_FILE}
598 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
600 else # This file does not correspond to any of the given primaries
602 ln -fv ${INPUT_FILE} ${REWEIGHTED_FILE}
603 ln -fv ${INPUT_FILE:r}.postfit.root ${POSTFIT_FILE}
606 rm -f ${REWEIGHTED_FILE}
612 hadd -v 0 -f ${REWEIGHTED_POSTFIT_FILE} ${POSTFIT_FILES[@]}
617 -a ${REFERENCE_POSTFIT_FILE} \\
618 -b ${REWEIGHTED_POSTFIT_FILE} \\
619 -s ${STEERING_FILE} \\
620 -o ${REWEIGHTED_COMPARISON_FILE} \\
621 -t ${REWEIGHTED_COMPARISON_FILE:r}.txt \\
622 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
627 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
629 ${SUBMIT_JOB+#rm -f ${EXEC_FILE}}
632 if (( ${+SUBMIT_JOB} )); then
637 ${EXEC_FILE} ${(z)${BATCH_PROCESSING:+${BATCH_QUEUE_NAME} ${WORKDIR} ${PREREQUISITES[@]}}}
639 write_jobID ${JOBID_FILE}
644 echo "${REWEIGHTED_COMPARISON_FILE:r}.txt\t${OSCPARAMETERS_FILE}\t${WEIGHT_FACTOR_FILE}" >> ${META_FILE}
648 done 4< ${OSCPARAMETERS_FILE_LIST}
650 done 3< ${WEIGHT_FACTOR_FILELIST}
654 set_variable IFS ${IFS0}