8 # ------------------------------------------------------------------------------------------
10 # Utility script to evaluate MUPAGE reweightings
12 # ------------------------------------------------------------------------------------------
15 if [ -z $JPP_DIR ]; then
16 echo "Variable JPP_DIR undefined."
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 source $JPP_DIR/software/scripts/qlib.sh
24 set_variable IFS0 ${IFS}
25 set_variable IFS $'\n'
27 set_variable: DEBUG REWEIGHT_MUPAGE_DEBUG 2
29 set_variable+ BATCH_PROCESSING REWEIGHT_MUPAGE_BATCH_PROCESSING 1
30 set_variable+ BATCH_LOGGING REWEIGHT_MUPAGE_BATCH_LOGGING 1
31 set_variable: QUEUE_SIZE REWEIGHT_MUPAGE_QUEUE_SIZE 3
32 set_variable: QUEUE_NAME REWEIGHT_MUPAGE_QUEUE_NAME ${TMPDIR:-/tmp}/queue
33 set_variable: JOB_SIZE REWEIGHT_MUPAGE_JOB_SIZE 1
35 set_variable: TEST_TYPE1D REWEIGHT_MUPAGE_TEST_TYPE1D 0
36 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE1D_PARAMETERS 0.05
37 set_variable: TEST_TYPE2D REWEIGHT_MUPAGE_TEST_TYPE2D 2
38 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE2D_PARAMETERS 0.05
40 set_variable: SCRATCHDIR REWEIGHT_MUPAGE_SCRATCHDIR ${TMPDIR:-/tmp}
42 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHT_MUPAGE_N_TRIGGERED_HITS_RANGE "20 $((2**31))"
43 set_variable: COSTH_RANGE REWEIGHT_MUPAGE_COSTH_RANGE "-1.0 1.0"
44 set_variable: ENERGY_RANGE REWEIGHT_MUPAGE_ENERGY_RANGE "0.0 $((2.0**31))"
46 set_variable: RECOTYPE REWEIGHT_MUPAGE_RECOTYPE "track"
48 set_variable FORMULA "[0]"
49 set_variable PARAMETERS_FILE "${WORKDIR}/parameters.txt"
50 set_variable WORKDIR ${TMPDIR:-/tmp}
54 usage "$script <detector file> <input file>+ [reweighting formula [parameter file [working directory]]]"
58 5) set_variable WORKDIR $5;&
59 4) set_variable PARAMETER_FILE $4;&
60 3) set_variable FORMULA $3;&
61 2) set_array INPUT_FILES $2
62 set_variable DETX_FILE $1;;
63 *) set_variable DETX_FILE ${argv[1]}
64 set_array INPUT_FILES ${argv[2,-4]}
65 set_variable FORMULA ${argv[-3]}
66 set_variable PARAMETER_FILE ${argv[-2]}
67 set_variable WORKDIR ${argv[-1]};;
71 # Determine number of parameters in given formula
73 set_variable N_FPAR $(echo -n ${FORMULA} | grep -oP "(^|[^x])\[[0-9]*\]" | cut -d'[' -f2- | sort -u | wc -l)
74 set_variable N_PARS $(( $(wc -w < ${PARAMETER_FILE}) - \
75 $(grep "#" ${PARAMETER_FILE} | wc -w) ))
76 set_variable N_PARSETS $(( $(wc -l < ${PARAMETER_FILE}) - \
77 $(grep "#" ${PARAMETER_FILE} | wc -l) ))
80 # Initialise local batch processor
82 if [[ ${BATCH_PROCESSING} == 1 ]]; then
84 if ( is_CCNikhef ); then
86 echo "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${QUEUE_NAME}..."
88 if [[ ${SCRATCHDIR} == "${TMPDIR:-/tmp}" ]]; then
89 fatal "Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
92 elif ( is_CCLyon ); then
94 fatal "No batch-processing option available yet for CC-Lyon."
98 echo "Submitting reweighting and histogram comparison jobs to local batch queue: ${QUEUE_NAME}..."
102 $JPP_DIR/examples/scripts/mkqueue.sh ${QUEUE_NAME} ${QUEUE_SIZE} "start"
107 # Check input and retrieve MC files
109 if (( N_PARS % N_FPAR != 0 )); then
110 fatal "Number of input parameters (= ${N_PARS}) is incompatible with given formula: ${FORMULA}"
113 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
115 if (( ${#MC_MERGE_FILES} != 0 )); then
117 echo "----------------------------"
118 printf '%s\n' ${MC_MERGE_FILES[@]}
119 echo "----------------------------"
121 echo "Remove the above regular files? (y/n)"
123 read REMOVE_MC_MERGE_FILES
125 if [[ ${REMOVE_MC_MERGE_FILES} == "y" || ${REMOVE_MC_MERGE_FILES} == "Y" ||
126 ${REMOVE_MC_MERGE_FILES} == "yes" || ${REMOVE_MC_MERGE_FILES} == "Yes" || ${REMOVE_MC_MERGE_FILES} == "YES" ]]; then
128 rm ${MC_MERGE_FILES[@]}
133 # Divide into DAQ- and MC-files
135 echo "Dividing into DAQ- and MC-files..."
137 for INPUT_FILE in ${INPUT_FILES[@]}; do
139 if [[ "$(JPrintHeader -f ${INPUT_FILE} -k simul --)" != " 0" ]]; then # MC-file
140 set_array INPUT_MC_FILES ${INPUT_MC_FILES[@]} ${INPUT_FILE}
142 set_array INPUT_DAQ_FILES ${INPUT_DAQ_FILES[@]} ${INPUT_FILE}
146 if (( ${#INPUT_MC_FILES} <= 0 )); then
147 fatal "No MC input files specified!"
153 if [[ ${#MC_MERGE_FILES} == 0 ]]; then
155 $JPP_DIR/examples/JAAnet/JSortEvt \
156 -f "${INPUT_MC_FILES[*]}" \
157 -o ${WORKDIR}/\%.merge.root \
161 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
164 # Process reference files
166 set_variable REFERENCE_POSTFIT_FILE ${WORKDIR}/reference.postfit.root
168 if ( ! reuse_file ${REFERENCE_POSTFIT_FILE} ); then
170 if (( ${#INPUT_DAQ_FILES} > 0 )); then
171 set_array INPUT_REFERENCE_FILES ${INPUT_DAQ_FILES[@]}
173 set_array INPUT_REFERENCE_FILES ${MC_MERGE_FILES[@]}
177 ${SCRATCHDIR}/${REFERENCE_POSTFIT_FILE:t:r}.\%.sh 1
179 cat >> ${EXEC_FILE} <<EOF
185 -f "${INPUT_REFERENCE_FILES[*]}" \\
186 -o ${REFERENCE_POSTFIT_FILE} \\
188 -#energyRange="${ENERGY_RANGE}" \\
189 -#coszenithRange="${COSTH_RANGE}" \\
190 -#triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
192 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
201 ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR}}
207 # Create histogram comparison ASCII steering file
209 set_array NAMES $(JPrintName -f ${REFERENCE_POSTFIT_FILE}:.)
211 set_variable STEERING_FILE ${WORKDIR}/steeringfile.txt
213 rm -f ${STEERING_FILE}
218 for NAME in ${NAMES[@]}; do
220 set_variable BINFRAC $(JPrintResult \
221 -f ${REFERENCE_POSTFIT_FILE}:^${NAME}\$ \
222 -F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
225 if [[ ${NAME} != "META" && ${NAME} != "TObject" ]]; then
227 if (( BINFRAC > 1.0 )); then
228 warning "Cannot perform comparison test for histogram ${NAME} with >2 dimensions; skip."
229 elif (( BINFRAC < 1.0 )); then
230 echo "${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
231 set_array HISTOGRAMS1D ${HISTOGRAMS1D[@]} ${NAME}
233 echo "${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
234 set_array HISTOGRAMS2D ${HISTOGRAMS2D[@]} ${NAME}
240 # Process MC-files with standard weights
242 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
244 set_variable MC_MERGE_FILE ${MC_MERGE_FILES[$i]}
245 set_variable POSTFIT_FILE ${MC_MERGE_FILE:r}.postfit.root
246 set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
248 echo "Submitting histogram comparison job for ${MC_MERGE_FILE:t}"
251 ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
253 cat >> ${EXEC_FILE} <<EOF
254 ${START_JOB+#!/bin/zsh}
256 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
258 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
263 -f ${MC_MERGE_FILE} \\
264 -o ${POSTFIT_FILE} \\
266 -@energyRange="${ENERGY_RANGE}" \\
267 -@coszenithRange="${COSTH_RANGE}" \\
268 -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
270 -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE}}
273 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
278 -a ${REFERENCE_POSTFIT_FILE} \\
279 -b ${POSTFIT_FILE} \\
280 -s ${STEERING_FILE} \\
281 -o ${COMPARISON_FILE} \\
282 -t ${COMPARISON_FILE:r}.txt \\
283 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
286 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
288 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
291 if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
293 ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
300 # Process MC-files with redefined weights
302 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
304 set_variable MC_MERGE_FILE ${MC_MERGE_FILES[$i]}
306 echo "Submitting reweighting job for ${MC_MERGE_FILE:t}"
308 if [[ -f ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt ]]; then
309 rm -i ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt
312 while read PARAMETERS; do
314 if [[ ${${(z)PARAMETERS}[1]} == "#" ]]; then continue; fi # skip comments
316 if (( ${#${(z)PARAMETERS}} != N_FPAR )); then
317 fatal "Parameterset (${PARAMETERS}) is incompatible with given formula (${FORMULA})"
322 for (( j=0; j < N_FPAR; ++j )); do
323 PARSET+=" p${j}=$(printf '%.2e' ${${(z)PARAMETERS}[$((j+1))]});"
326 set_variable REWEIGHTED_FILE ${MC_MERGE_FILE:r}.reweighting.${i}.root
327 set_variable POSTFIT_FILE ${REWEIGHTED_FILE:r}.postfit.root
328 set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
331 ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
333 cat >> ${EXEC_FILE} <<EOF
334 ${START_JOB+#!/bin/zsh}
336 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
338 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
342 JEvtReweightMupage \\
343 -f ${MC_MERGE_FILE} \\
344 -o ${REWEIGHTED_FILE} \\
347 -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE} 2>&1}
350 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
355 -f ${REWEIGHTED_FILE} \\
356 -o ${POSTFIT_FILE} \\
358 -@energyRange="${ENERGY_RANGE}" \\
359 -@coszenithRange="${COSTH_RANGE}" \\
360 -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
362 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
365 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
370 -a ${REFERENCE_POSTFIT_FILE} \\
371 -b ${POSTFIT_FILE} \\
372 -s ${STEERING_FILE} \\
373 -o ${COMPARISON_FILE} \\
374 -t ${COMPARISON_FILE:r}.txt \\
375 -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
378 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
380 rm -f ${REWEIGHTED_FILE}
381 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
384 if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
386 ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
389 echo "${COMPARISON_FILE:r}.txt; ${FORMULA}; ${PARSET}" >> ${WORKDIR}/comparisonResults.meta.txt
391 done < ${PARAMETER_FILE}
394 if (( ${+JOBIDS} )); then
395 rm -f ${SCRATCHDIR}/reweighting.jobIDs.txt
396 printf '%s\n' ${JOBIDS[@]} > ${SCRATCHDIR}/reweighting.jobIDs.txt
401 set_variable IFS ${IFS0}