Jpp  19.1.0-rc.1
the software that should make you happy
JEvtReweightMupageParameterScan.sh
Go to the documentation of this file.
1 #!/bin/zsh
2 #
3 # \author bjung
4 #
5 version=1.0
6 script=${0##*/}
7 
8 # ------------------------------------------------------------------------------------------
9 #
10 # Utility script to evaluate MUPAGE reweightings
11 #
12 # ------------------------------------------------------------------------------------------
13 
14 
15 if [ -z $JPP_DIR ]; then
16  echo "Variable JPP_DIR undefined."
17  exit
18 fi
19 
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 source $JPP_DIR/software/scripts/qlib.sh
22 
23 
24 set_variable IFS0 ${IFS}
25 set_variable IFS $'\n'
26 
27 set_variable: DEBUG REWEIGHT_MUPAGE_DEBUG 2
28 
29 set_variable+ BATCH_PROCESSING REWEIGHT_MUPAGE_BATCH_PROCESSING 1
30 set_variable+ BATCH_LOGGING REWEIGHT_MUPAGE_BATCH_LOGGING 1
31 set_variable: QUEUE_SIZE REWEIGHT_MUPAGE_QUEUE_SIZE 3
32 set_variable: QUEUE_NAME REWEIGHT_MUPAGE_QUEUE_NAME ${TMPDIR:-/tmp}/queue
33 set_variable: JOB_SIZE REWEIGHT_MUPAGE_JOB_SIZE 1
34 
35 set_variable: TEST_TYPE1D REWEIGHT_MUPAGE_TEST_TYPE1D 0
36 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE1D_PARAMETERS 0.05
37 set_variable: TEST_TYPE2D REWEIGHT_MUPAGE_TEST_TYPE2D 2
38 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE2D_PARAMETERS 0.05
39 
40 set_variable: SCRATCHDIR REWEIGHT_MUPAGE_SCRATCHDIR ${TMPDIR:-/tmp}
41 
42 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHT_MUPAGE_N_TRIGGERED_HITS_RANGE "20 $((2**31))"
43 set_variable: COSTH_RANGE REWEIGHT_MUPAGE_COSTH_RANGE "-1.0 1.0"
44 set_variable: ENERGY_RANGE REWEIGHT_MUPAGE_ENERGY_RANGE "0.0 $((2.0**31))"
45 
46 set_variable: RECOTYPE REWEIGHT_MUPAGE_RECOTYPE "track"
47 
48 set_variable FORMULA "[0]"
49 set_variable PARAMETERS_FILE "${WORKDIR}/parameters.txt"
50 set_variable WORKDIR ${TMPDIR:-/tmp}
51 
52 
53 if do_usage $*; then
54  usage "$script <detector file> <input file>+ [reweighting formula [parameter file [working directory]]]"
55 fi
56 
57 case $# in
58  5) set_variable WORKDIR $5;&
59  4) set_variable PARAMETER_FILE $4;&
60  3) set_variable FORMULA $3;&
61  2) set_array INPUT_FILES $2
62  set_variable DETX_FILE $1;;
63  *) set_variable DETX_FILE ${argv[1]}
64  set_array INPUT_FILES ${argv[2,-4]}
65  set_variable FORMULA ${argv[-3]}
66  set_variable PARAMETER_FILE ${argv[-2]}
67  set_variable WORKDIR ${argv[-1]};;
68 esac
69 
70 
71 # Determine number of parameters in given formula
72 
73 set_variable N_FPAR $(echo -n ${FORMULA} | grep -oP "(^|[^x])\[[0-9]*\]" | cut -d'[' -f2- | sort -u | wc -l)
74 set_variable N_PARS $(( $(wc -w < ${PARAMETER_FILE}) - \
75  $(grep "#" ${PARAMETER_FILE} | wc -w) ))
76 set_variable N_PARSETS $(( $(wc -l < ${PARAMETER_FILE}) - \
77  $(grep "#" ${PARAMETER_FILE} | wc -l) ))
78 
79 
80 # Initialise local batch processor
81 
82 if [[ ${BATCH_PROCESSING} == 1 ]]; then
83 
84  if ( is_CCNikhef ); then
85 
86  echo "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${QUEUE_NAME}..."
87 
88  if [[ ${SCRATCHDIR} == "${TMPDIR:-/tmp}" ]]; then
89  fatal "Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
90  fi
91 
92  elif ( is_CCLyon ); then
93 
94  fatal "No batch-processing option available yet for CC-Lyon."
95 
96  else
97 
98  echo "Submitting reweighting and histogram comparison jobs to local batch queue: ${QUEUE_NAME}..."
99 
100  source qlib.sh
101 
102  $JPP_DIR/examples/scripts/mkqueue.sh ${QUEUE_NAME} ${QUEUE_SIZE} "start"
103  fi
104 fi
105 
106 
107 # Check input and retrieve MC files
108 
109 if (( N_PARS % N_FPAR != 0 )); then
110  fatal "Number of input parameters (= ${N_PARS}) is incompatible with given formula: ${FORMULA}"
111 fi
112 
113 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
114 
115 if (( ${#MC_MERGE_FILES} != 0 )); then
116 
117  echo "----------------------------"
118  printf '%s\n' ${MC_MERGE_FILES[@]}
119  echo "----------------------------"
120 
121  echo "Remove the above regular files? (y/n)"
122 
123  read REMOVE_MC_MERGE_FILES
124 
125  if [[ ${REMOVE_MC_MERGE_FILES} == "y" || ${REMOVE_MC_MERGE_FILES} == "Y" ||
126  ${REMOVE_MC_MERGE_FILES} == "yes" || ${REMOVE_MC_MERGE_FILES} == "Yes" || ${REMOVE_MC_MERGE_FILES} == "YES" ]]; then
127 
128  rm ${MC_MERGE_FILES[@]}
129  fi
130 fi
131 
132 
133 # Divide into DAQ- and MC-files
134 
135 echo "Dividing into DAQ- and MC-files..."
136 
137 for INPUT_FILE in ${INPUT_FILES[@]}; do
138 
139  if [[ "$(JPrintHeader -f ${INPUT_FILE} -k simul --)" != " 0" ]]; then # MC-file
140  set_array INPUT_MC_FILES ${INPUT_MC_FILES[@]} ${INPUT_FILE}
141  else # DAQ-file
142  set_array INPUT_DAQ_FILES ${INPUT_DAQ_FILES[@]} ${INPUT_FILE}
143  fi
144 done
145 
146 if (( ${#INPUT_MC_FILES} <= 0 )); then
147  fatal "No MC input files specified!"
148 fi
149 
150 
151 # Merge MC-files
152 
153 if [[ ${#MC_MERGE_FILES} == 0 ]]; then
154 
155  $JPP_DIR/examples/JAAnet/JSortEvt \
156  -f "${INPUT_MC_FILES[*]}" \
157  -o ${WORKDIR}/\%.merge.root \
158  -d ${DEBUG} --!
159 fi
160 
161 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
162 
163 
164 # Process reference files
165 
166 set_variable REFERENCE_POSTFIT_FILE ${WORKDIR}/reference.postfit.root
167 
168 if ( ! reuse_file ${REFERENCE_POSTFIT_FILE} ); then
169 
170  if (( ${#INPUT_DAQ_FILES} > 0 )); then
171  set_array INPUT_REFERENCE_FILES ${INPUT_DAQ_FILES[@]}
172  else
173  set_array INPUT_REFERENCE_FILES ${MC_MERGE_FILES[@]}
174  fi
175 
176  configure_job \
177  ${SCRATCHDIR}/${REFERENCE_POSTFIT_FILE:t:r}.\%.sh 1
178 
179  cat >> ${EXEC_FILE} <<EOF
180 #!/bin/zsh
181 
182 timer_start
183 
184 JAAPostfit \\
185  -f "${INPUT_REFERENCE_FILES[*]}" \\
186  -o ${REFERENCE_POSTFIT_FILE} \\
187  -a ${DETX_FILE} \\
188  -#energyRange="${ENERGY_RANGE}" \\
189  -#coszenithRange="${COSTH_RANGE}" \\
190  -#triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
191  -R ${RECOTYPE} -W \\
192  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
193 
194 timer_stop
195 timer_print
196 
197 rm -f ${EXEC_FILE}
198 EOF
199 
200  submit_job \
201  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR}}
202 
203  reset_job_config
204 fi
205 
206 
207 # Create histogram comparison ASCII steering file
208 
209 set_array NAMES $(JPrintName -f ${REFERENCE_POSTFIT_FILE}:.)
210 
211 set_variable STEERING_FILE ${WORKDIR}/steeringfile.txt
212 
213 rm -f ${STEERING_FILE}
214 
215 unset HISTOGRAMS1D
216 unset HISTOGRAMS2D
217 
218 for NAME in ${NAMES[@]}; do
219 
220  set_variable BINFRAC $(JPrintResult \
221  -f ${REFERENCE_POSTFIT_FILE}:^${NAME}\$ \
222  -F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
223  -O "%f" --)
224 
225  if [[ ${NAME} != "META" && ${NAME} != "TObject" ]]; then
226 
227  if (( BINFRAC > 1.0 )); then
228  warning "Cannot perform comparison test for histogram ${NAME} with >2 dimensions; skip."
229  elif (( BINFRAC < 1.0 )); then
230  echo "${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
231  set_array HISTOGRAMS1D ${HISTOGRAMS1D[@]} ${NAME}
232  else
233  echo "${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
234  set_array HISTOGRAMS2D ${HISTOGRAMS2D[@]} ${NAME}
235  fi
236  fi
237 done
238 
239 
240 # Process MC-files with standard weights
241 
242 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
243 
244  set_variable MC_MERGE_FILE ${MC_MERGE_FILES[$i]}
245  set_variable POSTFIT_FILE ${MC_MERGE_FILE:r}.postfit.root
246  set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
247 
248  echo "Submitting histogram comparison job for ${MC_MERGE_FILE:t}"
249 
250  configure_job \
251  ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
252 
253  cat >> ${EXEC_FILE} <<EOF
254 ${START_JOB+#!/bin/zsh}
255 
256 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
257 
258 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
259 
260 timer_start
261 
262 JAAPostfit \\
263  -f ${MC_MERGE_FILE} \\
264  -o ${POSTFIT_FILE} \\
265  -a ${DETX_FILE} \\
266  -@energyRange="${ENERGY_RANGE}" \\
267  -@coszenithRange="${COSTH_RANGE}" \\
268  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
269  -R ${RECOTYPE} -W \\
270  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE}}
271 
272 timer_stop
273 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
274 
275 timer_start
276 
277 JZebraMantis \\
278  -a ${REFERENCE_POSTFIT_FILE} \\
279  -b ${POSTFIT_FILE} \\
280  -s ${STEERING_FILE} \\
281  -o ${COMPARISON_FILE} \\
282  -t ${COMPARISON_FILE:r}.txt \\
283  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
284 
285 timer_stop
286 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
287 
288 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
289 EOF
290 
291  if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
292  submit_job \
293  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
294  fi
295 done
296 
297 reset_job_config
298 
299 
300 # Process MC-files with redefined weights
301 
302 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
303 
304  set_variable MC_MERGE_FILE ${MC_MERGE_FILES[$i]}
305 
306  echo "Submitting reweighting job for ${MC_MERGE_FILE:t}"
307 
308  if [[ -f ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt ]]; then
309  rm -i ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt
310  fi
311 
312  while read PARAMETERS; do
313 
314  if [[ ${${(z)PARAMETERS}[1]} == "#" ]]; then continue; fi # skip comments
315 
316  if (( ${#${(z)PARAMETERS}} != N_FPAR )); then
317  fatal "Parameterset (${PARAMETERS}) is incompatible with given formula (${FORMULA})"
318  fi
319 
320  unset PARSET
321 
322  for (( j=0; j < N_FPAR; ++j )); do
323  PARSET+=" p${j}=$(printf '%.2e' ${${(z)PARAMETERS}[$((j+1))]});"
324  done
325 
326  set_variable REWEIGHTED_FILE ${MC_MERGE_FILE:r}.reweighting.${i}.root
327  set_variable POSTFIT_FILE ${REWEIGHTED_FILE:r}.postfit.root
328  set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
329 
330  configure_job \
331  ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
332 
333  cat >> ${EXEC_FILE} <<EOF
334 ${START_JOB+#!/bin/zsh}
335 
336 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
337 
338 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
339 
340 timer_start
341 
342 JEvtReweightMupage \\
343  -f ${MC_MERGE_FILE} \\
344  -o ${REWEIGHTED_FILE} \\
345  -F "${FORMULA}" \\
346  -@ "${PARSET}" \\
347  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE} 2>&1}
348 
349 timer_stop
350 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
351 
352 timer_start
353 
354 JAAMuonPostfit \\
355  -f ${REWEIGHTED_FILE} \\
356  -o ${POSTFIT_FILE} \\
357  -a ${DETX_FILE} \\
358  -@energyRange="${ENERGY_RANGE}" \\
359  -@coszenithRange="${COSTH_RANGE}" \\
360  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
361  -R ${RECOTYPE} -W \\
362  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
363 
364 timer_stop
365 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
366 
367 timer_start
368 
369 JZebraMantis \\
370  -a ${REFERENCE_POSTFIT_FILE} \\
371  -b ${POSTFIT_FILE} \\
372  -s ${STEERING_FILE} \\
373  -o ${COMPARISON_FILE} \\
374  -t ${COMPARISON_FILE:r}.txt \\
375  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
376 
377 timer_stop
378 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
379 
380 rm -f ${REWEIGHTED_FILE}
381 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
382 EOF
383 
384  if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
385  submit_job \
386  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
387  fi
388 
389  echo "${COMPARISON_FILE:r}.txt; ${FORMULA}; ${PARSET}" >> ${WORKDIR}/comparisonResults.meta.txt
390 
391  done < ${PARAMETER_FILE}
392 done
393 
394 if (( ${+JOBIDS} )); then
395  rm -f ${SCRATCHDIR}/reweighting.jobIDs.txt
396  printf '%s\n' ${JOBIDS[@]} > ${SCRATCHDIR}/reweighting.jobIDs.txt
397 fi
398 
399 reset_job_config
400 
401 set_variable IFS ${IFS0}
402 
403 
Generated by doxygen 1.9.1