Jpp  17.3.1
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JEvtReweightMupageParameterScan.sh
Go to the documentation of this file.
1 #!/bin/zsh
2 #
3 # \author bjung
4 #
5 version=1.0
6 script=${0##*/}
7 
8 # ------------------------------------------------------------------------------------------
9 #
10 # Utility script to evaluate MUPAGE reweightings
11 #
12 # ------------------------------------------------------------------------------------------
13 
14 
15 if [ -z $JPP_DIR ]; then
16  echo "Variable JPP_DIR undefined."
17  exit
18 fi
19 
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 
23 set_variable IFS $'\n'
24 
25 set_variable: DEBUG REWEIGHT_MUPAGE_DEBUG 2
26 
27 set_variable+ BATCH_PROCESSING REWEIGHT_MUPAGE_BATCH_PROCESSING 1
28 set_variable+ BATCH_LOGGING REWEIGHT_MUPAGE_BATCH_LOGGING 1
29 set_variable: QUEUE_SIZE REWEIGHT_MUPAGE_QUEUE_SIZE 3
30 set_variable: QUEUE_NAME REWEIGHT_MUPAGE_QUEUE_NAME ${TMPDIR:-/tmp}/queue
31 set_variable: JOB_SIZE REWEIGHT_MUPAGE_JOB_SIZE 1
32 
33 set_variable: TEST_TYPE1D REWEIGHT_MUPAGE_TEST_TYPE1D 0
34 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE1D_PARAMETERS 0.05
35 set_variable: TEST_TYPE2D REWEIGHT_MUPAGE_TEST_TYPE2D 2
36 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHT_MUPAGE_TEST_TYPE2D_PARAMETERS 0.05
37 
38 set_variable: SCRATCHDIR REWEIGHT_MUPAGE_SCRATCHDIR ${TMPDIR:-/tmp}
39 
40 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHT_MUPAGE_N_TRIGGERED_HITS_RANGE "20 $((2**31))"
41 set_variable: COSTH_RANGE REWEIGHT_MUPAGE_COSTH_RANGE "-1.0 1.0"
42 set_variable: ENERGY_RANGE REWEIGHT_MUPAGE_ENERGY_RANGE "0.0 $((2.0**31))"
43 
46 set_array PARAMETERS "1.0"
47 
48 if do_usage $*; then
49  usage "$script <detector file> <input file>+ [working directory [reweighting formula [parameter setting]+]]"
50 fi
51 
52 case $# in
56  2) set_array INPUT_FILES $2
58  *) set_variable DETX_FILE ${argv[1]}
59  set_array INPUT_FILES ${argv[2,-4]}
60  set_variable WORKDIR ${argv[-3]}
61  set_variable FORMULA ${argv[-2]}
62  set_variable PARAMETER_FILE ${argv[-1]};;
63 esac
64 
65 
66 # Determine number of parameters in given formula
67 
68 set_variable N_FPAR $(echo -n ${FORMULA} | grep -oP "(^|[^x])\[[0-9]*\]" | cut -d'[' -f2- | sort -u | wc -l)
69 set_variable N_PARS $(( $(wc -w < ${PARAMETER_FILE}) - \
70  $(grep "#" ${PARAMETER_FILE} | wc -w) ))
71 set_variable N_PARSETS $(( $(wc -l < ${PARAMETER_FILE}) - \
72  $(grep "#" ${PARAMETER_FILE} | wc -l) ))
73 
74 # Check input
75 
76 if (( N_PARS % N_FPAR != 0 )); then
77  fatal "Number of input parameters (= ${N_PARS}) is incompatible with given formula: ${FORMULA}"
78 fi
79 
80 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
81 
82 if (( ${#MC_MERGE_FILES} != 0 )); then
83 
84  echo "----------------------------"
85  printf '%s\n' ${MC_MERGE_FILES[@]}
86  echo "----------------------------"
87 
88  echo "Remove the above regular files? (y/n)"
89 
90  read REMOVE_MC_MERGE_FILES
91 
92  if [[ ${REMOVE_MC_MERGE_FILES} == "y" || ${REMOVE_MC_MERGE_FILES} == "Y" ||
93  ${REMOVE_MC_MERGE_FILES} == "yes" || ${REMOVE_MC_MERGE_FILES} == "Yes" || ${REMOVE_MC_MERGE_FILES} == "YES" ]]; then
94 
96  fi
97 fi
98 
99 
100 # Initialise local batch processor
101 
102 if [[ ${BATCH_PROCESSING} == 1 ]]; then
103 
104  if ( is_CCNikhef ); then
105 
106  echo "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${QUEUE_NAME}..."
107 
108  if [[ ${SCRATCHDIR} == "${TMPDIR:-/tmp}" ]]; then
109  fatal "Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
110  fi
111 
112  elif ( is_CCLyon ); then
113 
114  fatal "No batch-processing option available yet for CC-Lyon."
115 
116  else
117 
118  echo "Submitting reweighting and histogram comparison jobs to local batch queue: ${QUEUE_NAME}..."
119 
120  source qlib.sh
121 
122  $JPP_DIR/examples/scripts/mkqueue.sh ${QUEUE_NAME} ${QUEUE_SIZE} "start"
123  fi
124 fi
125 
126 
127 # Divide into DAQ- and MC-files
128 
129 echo "Dividing into DAQ- and MC-files..."
130 
131 for INPUT_FILE in ${INPUT_FILES[@]}; do
132 
133  if [[ "$(JPrintHeader -f ${INPUT_FILE} -k simul --)" != " 0" ]]; then # MC-file
135  else # DAQ-file
136  set_array INPUT_DAQ_FILES ${INPUT_DAQ_FILES[@]} ${INPUT_FILE}
137  fi
138 done
139 
140 if (( ${#INPUT_MC_FILES} <= 0 )); then
141  fatal "No MC input files specified!"
142 fi
143 
144 if (( ${#INPUT_DAQ_FILES} <= 0 )); then
145  fatal "No DAQ input files specified!"
146 fi
147 
148 
149 # Process DAQ-files
150 
152 
153 if ( ! reuse_file ${DAQ_POSTFIT_FILE} ); then
154 
155  # Create histograms
156 
158 
159  JAAMuonPostfit \
160  -f "${INPUT_DAQ_FILES[*]}" \
161  -o ${DAQ_POSTFIT_FILE} \
162  -a ${DETX_FILE} \
163  -@energyRange="${ENERGY_RANGE}" \
164  -@coszenithRange="${COSTH_RANGE}" \
165  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \
166  -W "standardFlux" \
167  -d ${DEBUG} --!
168 
169  timer_stop
170  timer_print
171 fi
172 
173 
174 # Create histogram comparison ASCII steering file
175 
176 set_array NAMES ${(f)$(JPrintName -f ${DAQ_POSTFIT_FILE}:.)}
177 
178 set_variable STEERING_FILE ${WORKDIR}/steeringfile.txt
179 
180 rm -f ${STEERING_FILE}
181 
182 unset HISTOGRAMS1D
183 unset HISTOGRAMS2D
184 
185 for NAME in ${NAMES[@]}; do
186 
187  set_variable BINFRAC $(JPrintResult \
188  -f ${DAQ_POSTFIT_FILE}:^${NAME}\$ \
189  -F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
190  -O "%f" --)
191 
192  if [[ ${NAME} != "META" && ${NAME} != "TObject" ]]; then
193 
194  if (( BINFRAC > 1.0 )); then
195  warning "Cannot perform comparison test for histogram ${NAME} with >2 dimensions; skip."
196  elif (( BINFRAC < 1.0 )); then
197  echo "${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
198  set_array HISTOGRAMS1D ${HISTOGRAMS1D[@]} ${NAME}
199  else
200  echo "${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
201  set_array HISTOGRAMS2D ${HISTOGRAMS2D[@]} ${NAME}
202  fi
203  fi
204 done
205 
206 
207 # Merge MC-files
208 
209 set_variable MC_MERGE_FILE ${WORKDIR}/\%.merge.root
210 
211 if [[ ${#MC_MERGE_FILES} == 0 ]]; then
212 
213  JMergeEvt \
214  -f "${INPUT_MC_FILES[*]}" \
215  -o ${MC_MERGE_FILE} \
216  -d ${DEBUG} --!
217 fi
218 
219 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
220 
221 
222 # Process MC-files with standard weights
223 
224 unset STANDARD_OUTPUT
225 
226 for MC_MERGE_FILE in ${MC_MERGE_FILES[@]}; do
227 
228  echo "Submitting histogram comparison job for ${MC_MERGE_FILE:t}"
229 
230  set_variable EXEC_FILE ${SCRATCHDIR}/${MC_MERGE_FILE:t:r}.sh
231  set_variable LOG_FILE ${EXEC_FILE:r}.log
232 
233  rm -f ${EXEC_FILE}
234 
235  cat >> ${EXEC_FILE} <<EOF
236 #!/bin/zsh
237 
239 
240 JAAMuonPostfit \\
241  -f ${MC_MERGE_FILE} \\
242  -o ${MC_MERGE_FILE:r}.postfit.root \\
243  -a ${DETX_FILE} \\
244  -@energyRange="${ENERGY_RANGE}" \\
245  -@coszenithRange="${COSTH_RANGE}" \\
246  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
247  -W "standardFlux" \\
248  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE}}
249 
250 timer_stop
251 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
252 
254 
255 JZebraMantis \\
256  -a ${DAQ_POSTFIT_FILE} \\
257  -b ${MC_MERGE_FILE:r}.postfit.root \\
258  -s ${STEERING_FILE} \\
259  -o ${MC_MERGE_FILE:r}.postfit.comparison.root \\
260  -t ${MC_MERGE_FILE:r}.postfit.comparison.txt \\
261  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
262 
263 timer_stop
264 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
265 
266 rm -f ${EXEC_FILE}
267 EOF
268 
269  chmod +x ${EXEC_FILE}
270 
271  if [[ ${BATCH_PROCESSING} == 1 ]]; then
272 
273  if ( is_CCNikhef ); then
274 
275  set_array JOBIDS \
276  ${JOBIDS[@]} \
277  ${$(qsub \
278  -N ${EXEC_FILE:t:r}.${USER} \
279  -q ${QUEUE_NAME} \
280  -o ${SCRATCHDIR} \
281  -e ${SCRATCHDIR} \
282  -j oe \
283  ${EXEC_FILE})%%.*}
284  else
285 
286  puts_queue ${QUEUE_NAME} ${EXEC_FILE}
287  fi
288 
289  else
290 
291  zsh ${EXEC_FILE}
292  fi
293 
294  set_array STANDARD_OUTPUT ${MC_MERGE_FILE:r}.postfit.comparison.root
295 done
296 
297 
298 # Process MC-files with redefined weights
299 
300 unset REWEIGHTED_OUTPUT
301 typeset -A REWEIGHTED_OUTPUT
302 
303 for MC_MERGE_FILE in ${MC_MERGE_FILES[@]}; do
304 
305  unset i
306  integer i=1
307 
308  if [[ -f ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt ]]; then
309  rm -i ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt
310  fi
311 
312  while read PARAMETERS; do
313 
314  if [[ ${${(z)PARAMETERS}[1]} == "#" ]]; then continue; fi # skip comments
315 
316  if (( ${#${(z)PARAMETERS}} != N_FPAR )); then
317  fatal "Parameterset (${PARAMETERS}) is incompatible with given formula (${FORMULA})"
318  fi
319 
320  unset PARSET
321 
322  for (( j=0; j < N_FPAR; ++j )); do
323  PARSET+=" p${j}=$(printf '%.2e' ${${(z)PARAMETERS}[$((j+1))]});"
324  done
325 
326  set_variable OUTPUT_FILE ${MC_MERGE_FILE:r}.reweighting.${i}.root
327 
328  if (( i % JOB_SIZE == 1 )); then
329 
330  if (( i + JOB_SIZE - 1 <= N_PARSETS )); then
331  set_variable EXEC_FILE ${SCRATCHDIR}/${MC_MERGE_FILE:t:r}.reweighting.${i}.$((i+JOB_SIZE-1)).sh
332  else
333  set_variable EXEC_FILE ${SCRATCHDIR}/${MC_MERGE_FILE:t:r}.reweighting.${i}.${N_PARSETS}.sh
334  fi
335 
336  set_variable START_JOB 1
337  unset SUBMIT_JOB
338 
339  rm -f ${EXEC_FILE}
340 
341  elif (( i % JOB_SIZE == 0 || i == N_PARSETS )); then
342 
343  set_array JOBRANGE ${${(s/./)EXEC_FILE}[-3,-2]}
344 
345  echo "Submitting reweighting jobs ${JOBRANGE[1]} to ${JOBRANGE[2]} for ${MC_MERGE_FILE:t}"
346 
347  unset START_JOB
348  set_variable SUBMIT_JOB 1
349 
350  else
351 
352  unset START_JOB
353  unset SUBMIT_JOB
354  fi
355 
356  set_variable LOG_FILE ${EXEC_FILE:r}.log
357 
358  eval cat >> ${EXEC_FILE} <<EOF
359 ${START_JOB:+#!/bin/zsh}
360 
361 ${START_JOB:+"if [ -z ${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
362 
363 ${START_JOB:+"source ${JPP_DIR}/setenv.sh ${JPP_DIR}"}
364 
366 
367 JEvtReweightMupage \\
368  -f ${MC_MERGE_FILE} \\
369  -o ${OUTPUT_FILE} \\
370  -F "${FORMULA}" \\
371  -@ "${PARSET}" \\
372  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE} 2>&1}
373 
374 timer_stop
375 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
376 
378 
379 JAAMuonPostfit \\
380  -f ${OUTPUT_FILE} \\
381  -o ${OUTPUT_FILE:r}.postfit.root \\
382  -a ${DETX_FILE} \\
383  -@energyRange="${ENERGY_RANGE}" \\
384  -@coszenithRange="${COSTH_RANGE}" \\
385  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
386  -W "rescaledFlux" \\
387  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
388 
389 timer_stop
390 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
391 
393 
394 JZebraMantis \\
395  -a ${DAQ_POSTFIT_FILE} \\
396  -b ${OUTPUT_FILE:r}.postfit.root \\
397  -s ${STEERING_FILE} \\
398  -o ${OUTPUT_FILE:r}.postfit.comparison.root \\
399  -t ${OUTPUT_FILE:r}.postfit.comparison.txt \\
400  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
401 
402 timer_stop
403 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
404 
405 rm -f ${OUTPUT_FILE}
406 ${SUBMIT_JOB:+rm -f ${EXEC_FILE}}
407 EOF
408 
409  if (( ${+SUBMIT_JOB} )); then
410 
411  if [[ ${BATCH_PROCESSING} == 1 ]]; then
412 
413  chmod +x ${EXEC_FILE}
414 
415  if ( is_CCNikhef ); then
416 
417  set_array JOBIDS \
418  ${JOBIDS[@]} \
419  ${$(qsub \
420  -N ${EXEC_FILE:t:r}.${USER} \
421  -q ${QUEUE_NAME} \
422  -o ${SCRATCHDIR} \
423  -e ${SCRATCHDIR} \
424  -j oe \
425  ${EXEC_FILE})%%.*}
426  else
427 
428  puts_queue ${QUEUE_NAME} ${EXEC_FILE}
429  fi
430 
431  else
432 
433  zsh ${EXEC_FILE}
434  fi
435  fi
436 
437  echo "${OUTPUT_FILE:r}.postfit.comparison.txt; ${FORMULA}; ${PARSET}" >> ${WORKDIR}/comparisonResults.meta.txt
438 
439  i+=1
440 
441  done < ${PARAMETER_FILE}
442 done
443 
444 if (( ${+JOBIDS} )); then
445  rm -f ${SCRATCHDIR}/reweighting.jobIDs.txt
446  printf '%s\n' ${JOBIDS[@]} > ${SCRATCHDIR}/reweighting.jobIDs.txt
447 fi
448 
449 set_variable IFS ${IFS0}
450 
451 
esac set_variable N_FPAR(echo-n ${FORMULA}|grep-oP"(^|[^x])\[[0-9]*\]"|cut-d'['-f2-|sort-u|wc-l) set_variable N_PARS $(($(wc-w< $
data_type w[N+1][M+1]
Definition: JPolint.hh:778
then set_array JOBRANGE
then set_array JOBIDS
std::istream & read(std::istream &in, JTestSummary &summary, const char delimiter= ' ')
Read test summary.
then usage $script[< detector identifier >< run range >]< QA/QCfile > nExample script to produce data quality plots nWhen a detector identifier and run range are data are downloaded from the database nand subsequently stored in the given QA QC file
Definition: JDataQuality.sh:19
set_variable USER
Definition: test-tuna.sh:26
version
Definition: JEditTuneHV.sh:5
then JLigier sh continue fi cat
Definition: JDAQDriver.sh:51
then echo Submitting reweighting and histogram comparison jobs to nikhef stbc batch queue
grep set_variable N_PARSETS(($(wc-l< ${PARAMETER_FILE})-$(grep"#"${PARAMETER_FILE}|wc-l))) if((N_PARS%N_FPAR!=0))
then echo Enter input within $TIMEOUT_S seconds echo n User name
Definition: JCookie.sh:42
o $QUALITY_ROOT d $DEBUG!CHECK_EXIT_CODE JPlot1D f
Definition: JDataQuality.sh:76
then JShowerPostfit f $INPUT_FILE o $OUTPUT_FILE N
usr share Modules init zsh export TMPDIR
exit
Definition: JPizza.sh:36
data_type r[M+1]
Definition: JPolint.hh:779
then usage $script< input file > nFor possible input files
Definition: JMechanics.sh:23
next weights
Definition: JConvertDusj.sh:45
then fatal Wrong number of arguments fi JConvertDetectorFormat a o
&set_array INPUT_FILES set_variable DETX_FILE
function puts_queue()
Definition: qlib.sh:34
then echo
const int n
Definition: JPolint.hh:697
then timer_start(JRandomTimesliceWriter-a ${DETECTOR}-o ${PIPE}-n ${NUMBER_OF_SLICES}-P ${PMT_FILE}-d ${DEBUG}-S ${SEED}-B"$BACKGROUND_HZ"-N"$RECYCLING"--!&&for((i=0;$i!=10;++i));do;echo 'EOF' > $PIPE;usleep 100000;done &&rm-f $PIPE)&JTimesliceReprocessor-a $
then usage $script< detector file >< inputfile > fi case set_variable PARAMETER_FILE
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable
Definition: JAcoustics.sh:21
then JCalibrateToT a
Definition: JTuneHV.sh:116
Q DAQ
Definition: JDataQuality.sh:59
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable DEBUG set_variable WORKDIR
Definition: JLegolas.sh:20
then awk F
* usage
&set_variable OUTPUT_FILE
set_variable INPUT_FILE
skip elif((BINFRAC< 1.0))
do sput $STRING $FLOOR sget MODULE INPUT_FILES
Definition: JCanberra.sh:77
&set_variable FORMULA
then JMuonMCEvt f $INPUT_FILE o $INTERMEDIATE_FILE d
Definition: JMuonPath.sh:47
int j
Definition: JPolint.hh:703
fi set_array MC_MERGE_FILES(find ${WORKDIR}-maxdepth 1-name"*.merge.root") if(($
then set_variable EXEC_FILE
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR set_variable IFS0
double u[N+1]
Definition: JPolint.hh:776
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY JAcoustics sh $DETECTOR_ID source JAcousticsToolkit sh CHECK_EXIT_CODE typeset A EMITTERS get_tripods $WORKDIR tripod txt EMITTERS get_transmitters $WORKDIR transmitter txt EMITTERS for EMITTER in
Definition: JCanberra.sh:46
source $JPP_DIR setenv csh $JPP_DIR &dev null eval JShellParser o a A
then set_array INPUT_MC_FILES
esac done
Definition: JAddHDE.sh:21
#define DEBUG(A)
Message macros.
Definition: JMessage.hh:62
then fatal No DAQ input files specified fi set_variable DAQ_POSTFIT_FILE