Jpp  18.0.0
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
JEvtReweightGSeaGenParameterScan.sh
Go to the documentation of this file.
1 #!/bin/zsh
2 #
3 # \author bjung
4 #
5 version=1.0
6 script=${0##*/}
7 
8 # ------------------------------------------------------------------------------------------
9 #
10 # Utility script to evaluate gSeaGen reweightings
11 #
12 # ------------------------------------------------------------------------------------------
13 
14 
15 if [ -z $JPP_DIR ]; then
16  echo "Variable JPP_DIR undefined."
17  exit
18 fi
19 
20 source $JPP_DIR/setenv.sh $JPP_DIR
21 source $JPP_DIR/software/scripts/qlib.sh
22 
23 
25 
26 set_variable: DEBUG REWEIGHTING_DEBUG 2
27 
28 set_variable+ BATCH_PROCESSING REWEIGHTING_BATCH_PROCESSING 1
29 set_variable+ BATCH_LOGGING REWEIGHTING_BATCH_LOGGING 1
30 set_variable: QUEUE_SIZE REWEIGHTING_QUEUE_SIZE 3
31 set_variable: QUEUE_NAME REWEIGHTING_QUEUE_NAME ${TMPDIR:-/tmp}/queue
32 set_variable: JOB_SIZE REWEIGHTING_JOB_SIZE 1
33 
34 set_variable: TEST_TYPE1D REWEIGHTING_TEST_TYPE1D 0
35 set_variable: TEST_TYPE1D_PARAMETERS REWEIGHTING_TEST_TYPE1D_PARAMETERS 0.05
36 set_variable: TEST_TYPE2D REWEIGHTING_TEST_TYPE2D 2
37 set_variable: TEST_TYPE2D_PARAMETERS REWEIGHTING_TEST_TYPE2D_PARAMETERS 0.05
38 
39 set_variable: SCRATCHDIR REWEIGHTING_SCRATCHDIR ${TMPDIR:-/tmp}
40 
41 set_variable: N_TRIGGERED_HITS_RANGE REWEIGHTING_N_TRIGGERED_HITS_RANGE "20 $((2**31))"
42 set_variable: COSTH_RANGE REWEIGHTING_COSTH_RANGE "-1.0 1.0"
43 set_variable: ENERGY_RANGE REWEIGHTING_ENERGY_RANGE "0.0 $((2.0**31))"
44 
45 set_variable: RECOTYPE REWEIGHTING_RECOTYPE "track"
46 
49 
50 
51 if do_usage $*; then
52  usage "$script <detector file> <input file>+ <particle PDG code <reweighting formula <parameter file>>>+ <oscillation probability table> [oscillation parameter file [working directory]]\n" \
53  "Note that if more than one input file is given all optional parameters need to be specified."
54 fi
55 
56 case $# in
60  set_array PARAMETER_FILES $5;
61  set_array FORMULAE $4;
62  set_array NUTYPES $3;
63  set_array INPUT_FILES $2;
65  *) set_variable DETX_FILE ${argv[1]}
66 
67  i=2;
68 
69  while [[ -f ${argv[$i]} ]]; do
70  set_array INPUT_FILES \
71  ${INPUT_FILES[@]} ${argv[$i]}
72  (( ++i ))
73  done
74 
75  while (( i < $# - 2 )); do
76  set_array NUTYPES \
77  ${NUTYPES[@]} ${argv[$i]}
78  set_array FORMULAE \
79  ${FORMULAE[@]} ${argv[$((i+1))]}
80  set_array PARAMETER_FILES \
81  ${PARAMETER_FILES[@]} ${argv[$((i+2))]}
82  (( i+=3 ))
83  done
84 
85  set_variable OSCPROBTABLE ${argv[-3]}
86  set_variable OSCPARAMETERS ${argv[-2]}
87  set_variable WORKDIR ${argv[-1]}
88 esac
89 
90 
91 # Set neutrino labels
92 
93 typeset -A NULABELS
94 set_array NULABELS \
95  -16 "anutau" \
96  -14 "anumu" \
97  -12 "anue" \
98  12 "nue" \
99  14 "numu" \
100  16 "nutau"
101 
102 
103 # Initialise local batch processor
104 
105 if [[ ${BATCH_PROCESSING} == 1 ]]; then
106 
107  if ( is_CCNikhef ); then
108 
109  echo "Submitting reweighting and histogram comparison jobs to nikhef-stbc batch queue: ${QUEUE_NAME}..."
110 
111  if [[ ${SCRATCHDIR} == "${TMPDIR:-/tmp}" ]]; then
112  fatal "Please specify a valid directory for local \'scratch\' files other than \"${TMPDIR:-/tmp}\""
113  fi
114 
115  elif ( is_CCLyon ); then
116 
117  fatal "No batch-processing option available yet for CC-Lyon."
118 
119  else
120 
121  echo "Submitting reweighting and histogram comparison jobs to local batch queue: ${QUEUE_NAME}..."
122 
123  source qlib.sh
124 
125  $JPP_DIR/examples/scripts/mkqueue.sh ${QUEUE_NAME} ${QUEUE_SIZE} "start"
126  fi
127 fi
128 
129 
130 # Retrieve MC files
131 
132 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
133 
134 if (( ${#MC_MERGE_FILES} != 0 )); then
135 
136  echo "----------------------------"
137  printf '%s\n' ${MC_MERGE_FILES[@]}
138  echo "----------------------------"
139 
140  echo "Remove the above regular files? (y/n)"
141 
142  read REMOVE_MC_MERGE_FILES
143 
144  if [[ ${REMOVE_MC_MERGE_FILES} == "y" || ${REMOVE_MC_MERGE_FILES} == "Y" ||
145  ${REMOVE_MC_MERGE_FILES} == "yes" || ${REMOVE_MC_MERGE_FILES} == "Yes" || ${REMOVE_MC_MERGE_FILES} == "YES" ]]; then
146 
148  fi
149 fi
150 
151 
152 # Divide into DAQ- and MC-files
153 
154 echo "Dividing into DAQ- and MC-files..."
155 
156 for INPUT_FILE in ${INPUT_FILES[@]}; do
157 
158  if [[ "$(JPrintHeader -f ${INPUT_FILE} -k simul --)" != " 0" ]]; then # MC-file
160  else # DAQ-file
161  set_array INPUT_DAQ_FILES ${INPUT_DAQ_FILES[@]} ${INPUT_FILE}
162  fi
163 done
164 
165 if (( ${#INPUT_MC_FILES} <= 0 )); then
166  fatal "No MC input files specified!"
167 fi
168 
169 
170 # Merge MC-files
171 
172 if [[ ${#MC_MERGE_FILES} == 0 ]]; then
173 
174  $JPP_DIR/examples/JAAnet/JSortEvt \
175  -f "${INPUT_MC_FILES[*]}" \
176  -o ${WORKDIR}/\%.merge.root \
177  -d ${DEBUG} --!
178 fi
179 
180 set_array MC_MERGE_FILES $(find ${WORKDIR} -maxdepth 1 -name "*.merge.root")
181 
182 
183 # Process reference files
184 
185 set_variable REFERENCE_POSTFIT_FILE ${WORKDIR}/reference.postfit.root
186 
187 if ( ! reuse_file ${REFERENCE_POSTFIT_FILE} ); then
188 
189  if (( ${#INPUT_DAQ_FILES} > 0 )); then
190  set_array INPUT_REFERENCE_FILES ${INPUT_DAQ_FILES[@]}
191  else
192  set_array INPUT_REFERENCE_FILES ${MC_MERGE_FILES[@]}
193  fi
194 
195  configure_job \
196  ${SCRATCHDIR}/${REFERENCE_POSTFIT_FILE:t:r}.\%.sh 1
197 
198  cat >> ${EXEC_FILE} <<EOF
199 #!/bin/zsh
200 
201 if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi
202 
203 source \${JPP_DIR}/setenv.sh \${JPP_DIR}
204 
205 timer_start
206 
207 JAAPostfit \\
208  -f "${INPUT_REFERENCE_FILES[*]}" \\
209  -o ${REFERENCE_POSTFIT_FILE} \\
210  -a ${DETX_FILE} \\
211  -#energyRange="${ENERGY_RANGE}" \\
212  -#coszenithRange="${COSTH_RANGE}" \\
213  -#triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
214  -R ${RECOTYPE} -W \\
215  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
216 
217 timer_stop
218 timer_print
219 
220 rm -f ${EXEC_FILE}
221 EOF
222 
223  submit_job \
224  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR}}
225 
226  reset_job_config
227 fi
228 
229 
230 # Create histogram comparison ASCII steering file
231 
232 set_array NAMES $(JPrintName -f ${REFERENCE_POSTFIT_FILE}:.)
233 
234 set_variable STEERING_FILE ${WORKDIR}/steeringfile.txt
235 
236 rm -f ${STEERING_FILE}
237 
238 unset HISTOGRAMS1D
239 unset HISTOGRAMS2D
240 
241 for NAME in ${NAMES[@]}; do
242 
243  set_variable BINFRAC $(JPrintResult \
244  -f ${REFERENCE_POSTFIT_FILE}:^${NAME}\$ \
245  -F "GetNcells / ((GetNbinsX+2)*(GetNbinsY+2))" \
246  -O "%f" --)
247 
248  if [[ ${NAME} != "META" && ${NAME} != "TObject" ]]; then
249 
250  if (( BINFRAC > 1.0 )); then
251  warning "Cannot perform comparison test for histogram ${NAME} with >2 dimensions; skip."
252  elif (( BINFRAC < 1.0 )); then
253  echo "${NAME}\$ ${TEST_TYPE1D} ${TEST_TYPE1D_PARAMETERS}" >> ${STEERING_FILE}
254  set_array HISTOGRAMS1D ${HISTOGRAMS1D[@]} ${NAME}
255  else
256  echo "${NAME}\$ ${TEST_TYPE2D} ${TEST_TYPE2D_PARAMETERS}" >> ${STEERING_FILE}
257  set_array HISTOGRAMS2D ${HISTOGRAMS2D[@]} ${NAME}
258  fi
259  fi
260 done
261 
262 
263 # Process MC-files with standard weights
264 
265 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
266 
267  set_variable MC_MERGE_FILE ${MC_MERGE_FILES[$i]}
268  set_variable POSTFIT_FILE ${MC_MERGE_FILE:r}.postfit.root
269  set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
270 
271  echo "Submitting histogram comparison job for ${MC_MERGE_FILE:t}"
272 
273  configure_job \
274  ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
275 
276  cat >> ${EXEC_FILE} <<EOF
277 ${START_JOB+#!/bin/zsh}
278 
279 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
280 
281 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
282 
284 
285 JAAPostfit \\
286  -f ${MC_MERGE_FILE} \\
287  -o ${POSTFIT_FILE} \\
288  -a ${DETX_FILE} \\
289  -@energyRange="${ENERGY_RANGE}" \\
290  -@coszenithRange="${COSTH_RANGE}" \\
291  -@triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
292  -R ${RECOTYPE} -W \\
293  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE}}
294 
295 timer_stop
296 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
297 
299 
300 JZebraMantis \\
301  -a ${REFERENCE_POSTFIT_FILE} \\
302  -b ${POSTFIT_FILE} \\
303  -s ${STEERING_FILE} \\
304  -o ${COMPARISON_FILE} \\
305  -t ${COMPARISON_FILE:r}.txt \\
306  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
307 
308 timer_stop
309 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
310 
311 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
312 EOF
313 
314  if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
315  submit_job \
316  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
317  fi
318 done
319 
321 
322 
323 # Process MC-files with redefined weights
324 
325 set_variable PARDELIM ';'
326 set_variable PARSET_LIST ${WORKDIR}/parametersets.txt
327 
328 paste -d"$PARDELIM" ${PARAMETER_FILES[@]} > ${PARSET_LIST}
329 
330 for (( i=1; i<=${#MC_MERGE_FILES}; ++i)); do
331 
333 
334  echo "Submitting reweighting job for ${MC_MERGE_FILE:t}"
335 
336  if [[ -f ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt ]]; then
337  rm -i ${MC_MERGE_FILE:r}.reweighting.comparison.meta.txt
338  fi
339 
340  set_array PRIMARIES \
341  $(JPrintHeader -f ${MC_MERGE_FILE} -k primary -- | cut -d'=' -f2)
342 
343  set_array PRIMARIES \
344  ${PRIMARIES[@]} \
345  $(JPrintHeader -f ${MC_MERGE_FILE} -k flux | cut -d' ' -f3)
346 
347  while IFS="$PARDELIM" read -r PARAMETERSETS; do
348 
349  unset PARINPUTS
350 
351  if [[ ${PARAMETERSETS:0:1} == \# ]]; then continue; fi
352 
353  for (( j=1; j<=${#PARAMETER_FILES}; ++j )); do
354 
355  # Read formula and corresponding parameter set
356 
359  set_array PARAMETERS $(echo $PARAMETERSETS | cut -d"$PARDELIM" -f$j)
360 
361  # Check input
362 
363  set_variable N_FPAR $(echo -n $FORMULA | grep -oP "(^|[^x])\[[0-9]*\]" | cut -d'[' -f2- | sort -u | wc -l)
364 
365  if (( ${#PARAMETERS} != N_FPAR )); then
366  fatal "Number of input parameters (= ${N_PARS}) is incompatible with given formula: ${FORMULA}"
367  fi
368 
369  unset PAREQ
370 
371  for (( k=0; k<${#PARAMETERS}; ++k )); do
372  set_array PAREQS \
373  ${PAREQS[@]} "p${k}=${PARAMETERS[$((k+1))]};"
374  done
375 
376  for PRIMARY in ${PRIMARIES[@]}; do
377 
378  if (( $PRIMARY == $NUTYPE )); then
379 
380  set_array PARINPUTS \
381  ${PARINPUTS[@]} "${NULABELS[${NUTYPE}]}=\"${FORMULA}; ${PAREQS[@]}\""
382  break;
383  fi
384  done
385  done
386 
387  if (( ${#PARINPUTS} == 0 )); then continue; fi
388 
390  set_variable POSTFIT_FILE ${REWEIGHTED_FILE:r}.postfit.root
391  set_variable COMPARISON_FILE ${POSTFIT_FILE:r}.comparison.root
392 
393  configure_job \
394  ${SCRATCHDIR}/${COMPARISON_FILE:t:r}.job\%.sh ${JOB_SIZE}
395 
396  cat >> ${EXEC_FILE} <<EOF
397 ${START_JOB+#!/bin/zsh}
398 
399 ${START_JOB+"if [ -z \${JPP_DIR} ]; then echo \"Variable JPP_DIR undefined.\"; exit; fi"}
400 
401 ${START_JOB+"source \${JPP_DIR}/setenv.sh \${JPP_DIR}"}
402 
404 
405 JEvtReweightGSeaGen \\
406  -f ${MC_MERGE_FILE} \\
407  -o ${REWEIGHTED_FILE} \\
408  -P ${OSCPROBTABLE} \\
409  $(printf '-#%s\n' "${PARINPUTS[*]}") \\
410  ${OSCPARAMETERS:+$(printf '-@%s' ${OSCPARAMETERS})} \\
411  -d ${DEBUG} --! ${BATCH_LOGGING:+&> ${LOG_FILE} 2>&1}
412 
413 timer_stop
414 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
415 
417 
418 JAAPostfit \\
419  -f ${REWEIGHTED_FILE} \\
420  -o ${POSTFIT_FILE} \\
421  -a ${DETX_FILE} \\
422  -#energyRange="${ENERGY_RANGE}" \\
423  -#coszenithRange="${COSTH_RANGE}" \\
424  -#triggeredHitsRange="${N_TRIGGERED_HITS_RANGE}" \\
425  -R ${RECOTYPE} -W \\
426  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
427 
428 timer_stop
429 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
430 
432 
433 JZebraMantis \\
434  -a ${REFERENCE_POSTFIT_FILE} \\
435  -b ${POSTFIT_FILE} \\
436  -s ${STEERING_FILE} \\
437  -o ${COMPARISON_FILE} \\
438  -t ${COMPARISON_FILE:r}.txt \\
439  -d ${DEBUG} --! ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
440 
441 timer_stop
442 timer_print ${BATCH_LOGGING:+&>> ${LOG_FILE} 2>&1}
443 
444 rm -f ${REWEIGHTED_FILE}
445 ${SUBMIT_JOB+rm -f ${EXEC_FILE}}
446 EOF
447 
448  if (( ${+SUBMIT_JOB} || i == ${#MC_MERGE_FILES} )) ; then
449  submit_job \
450  ${EXEC_FILE} ${BATCH_PROCESSING:+${QUEUE_NAME} ${SCRATCHDIR} ${JOBIDS[1]}}
451  fi
452 
453  echo "${COMPARISON_FILE:r}.txt; ${PARINPUTS[*]}" >> ${WORKDIR}/comparisonResults.meta.txt
454 
455  done < ${PARSET_LIST}
456 done
457 
458 if (( ${+JOBIDS} )); then
459  rm -f ${SCRATCHDIR}/reweighting.jobIDs.txt
460  printf '%s\n' ${JOBIDS[@]} > ${SCRATCHDIR}/reweighting.jobIDs.txt
461 fi
462 
464 
465 set_variable IFS ${IFS0}
esac set_variable N_FPAR(echo-n ${FORMULA}|grep-oP"(^|[^x])\[[0-9]*\]"|cut-d'['-f2-|sort-u|wc-l) set_variable N_PARS $(($(wc-w< $
then fatal No hydrophone data file $HYDROPHONE_TXT fi sort gr k
std::istream & read(std::istream &in, JTestSummary &summary, const char delimiter= ' ')
Read test summary.
then usage $script[< detector identifier >< run range >]< QA/QCfile > nExample script to produce data quality plots nWhen a detector identifier and run range are data are downloaded from the database nand subsequently stored in the given QA QC file
Definition: JDataQuality.sh:19
version
Definition: JEditTuneHV.sh:5
&set_variable OSCPROBTABLE
fi set_variable REWEIGHTED_FILE
then JLigier sh continue fi cat
Definition: JDAQDriver.sh:51
do set_array PAREQS
then echo Enter input within $TIMEOUT_S seconds echo n User name
Definition: JCookie.sh:42
o $QUALITY_ROOT d $DEBUG!CHECK_EXIT_CODE JPlot1D f
Definition: JDataQuality.sh:76
Definition: JRoot.hh:19
usr share Modules init zsh export TMPDIR
exit
Definition: JPizza.sh:36
&set_variable OSCPARAMETERS
data_type r[M+1]
Definition: JPolint.hh:779
then usage $script< input file > nFor possible input files
Definition: JMechanics.sh:23
then fatal Wrong number of arguments fi JConvertDetectorFormat a o
reset_job_config()
Definition: qlib.sh:148
&set_variable PRIMARY
Definition: JMuonPostfit.sh:46
const int n
Definition: JPolint.hh:697
then timer_start(JRandomTimesliceWriter-a ${DETECTOR}-o ${PIPE}-n ${NUMBER_OF_SLICES}-P ${PMT_FILE}-d ${DEBUG}-S ${SEED}-B"$BACKGROUND_HZ"-N"$RECYCLING"--!&&for((i=0;$i!=10;++i));do;echo 'EOF' > $PIPE;usleep 100000;done &&rm-f $PIPE)&JTimesliceReprocessor-a $
then set_array PARINPUTS
then rm
Definition: sftpput.zsh:30
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable
Definition: JAcoustics.sh:21
then JCalibrateToT a
Definition: JTuneHV.sh:116
set_variable DETX_FILE
Q DAQ
Definition: JDataQuality.sh:59
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable DEBUG set_variable WORKDIR
Definition: JLegolas.sh:20
set_array PARAMETER_FILES
* usage
set_variable INPUT_FILE
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR source $JPP_DIR software scripts qlib sh set_variable IFS0
skip elif((BINFRAC< 1.0))
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable TRIPOD $argv[2] set_variable TX $argv[3] set_variable TY $argv[4] if[[!-f $DETECTOR]]
Definition: JFootprint.sh:28
configure_job()
Definition: qlib.sh:100
do sput $STRING $FLOOR sget MODULE INPUT_FILES
Definition: JCanberra.sh:77
&set_variable FORMULA
then JCookie sh JDataQuality D $DETECTOR_ID R
Definition: JDataQuality.sh:41
then set_array INPUT_MC_FILES
do grep(chi2\[.\]\|gradient\)" $INPUT_FILE > $WORKDIR/stage.txt awk 'BEGIN
then JMuonMCEvt f $INPUT_FILE o $INTERMEDIATE_FILE d
Definition: JMuonPath.sh:47
then echo Submitting reweighting and histogram comparison jobs to nikhef stbc batch queue
do set_variable NUTYPE
double f3(const double x, const double y, const double z)
3D function.
Definition: JPolynome3D.cc:23
int j
Definition: JPolint.hh:703
fi set_array MC_MERGE_FILES(find ${WORKDIR}-maxdepth 1-name"*.merge.root") if(($
double u[N+1]
Definition: JPolint.hh:776
then echo
Definition: JQAQC.sh:90
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY JAcoustics sh $DETECTOR_ID source JAcousticsToolkit sh CHECK_EXIT_CODE typeset A EMITTERS get_tripods $WORKDIR tripod txt EMITTERS get_transmitters $WORKDIR transmitter txt EMITTERS for EMITTER in
Definition: JCanberra.sh:46
source $JPP_DIR setenv csh $JPP_DIR &dev null eval JShellParser o a A
do set_variable MC_MERGE_FILE
then $DIR JPlotNPE PDG P
Definition: JPlotNPE-PDG.sh:62
esac done
Definition: JAddHDE.sh:21
#define DEBUG(A)
Message macros.
Definition: JMessage.hh:62
do alias $i