Jpp  18.2.1-ARCA-DF-PATCH
the software that should make you happy
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
qlib.sh
Go to the documentation of this file.
1 #!/bin/zsh
2 #
3 #
4 # \author mdejong, bjjung
5 #
6 #--------------------------------------------------------------------------------------
7 #
8 # Utility script for batch queue functions.
9 #
10 #--------------------------------------------------------------------------------------
11 
12 
13 #
14 # Put command(s) in given queue; do not submit job.
15 #
16 # \param 1 queue name
17 # \param 2-N command(s)
18 #
19 function put_queue()
20 {
21  queue_name=$1
22  shift
23 
24  echo -n "$*;" >> $queue_name
25 }
26 
27 
28 #
29 # Put command(s) in given queue and submit job.
30 #
31 # \param 1 queue name
32 # \param 2-N command(s)
33 #
34 function puts_queue()
35 {
36  queue_name=$1
37  shift
38 
39  echo "$*" >> $queue_name
40 }
41 
42 
43 #
44 # Run queue.
45 #
46 # \param 1 queue name
47 # \param 2 queue size
48 # \param 3 sleep time [s]
49 #
50 function queue()
51 {
52  queue_name=$1
53  queue_size=$2
54  sleep_time=$3
55 
56  if [[ -e $queue_name ]]; then
57 
58  if [[ -p $queue_name ]]; then
59  fatal "Queue $queue_name in use."
60  fi
61  else
62 
63  mkfifo $queue_name # create a new FIFO
64  fi
65 
66  fd=7 # file descriptor
67 
68  eval "exec $fd<>$queue_name" # attach file descriptor to FIFO
69 
70  while read job <& $fd; do # read job from file descriptor
71 
72  if [[ -n $job ]]; then
73 
74  if [[ $job = "exit" ]]; then
75  break
76  fi
77 
78  notice "submit $job"
79  eval exec $job & # submit job
80 
81  # wait for free slot
82 
83  while (( ${#jobstates} >= $queue_size )); do
84  sleep $sleep_time
85  done
86  fi
87  done
88 
89  rm -f $queue_name # remove FIFO
90 }
91 
92 
93 # Function to configure jobs.\n
94 # Sets the environment variable EXEC_FILE, LOG_FILE as well as START_JOB and SUBMIT_JOB\n
95 # depending upon whether a new job should be set up or submitted.
96 #
97 # \param groupsize task group size
98 # \param exec_filename execution file name
99 #
101 {
102  case $# in
103  2) local groupsize=$2;
104  local exec_filename=$1;;
105  *) fatal "configure_job(): Invalid number of arguments."
106  esac
107 
108  if [[ ! ${exec_filename} =~ '%' ]]; then
109  fatal "configure_job(): Missing wildcard character in ${exec_filename}."
110  fi
111 
112  if [[ -z $TASKID ]]; then # Initialize task ID
113  export TASKID=1
114  fi
115 
116  if [[ -z $GROUPID ]]; then # Initialize task group ID
117  export GROUPID=1
118  fi
119 
120  local integer SUBTASKID=$(( TASKID % groupsize ))
121 
122  if (( groupsize == 1 )); then
123 
124  export EXEC_FILE=${exec_filename/\%/${GROUPID}}
125  export LOG_FILE=${EXEC_FILE:r}.log
126 
127  export START_JOB
128  export SUBMIT_JOB
129 
130  rm -f ${EXEC_FILE}
131 
132  elif (( SUBTASKID == 1 )); then # configure job start
133 
134  export EXEC_FILE=${exec_filename/\%/${GROUPID}}
135  export LOG_FILE=${EXEC_FILE:r}.log
136 
137  export START_JOB
138  unset SUBMIT_JOB
139 
140  rm -f ${EXEC_FILE}
141 
142  elif (( SUBTASKID > 1 )); then
143 
144  unset START_JOB
145  unset SUBMIT_JOB
146 
147  elif (( SUBTASKID == 0 )); then # configure job submission
148 
149  unset START_JOB
150  export SUBMIT_JOB
151 
152  (( ++GROUPID ))
153  fi
154 
155  (( ++TASKID ))
156 
157  export GROUPID
158  export TASKID
159 }
160 
161 
162 # Function to reset job configuration variables.
163 #
165 {
166  unset TASKID
167  unset GROUPID
168  unset START_JOB
169  unset SUMBIT_JOB
170  unset EXEC_FILE
171  unset LOG_FILE
172 }
173 
174 
175 # Function to submit a job.
176 #
177 # \param exec_file job script
178 # \param queue_name batch queue name
179 # \param scratchdir scratch directory
180 # \param dependencies dependency PIDs
181 #
183 {
184  local scratchdir=${TMPDIR:-/tmp}
185 
186  case $# in
187  4) dependencies=($4);&
188  3) scratchdir=$3;&
189  2) local queue_name=$2;&
190  1) local exec_file=$1;;
191  *) local exec_file=${argv[1]}
192  local queue_name=${argv[2]}
193  scratchdir=${argv[3]}
194  dependencies=(${argv[4,-1]})
195  esac
196 
197  chmod +x ${exec_file}
198 
199  if (( ${+queue_name} )); then
200 
201  if ( is_CCNikhef ); then
202 
203  local depstr="depend=afterok$(printf ':%s' ${dependencies[@]})"
204 
205  qsub \
206  -N ${exec_file:t:r}.${USER} -V \
207  ${dependencies:+-W ${depstr}} \
208  -q ${queue_name} \
209  -o ${scratchdir} \
210  -e ${scratchdir} \
211  -j oe \
212  ${exec_file}
213  else
214 
215  if (( ${+dependencies} )); then
216 
217  local sleep_time=60
218 
219  local depstr=$(printf '_%s' ${dependencies[@]})
220 
221  set_variable wait_script ${scratchdir}/await${depstr}.sh
222 
223  if [[ ! -f ${wait_script} ]]; then
224 
225  exec 9> ${wait_script}
226 
227  cat <<EOF >&9
228 #!/bin/zsh
229 EOF
230 
231  for dep in ${dependencies[@]}; do
232 
233  cat <<EOF >&9
234 
235 while [[ -e /proc/${dep} ]]; do
236  sleep ${sleep_time}
237 done
238 EOF
239  done
240 
241  cat <<EOF >&9
242 
243 N=\$(ps | grep "${wait_script:t:r}" | wc -l)
244 
245 if (( N == 1 )); then
246  rm -f ${wait_script}
247 fi
248 EOF
249 
250  chmod u+x ${wait_script}
251 
252  exec 9>&-
253  fi
254 
255  sed -i "1a\\\n${wait_script}\n" ${exec_file}
256  fi
257 
258  puts_queue ${queue_name} ${exec_file}
259  fi
260 
261  else
262 
263  zsh ${exec_file}
264  fi
265 }
266 
std::istream & read(std::istream &in, JTestSummary &summary, const char delimiter= ' ')
Read test summary.
then usage $script[< detector identifier >< run range >]< QA/QCfile > nExample script to produce data quality plots nWhen a detector identifier and run range are data are downloaded from the database nand subsequently stored in the given QA QC file
Definition: JDataQuality.sh:19
set_variable USER
Definition: test-tuna.sh:26
o $QUALITY_ROOT d $DEBUG!CHECK_EXIT_CODE JPlot1D f
Definition: JDataQuality.sh:76
usr share Modules init zsh export TMPDIR
data_type r[M+1]
Definition: JPolint.hh:868
V(JDAQEvent-JTriggerReprocessor)*1.0/(JDAQEvent+1.0e-10)
then
Definition: datalogs.sh:31
reset_job_config()
Definition: qlib.sh:164
then fatal Wrong number of arguments fi JConvertDetectorFormat a o
function puts_queue()
Definition: qlib.sh:34
const int n
Definition: JPolint.hh:786
then rm
Definition: sftpput.zsh:30
then echo Variable JPP_DIR undefined exit fi source $JPP_DIR setenv sh $JPP_DIR &dev null set_variable
Definition: JAcoustics.sh:21
then JCalibrateToT a
Definition: JTuneHV.sh:113
skip elif((BINFRAC< 1.0))
do grep(chi2\[.\]\|gradient\)" $INPUT_FILE > $WORKDIR/stage.txt awk 'BEGIN
then usage $script< input file >[option[primary[working directory]]] nWhere option can be N
Definition: JMuonPostfit.sh:40
void configure(const T &value, const JAbstractCollection< JAbscissa_t > &bounds, JBool< false > option)
Configuration of value.
Q FIFO
Definition: JDataQuality.sh:61
then echo Submitting reweighting and histogram comparison jobs to nikhef stbc batch queue
function put_queue()
Definition: qlib.sh:19
int j
Definition: JPolint.hh:792
double u[N+1]
Definition: JPolint.hh:865
then echo
Definition: JQAQC.sh:90
then fatal Wrong number of arguments fi set_variable DETECTOR $argv[1] set_variable INPUT_FILE $argv[2] eval JPrintDetector a $DETECTOR O IDENTIFIER eval JPrintDetector a $DETECTOR O SUMMARY JAcoustics sh $DETECTOR_ID source JAcousticsToolkit sh CHECK_EXIT_CODE typeset A EMITTERS get_tripods $WORKDIR tripod txt EMITTERS get_transmitters $WORKDIR transmitter txt EMITTERS for EMITTER in
Definition: JCanberra.sh:46
esac done
Definition: JAddHDE.sh:21