qlib.sh File Reference

Go to the source code of this file.

Functions
function	put_queue ()
function	puts_queue ()
function	queue ()
	configure_job ()
	reset_job_config ()
	submit_job ()

Function Documentation

function put_queue

(

)

Definition at line 19 of file qlib.sh.

{
    queue_name=$1
    shift
    
    echo -n "$*;" >> $queue_name
}

function puts_queue

(

)

Definition at line 34 of file qlib.sh.

{
    queue_name=$1
    shift
    
    echo "$*" >> $queue_name
}

function queue

(

)

Definition at line 50 of file qlib.sh.

{
    queue_name=$1
    queue_size=$2
    sleep_time=$3

    if [[ -e $queue_name ]]; then

        if [[ -p $queue_name ]]; then
            fatal "Queue $queue_name in use."
        fi    
    else
        
        mkfifo $queue_name                                # create a new FIFO
    fi

    fd=7                                                  # file descriptor

    eval "exec $fd<>$queue_name"                          # attach file descriptor to FIFO

    while read job <& $fd; do                             # read job from file descriptor

        if [[ -n $job ]]; then
            
            if [[ $job = "exit" ]]; then
                break
            fi

            notice "submit $job" 
            eval exec $job &                              # submit job

            # wait for free slot
        
            while (( ${#jobstates} >= $queue_size )); do
                sleep $sleep_time
            done
        fi
    done
    
    rm -f $queue_name                                     # remove FIFO
}

configure_job

(

)

Definition at line 100 of file qlib.sh.

{
    case $# in
        2) local groupsize=$2;
           local exec_filename=$1;;
        *) fatal "configure_job(): Invalid number of arguments."
    esac

    if [[ ! ${exec_filename} =~ '%' ]]; then
        fatal "configure_job(): Missing wildcard character in ${exec_filename}."
    fi
    
    if [[ -z $TASKID ]]; then  # Initialize task ID
        export TASKID=1
    fi

    if [[ -z $GROUPID ]]; then # Initialize task group ID
        export GROUPID=1
    fi

    local integer SUBTASKID=$(( TASKID % groupsize ))

    if   (( SUBTASKID == 1 || groupsize == 1 )); then # configure job start

        export EXEC_FILE=${exec_filename/\%/${GROUPID}}
        export LOG_FILE=${EXEC_FILE:r}.log      
        
        export START_JOB
        
        rm -f ${EXEC_FILE}
        
    elif (( SUBTASKID > 1 )); then
        unset  START_JOB
        unset  SUBMIT_JOB
    fi

    if (( SUBTASKID == 0 )); then # configure job submission
        
        export SUBMIT_JOB
        (( ++GROUPID ))
    fi

    (( ++TASKID ))
}

reset_job_config

(

)

Definition at line 148 of file qlib.sh.

{
    unset TASKID
    unset GROUPID
    unset START_JOB
    unset SUMBIT_JOB
}

submit_job

(

)

Definition at line 164 of file qlib.sh.

{
    local scratchdir=${TMPDIR:-/tmp}
    
    case $# in
        4) local dependencies=($4);&
        3) scratchdir=$3;&
        2) local queue_name=$2;&
        1) local exec_file=$1;;
        *) local exec_file=${argv[1]}
           local queue_name=${argv[2]}
           scratchdir=${argv[3]}
           local dependencies=(${argv[4,-2]})
    esac
    
    chmod +x ${exec_file}
    
    if (( ${+queue_name} )); then           

        if ( is_CCNikhef ); then

            local depstr="depend=afterok$(printf ':%s' ${dependencies[@]})"
            
            set_array JOBIDS \
                      ${JOBIDS[@]}                          \
                      ${$(qsub \
                              -N ${exec_file:t:r}.${USER}   \
                              ${dependencies:+-W ${depstr}} \
                              -q ${queue_name}              \
                              -o ${scratchdir}              \
                              -e ${scratchdir}              \
                              -j oe                         \
                              ${exec_file})%%.*}
        else

            if (( ${+dependencies} )); then

                local sleep_time=60
                
                for dep in ${dependencies[@]}; do
                
                    cat > ${scratchdir}/await_${dep}.sh <<EOF
#!/bin/zsh

while [[ -e /proc/${dep} ]]; do
    sleep ${sleep_time}
done

rm -f \$0
EOF
                    put_queue ${queue_name} ${scratchdir}/await_${dep}.sh
                done
            fi
            
            puts_queue ${queue_name} ${exec_file}.sh
        fi
        
    else
        
        zsh ${exec_file}
    fi
}