/usr/share/arc/submit-ll-job is in nordugrid-arc-arex 1.1.1-1.
This file is owned by root:root, with mode 0o755.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 | #!/bin/sh
# set -xv
#
# Submits job to loadleveler.
#
# A temporary job script is created for the submission and then removed
# at the end of this script.
#
echo "----- starting submit_ll_job -----" 1>&2
joboption_lrms=ll
# ARC1 passes first the config file.
if [ "$1" = "--config" ]; then shift; ARC_CONFIG=$1; shift; fi
basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?
pkgdatadir="$basedir"
. ${pkgdatadir}/configure-ll-env.sh || exit $?
. ${pkgdatadir}/submit_common.sh || exit $?
joboption_localtransfer='no'
##############################################################
# Parse grami file, read arc config
##############################################################
init $1
read_arc_conf
# GD enforce this for the moment
RUNTIME_FRONTEND_SEES_NODE=''
RUNTIME_NODE_SEES_FRONTEND='yes'
##############################################################
# Zero stage of runtime environments
##############################################################
RTE_stage0
LL_SUB='llsubmit'
if [ ! -z "$LL_BIN_PATH" ] ; then
LL_SUB=${LL_BIN_PATH}/${LL_SUB}
fi
mktempscript
##############################################################
# Start job script
##############################################################
echo "# LL batch job script built by grid-manager" > $LRMS_JOB_SCRIPT
# job name for convenience
if [ ! -z "${joboption_jobname}" ] ; then
jobname=`echo "$joboption_jobname" | \
sed 's/^\([^[:alpha:]]\)/N\1/' | \
sed 's/[^[:alnum:]]/_/g' | \
sed 's/\(...............\).*/\1/'`
echo "# @ job_name = $jobname" >> $LRMS_JOB_SCRIPT
fi
echo "LL jobname: $jobname" 1>&2
echo "# @ output = ${joboption_directory}.comment" >> $LRMS_JOB_SCRIPT
echo "# @ error = ${joboption_directory}.comment" >> $LRMS_JOB_SCRIPT
# Project account number for accounting
if [ ! -z "${joboption_rsl_project}" ] ; then
echo "# @ account_no = $joboption_rsl_project" >> $LRMS_JOB_SCRIPT
fi
##############################################################
# (non-)parallel jobs
##############################################################
if [ -z "$joboption_count" ] || [ "$joboption_count" -le 1 ] ; then
joboption_count=1
else
#we only support serial jobs on LL
echo 'ERROR: No support for parallel jobs' 1>&2
fi
##############################################################
# Execution times (obtained in seconds)
##############################################################
# cputime/walltime is obtained in seconds via $joboption_cputime and $joboption_walltime
if ( [ -n "$joboption_cputime" ] && [ $joboption_cputime -gt 0 ] ) ; then
# CPU time must be given per-task for LL
cputime_pertask=$(( $joboption_cputime / $joboption_count ))
cputime_hard_pertask=$(($(( $cputime_pertask * $time_hardlimit_ratio))+30))
echo "# @ job_cpu_limit = ${cputime_hard_pertask} , ${cputime_pertask}" >> $LRMS_JOB_SCRIPT
fi
if [ -n "$joboption_walltime" ] ; then
if [ $joboption_walltime -lt 0 ] ; then
joboption_walltime=0
fi
joboption_walltime_hard=$(($(( $joboption_walltime * $time_hardlimit_ratio))+30))
echo "# @ wall_clock_limit = ${joboption_walltime_hard} , ${joboption_walltime}" >> $LRMS_JOB_SCRIPT
fi
##############################################################
# Requested memory (mb)
##############################################################
set_req_mem
# There are soft and hard limits for virtual memory consumption in LL
if [ -n "$joboption_memory" ] ; then
joboption_memory_hard=$(( $joboption_memory * $memory_hardlimit_ratio ))
requirements="(Memory > ${joboption_memory_hard})"
preferences="(Memory > ${joboption_memory})"
if [ "$LL_CONSUMABLE_RESOURCES" != "yes" ]; then
echo "# @ requirements = ${requirements}" >> $LRMS_JOB_SCRIPT
echo "# @ preferences = ${preferences}" >> $LRMS_JOB_SCRIPT
fi
fi
##############################################################
# Consumable resources
#############################################################
if [ "$LL_CONSUMABLE_RESOURCES" = "yes" ]; then
echo "# @ resources = ConsumableCpus($joboption_count) ConsumableMemory(${joboption_memory})" >> $LRMS_JOB_SCRIPT
fi
##############################################################
# Override umask
##############################################################
#echo "umask 077" >> $LRMS_JOB_SCRIPT
#echo 'exec > /var/tmp/grid-job-output.$$ 2>&1' >> $LRMS_JOB_SCRIPT
##############################################################
# Add environment variables
##############################################################
add_user_env
##############################################################
# Check for existence of executable,
##############################################################
if [ -z "${joboption_arg_0}" ] ; then
echo 'Executable is not specified' 1>&2
exit 1
fi
if [ ! "$joboption_localtransfer" = 'yes' ] ; then
program_start=`echo ${joboption_arg_0} | cut -c 1 2>&1`
if [ "$program_start" != '$' ] && [ "$program_start" != '/' ] ; then
if [ ! -f $joboption_directory/${joboption_arg_0} ] ; then
echo 'Executable does not exist, or permission denied.' 1>&2
echo " Executable $joboption_directory/${joboption_arg_0}" 1>&2
echo " whoami: "`whoami` 1>&2
echo " ls -l $joboption_directory/${joboption_arg_0}: "`ls -l $joboption_directory/${joboption_arg_0}`
exit 1
fi
if [ ! -x $joboption_directory/${joboption_arg_0} ] ; then
echo 'Executable is not executable' 1>&2
exit 1
fi
fi
fi
##################################################################
#Read queue from config or figure out which queue to use
##################################################################
if [ ! -z "${joboption_queue}" ] ; then
class=$joboption_queue
else
#if queue is not set we must choose one
LL_CLASS='llclass -l'
if [ ! -z "$LL_BIN_PATH" ] ; then
LL_CLASS=${LL_BIN_PATH}/${LL_CLASS}
fi
queue_names=`${LL_CLASS}|grep Name|awk '{split($0,field," ");print field[2]}'`
#default will be shortest queue
if [! -n "$joboption_walltime" ] ; then
joboption_walltime_hard=1
fi
queue_time_sel=0
for queue in $queue_names
do
queue_time=`${LL_CLASS} ${queue}|grep Wall_clock_limit|awk '{split($0,field,"(");print field[2]}'|awk '{split($0,field," ");print field[1]}'`
if [${joboption_walltime_hard} -lt ${queue_time}] ; then
if [${queue_time_sel} -eq 0] || [${queue_time_sel} -gt ${queue_time}] ; then
class=${queue}
queue_time_sel=${queue_time}
fi
fi
done
fi
echo "# @ class=${class}" >> $LRMS_JOB_SCRIPT
###################################################################
#Queue job
#No mail notification
##################################################################
echo "# @ notification = never" >> $LRMS_JOB_SCRIPT
echo "# @ queue" >> $LRMS_JOB_SCRIPT
#######################################################################
# copy information useful for transfering files to/from node directly
#######################################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
setup_local_transfer
fi
setup_runtime_env
###################################################################
#setup soft limit trap
##################################################################
echo "trap \"echo 'exitcode=24'>>\$RUNTIME_JOB_DIAG;exit 24\" SIGXCPU" >> $LRMS_JOB_SCRIPT
##############################################################
# Add std... to job arguments
##############################################################
include_std_streams
##############################################################
# Move files to local working directory (job is done on node only)
# RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
move_files_to_node
echo "" >> $LRMS_JOB_SCRIPT
echo "RESULT=0" >> $LRMS_JOB_SCRIPT
echo "" >> $LRMS_JOB_SCRIPT
#####################################################
# Download input files
####################################################
download_input_files
##############################################################
# Skip execution if something already failed
##############################################################
echo "" >> $LRMS_JOB_SCRIPT
echo "if [ \"\$RESULT\" = '0' ] ; then" >> $LRMS_JOB_SCRIPT
echo "echo \"runtimeenvironments=\$runtimeenvironments\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT
##############################################################
# Runtime configuration
##############################################################
RTE_stage1
if [ -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
echo "Nodes detached from gridarea are not supported when LL is used. Aborting job submit" 1>&2
rm -f "$LRMS_JOB_SCRIPT" "$LRMS_JOB_OUT" "$LRMS_JOB_ERR"
exit 1
fi
##############################################################
# Execution
##############################################################
cd_and_run
echo "fi" >> $LRMS_JOB_SCRIPT
##############################################################
# Runtime (post)configuration at computing node
##############################################################
configure_runtime
#####################################################
# Upload output files
####################################################
upload_output_files
##############################################################
# Move files back to session directory (job is done on node only)
# RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
move_files_to_frontend
#######################################
# Submit the job
#######################################
echo "ll job script built" 1>&2
# Execute sub command
cd "$joboption_directory"
echo "LL script follows:" 1>&2
cat "$LRMS_JOB_SCRIPT" 1>&2
echo "" 1>&2
${LL_SUB} $LRMS_JOB_SCRIPT 1>$LRMS_JOB_OUT 2>$LRMS_JOB_ERR
if [ $? -eq '0' ] ; then
echo "LRMS_JOB_OUT is $LRMS_JOB_OUT"
job_id=`cat $LRMS_JOB_OUT | awk '{split($0,field,"\"");print field[2]}'`.0
if [ "${job_id}" = "" ] ; then
echo "job *NOT* submitted successfully!" 1>&2
echo "failed getting the LL jobid for the job!" 1>&2
else
echo "joboption_jobid=$job_id" >> $arg_file
echo "job submitted successfully!" 1>&2
echo "local job id: $job_id" 1>&2
# Remove temporary job script file
rm -f $LRMS_JOB_SCRIPT $LRMS_JOB_OUT $LRMS_JOB_ERR
echo "----- exiting submit_ll_job -----" 1>&2
echo "" 1>&2
exit 0
fi
else
echo "job *NOT* submitted successfully!" 1>&2
echo "got error code from llsubmit!" 1>&2
fi
echo "Output is:" 1>&2
cat $LRMS_JOB_OUT 1>&2
echo "Error output is:"
cat $LRMS_JOB_ERR 1>&2
rm -f $LRMS_JOB_SCRIPT $LRMS_JOB_OUT $LRMS_JOB_ERR
echo "----- exiting submit_ll_job -----" 1>&2
echo "" 1>&2
exit 1
|