nordugrid-arc-arex 1.1.1-1 / usr / share / arc / submit_common.sh

######################################################
#Common functions for submit scripts
######################################################

# This script should not be executed directly but is sourced in
# from various backend scripts that itself are called from the
# grid manager. Its purpose is to prepare the runtime environments,
# which is almost the same procedure invariant of the backend
# used.
#
# To test the functionality of this script, you may source
# this script from the bourne command shell, i.e. 
# 
#	. /opt/nordugrid/libexec/submit_common.sh
# 
# and it should give no error. Then call the methods defined in
# this script directly. Also, you can test the parse_arg_file
# by executing
#
#	/opt/nordugrid/libexec/submit_common.sh <args>
#
# directly. More tests still require to be added.


init () {

   parse_arg_file $1

   if [ -z "$pkgdatadir" ]; then echo 'pkgdatadir must be set' 1>&2; exit 1; fi
   if [ -z "$joboption_lrms" ]; then echo 'joboption_lrms must be set' 1>&2; exit 1; fi

   # May use joboption_queue (set by parse_arg_file)
   . ${pkgdatadir}/configure-${joboption_lrms}-env.sh

   # Where runtime scripts can be found on computing nodes (empty if does not exist)
   RUNTIME_CONFIG_DIR=$CONFIG_runtimedir
   export RUNTIME_CONFIG_DIR
   # Description of (cross-)mounted disc space on cluster
   RUNTIME_FRONTEND_SEES_NODE=$CONFIG_shared_scratch
   RUNTIME_NODE_SEES_FRONTEND=$CONFIG_shared_filesystem
   RUNTIME_LOCAL_SCRATCH_DIR=$CONFIG_scratchdir
   
   #default is NFS
   if [ -z "${RUNTIME_NODE_SEES_FRONTEND}" ] ; then
     RUNTIME_NODE_SEES_FRONTEND=yes
   fi
   # locally empty means no
   if [ "${RUNTIME_NODE_SEES_FRONTEND}" = 'no' ] ; then
     RUNTIME_NODE_SEES_FRONTEND=
   fi

   # Only CPU time specified in job limits, rough limit for wall time
   walltime_ratio='1'
   # Use specified CPU time as soft limit, allow to run a bit longer before hard limit
   time_hardlimit_ratio='1/1'
   # Use specified memory requirement as soft limit, allow a bit more before hard limit
   memory_hardlimit_ratio='1/1'
   # Where to store temporary files on gatekeeper
   TMPDIR=${TMPDIR:-/tmp}
   # Where GNU time utility is located on computing nodes (empty if does not exist)
   GNU_TIME=${CONFIG_gnu_time:-/usr/bin/time}
   # Command to get name of executing node
   NODENAME=${CONFIG_nodename:-"/bin/hostname -f"}
} 

read_arc_conf () {
   ARC_CONFIG=${ARC_CONFIG:-/etc/arc.conf}
   . $pkgdatadir/config_parser.sh || exit $?

   config_parse_file $ARC_CONFIG 1>&2 || exit $?
   config_import_section "common"
   config_import_section "cluster"

   # Also read queue section
   if [ ! -z "$joboption_queue" ]; then
     if ! config_import_section "queue/$joboption_queue"; then
         echo "ERROR: Job requested invalid share: $joboption_queue" 1>&2
         exit 1
     fi
   fi
}


usage="usage: `basename $0` (<arguments file>|-h|--help)"

parse_arg_file () {
   arg_file=$1
   if [ -z "$arg_file" ] ; then
      echo "Arguments file should be specified" 1>&2
      echo "$usage" 1>&2
      exit 1
   fi
   if [ "--help" = "$1" -o "-h" = "$1" ]; then
      echo "$usage" 1>&2
      cat <<EOHELP 1>&2

This script should not be executed directly but it is
called from the grid manager.
EOHELP
      exit 1
   fi
   if [ ! -f $arg_file ] ; then
      echo "No such arguments file at '$arg_file'" 1>&2
      echo "$usage" 1>&2
      exit 1
   fi
   . $arg_file

   if [ -z "$joboption_controldir" ] ; then
     joboption_controldir=`dirname "$arg_file"`
     if [ "$joboption_controldir" = '.' ] ; then
        joboption_controldir="$PWD"
     fi
   fi
   if [ -z "$joboption_gridid" ] ; then
     joboption_gridid=`basename "$arg_file" | sed 's/^job\.\(.*\)\.grami$/\1/'`
   fi

   ##############################################################
   # combine arguments to command -  easier to use
   ##############################################################
   i=0
   joboption_args=
   eval "var_is_set=\${joboption_arg_$i+yes}"
   while [ ! -z "${var_is_set}" ] ; do
      eval "var_value=\${joboption_arg_$i}"
      # Use -- to avoid echo eating arguments it understands
      var_value=`echo -- "$var_value" |cut -f2- -d' '| sed 's/\\\\/\\\\\\\\/g' | sed 's/"/\\\"/g'`
      joboption_args="$joboption_args \"${var_value}\""
      i=$(( $i + 1 ))
      eval "var_is_set=\${joboption_arg_$i+yes}"
   done
}

#
# Exits with 0 if the argument is all digits
#
is_number () {
    /usr/bin/perl -e 'exit 1 if $ARGV[0] !~ m/^\d+$/' "$1"
}

#
# Sets a default memory limit for jobs that don't have one
#
set_req_mem () {
    if ! is_number "$joboption_memory"; then
        if is_number "$CONFIG_defaultmemory"; then
            joboption_memory=$CONFIG_defaultmemory

            echo "---------------------------------------------------------------------" 1>&2
            echo "WARNING: The job description contains no explicit memory requirement." 1>&2
            echo "         A default memory limit taken from 'defaultmemory' in        " 1>&2
            echo "         arc.conf will apply.                                        " 1>&2
            echo "         Limit is: $CONFIG_nodememory mb.                            " 1>&2
            echo "                                                                     " 1>&2

            if [ "$joboption_localtransfer" = 'yes' ] && [ "$CONFIG_nodememory" -lt 1000 ]; then
                echo "WARNING: localtransfers are enabled and nodes have less than 1GB of  " 1>&2
                echo "         ram. up- and downloaders take up a lot of ram, this can give" 1>&2
                echo "         you problems.                                               " 1>&2
            fi
            echo "---------------------------------------------------------------------" 1>&2
	elif is_number "$CONFIG_nodememory"; then
            joboption_memory=$CONFIG_nodememory

            echo "---------------------------------------------------------------------" 1>&2
            echo "WARNING: The job description contains no explicit memory requirement." 1>&2
            echo "         A default memory limit taken from 'nodememory' in arc.conf  " 1>&2
            echo "         will apply. You may want to set 'defaultmemory' to something" 1>&2
            echo "         else in arc.conf to better handle jobs with no memory       " 1>&2
            echo "         specified.                                                  " 1>&2
            echo "         Limit is: $CONFIG_nodememory mb.                            " 1>&2
            echo "                                                                     " 1>&2

            if [ "$joboption_localtransfer" = 'yes' ] && [ "$CONFIG_nodememory" -lt 1000 ]; then
                echo "WARNING: localtransfers are enabled and nodes have less than 1GB of  " 1>&2
                echo "         ram. up- and downloaders take up a lot of ram, this can give" 1>&2
                echo "         you problems.                                               " 1>&2
            fi
            echo "---------------------------------------------------------------------" 1>&2
        else
            joboption_memory=1000

            echo "---------------------------------------------------------------------" 1>&2
            echo "WARNING: The job description contains no explicit memory requirement " 1>&2
            echo "         and nodememory is not specified in arc.conf. A default      " 1>&2
            echo "         memory limit of 1GB will apply.                             " 1>&2
            echo "---------------------------------------------------------------------" 1>&2
        fi
    fi
}

##############################################################
# Zero stage of runtime environments
##############################################################
# should we write failures to job.111.failed like the old pbs?

RTE_stage0 () {
   joboption_num=0
   eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
   while [ ! -z "${var_is_set}" ] ; do
      eval "var_value=\${joboption_runtime_$joboption_num}"
      if [ -r "$RUNTIME_CONFIG_DIR/${var_value}" ] ; then
         . "$RUNTIME_CONFIG_DIR/${var_value}" 0 1>&2
         if [ $? -ne '0' ] ; then
            echo "ERROR: runtime script ${var_value} failed" 1>&2
            exit 1
         fi
      else
         echo "Warning: runtime script ${var_value} is missing" 1>&2
         #should we exit here?
      fi
      joboption_num=$(( $joboption_num + 1 ))
      eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
   done

  # joboption_count might have been changed by an RTE. Save it for accouting purposes.
  if [ -n "$joboption_count" ]; then
    diagfile="${joboption_controldir}/job.${joboption_gridid}.diag"
    echo "ExecutionUnits=$joboption_count" >> "$diagfile"
  fi
}

##############################################################
# create temp job script
##############################################################

mktempscript () {
   # File name to be used for temporary job script
   LRMS_JOB_SCRIPT=`mktemp ${TMPDIR}/${joboption_lrms}_job_script.XXXXXX`
   echo "Created file $LRMS_JOB_SCRIPT"
   if [ -z "$LRMS_JOB_SCRIPT" ] ; then
      echo "Creation of temporary file failed"
      exit 1
   fi

   LRMS_JOB_OUT="${LRMS_JOB_SCRIPT}.out"
   touch $LRMS_JOB_OUT
   LRMS_JOB_ERR="${LRMS_JOB_SCRIPT}.err"
   touch $LRMS_JOB_ERR
   if [ ! -f "$LRMS_JOB_SCRIPT" ] || [ ! -f "$LRMS_JOB_OUT" ] || [ ! -f "$LRMS_JOB_ERR" ] ; then
      echo "Something is wrong. Either somebody is deleting files or I cannot write to ${TMPDIR}"
      rm -f "$LRMS_JOB_SCRIPT" "$LRMS_JOB_OUT" "$LRMS_JOB_ERR"
      exit 1
   fi
}

##############################################################
# Execution times (obtained in seconds)
##############################################################

##############################################################
# Add environment variables
##############################################################

add_user_env () {
   echo "# Setting environment variables as specified by user" >> $LRMS_JOB_SCRIPT
   has_gridglobalid=''
   i=0
   eval "var_is_set=\${joboption_env_$i+yes}"
   while [ ! -z "${var_is_set}" ] ; do
      eval "var_value=\${joboption_env_$i}"
      if [ "$var_value" ] && [ -z "${var_value##GRID_GLOBAL_JOBID=*}" ]; then
          has_gridglobalid=yes
      fi
      var_escaped=`echo "$var_value" | sed "s/'/'\\\\\''/g"`
      echo "export '${var_escaped}'" >> $LRMS_JOB_SCRIPT
      i=$(( $i + 1 ))
      eval "var_is_set=\${joboption_env_$i+yes}"
   done

   # guess globalid in case not already provided
   if [ -z "$has_gridglobalid" ]; then
      hostname=`/usr/bin/perl -MSys::Hostname -we 'print hostname'`
      hostname=${CONFIG_hostname:-$hostname}
      gm_port=${CONFIG_gm_port:-2811}
      gm_mount_point=${CONFIG_gm_mount_point:-/jobs}
      echo "export GRID_GLOBAL_JOBID='gsiftp://$hostname:$gm_port$gm_mount_point/$joboption_gridid'" >> $LRMS_JOB_SCRIPT
   fi

   echo "" >> $LRMS_JOB_SCRIPT
}

##############################################################
#  Runtime configuration
##############################################################
RTE_stage1 () {
   echo "# Running runtime scripts" >> $LRMS_JOB_SCRIPT
   echo "export RUNTIME_CONFIG_DIR=\${RUNTIME_CONFIG_DIR:-$RUNTIME_CONFIG_DIR}" >> $LRMS_JOB_SCRIPT
   i=0
   eval "var_is_set=\${joboption_runtime_$i+yes}"
   echo "runtimeenvironments=" >> $LRMS_JOB_SCRIPT
   while [ ! -z "${var_is_set}" ] ; do
      if [ "$i" = '0' ] ; then
         echo "if [ ! -z \"\$RUNTIME_CONFIG_DIR\" ] ; then" >> $LRMS_JOB_SCRIPT
      fi
      eval "var_value=\"\${joboption_runtime_$i}\""
      echo "  if [ -r \"\${RUNTIME_CONFIG_DIR}/${var_value}\" ] ; then" >> $LRMS_JOB_SCRIPT
      echo "    runtimeenvironments=\"\${runtimeenvironments}${var_value};\"" >> $LRMS_JOB_SCRIPT
      echo "    . \${RUNTIME_CONFIG_DIR}/${var_value} 1 " >> $LRMS_JOB_SCRIPT
      echo "    if [ \$? -ne '0' ] ; then " >> $LRMS_JOB_SCRIPT
      echo "      echo \"Runtime ${var_value} script failed \" 1>&2 " >> $LRMS_JOB_SCRIPT
      echo "      echo \"Runtime ${var_value} script failed \" 1>\"\$RUNTIME_JOB_DIAG\" " >> $LRMS_JOB_SCRIPT
      echo "      exit 1" >> $LRMS_JOB_SCRIPT
      echo "    fi " >> $LRMS_JOB_SCRIPT
      echo "  fi" >> $LRMS_JOB_SCRIPT
      i=$(( $i + 1 ))
      eval "var_is_set=\${joboption_runtime_$i+yes}"
   done

   if [ ! "$i" = '0' ] ; then
     echo "fi" >> $LRMS_JOB_SCRIPT
   fi

   echo "" >> $LRMS_JOB_SCRIPT
}


#######################################################################
# copy information useful for transfering files to/from node directly
#######################################################################

setup_local_transfer () {
  RUNTIME_CONTROL_DIR=`mktemp ${joboption_directory}/control.XXXXXX`
  if [ -z "$RUNTIME_CONTROL_DIR" ] ; then
    echo 'Failed to choose name for temporary control directory' 1>&2
    rm -f "$LRMS_JOB_SCRIPT" "$LRMS_JOB_OUT" "$LRMS_JOB_ERR"
    exit 1
  fi
  rm -f "$RUNTIME_CONTROL_DIR"
  mkdir "$RUNTIME_CONTROL_DIR"
  if [ $? -ne '0' ] ; then
    echo 'Failed to create temporary control directory' 1>&2
    rm -f "$LRMS_JOB_SCRIPT" "$LRMS_JOB_OUT" "$LRMS_JOB_ERR"
    exit 1
  fi
  chmod go-rwx,u+rwx "${RUNTIME_CONTROL_DIR}"
  echo '' >"${RUNTIME_CONTROL_DIR}/job.local.proxy"
  chmod go-rw,u+r,a-x "${RUNTIME_CONTROL_DIR}/job.local.proxy"
  cat "${joboption_controldir}/job.${joboption_gridid}.proxy" >"${RUNTIME_CONTROL_DIR}/job.local.proxy"
  cat "${joboption_controldir}/job.${joboption_gridid}.input" >"${RUNTIME_CONTROL_DIR}/job.local.input"
  cat "${joboption_controldir}/job.${joboption_gridid}.output">"${RUNTIME_CONTROL_DIR}/job.local.output"
  cat "${joboption_controldir}/job.${joboption_gridid}.local">"${RUNTIME_CONTROL_DIR}/job.local.local"
  RUNTIME_CONTROL_DIR_REL=`basename "$RUNTIME_CONTROL_DIR"`
  echo "$RUNTIME_CONTROL_DIR_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "$RUNTIME_CONTROL_DIR_REL" >>"${RUNTIME_CONTROL_DIR}/job.local.output"
  echo "$RUNTIME_CONTROL_DIR_REL" >>"${joboption_controldir}/job.${joboption_gridid}.output"
  RUNTIME_STDOUT_REL=`echo "${joboption_stdout}" | sed "s#^${joboption_directory}##"`
  RUNTIME_STDERR_REL=`echo "${joboption_stderr}" | sed "s#^${joboption_directory}##"`
  echo "$RUNTIME_STDOUT_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "$RUNTIME_STDERR_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "RUNTIME_CONTROL_DIR=$RUNTIME_CONTROL_DIR" >> $LRMS_JOB_SCRIPT
}


#######################################################################
#uploading output files
#######################################################################

upload_output_files () {
   if [ "$joboption_localtransfer" = 'yes' ] ; then
     echo "UPLOADER=\${UPLOADER:-$basedir/uploader}" >> $LRMS_JOB_SCRIPT
     cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
     if [ "$RESULT" = '0' ] ; then
       $UPLOADER -p -c 'local' "$RUNTIME_CONTROL_DIR" "$RUNTIME_JOB_DIR" 2>>${RUNTIME_CONTROL_DIR}/job.local.errors
       if [ $? -ne '0' ] ; then
         echo 'ERROR: Uploader failed.' 1>&2
         if [ "$RESULT" = '0' ] ; then RESULT=1 ; fi
       fi
     fi
     rm -f "${RUNTIME_CONTROL_DIR}/job.local.proxy"
EOSCR
   fi
}

#######################################################################
# downloading input files (this might fail for fork)
#######################################################################
download_input_files () {
  if [ "$joboption_localtransfer" = 'yes' ] ; then
    RUNTIME_ARC_LOCATION="$ARC_LOCATION"
    RUNTIME_GLOBUS_LOCATION="$GLOBUS_LOCATION"
    echo "ARC_LOCATION=\${ARC_LOCATION:-$RUNTIME_ARC_LOCATION}" >> $LRMS_JOB_SCRIPT
    echo "GLOBUS_LOCATION=\${GLOBUS_LOCATION:-$RUNTIME_GLOBUS_LOCATION}" >> $LRMS_JOB_SCRIPT
    echo "DOWNLOADER=\${DOWNLOADER:-$basedir/downloader}" >> $LRMS_JOB_SCRIPT
    cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
    if [ -z "$ARC_LOCATION" ] ; then
      echo 'Variable ARC_LOCATION is not set' 1>&2
      exit 1
    fi
    if [ -z "$GLOBUS_LOCATION" ] ; then
      echo 'Variable GLOBUS_LOCATION is not set' 1>&2
      exit 1
    fi
    export GLOBUS_LOCATION
    export ARC_LOCATION
    if [ "x$LD_LIBRARY_PATH" = "x" ]; then
      export LD_LIBRARY_PATH="$GLOBUS_LOCATION/lib"
    else
      export LD_LIBRARY_PATH="$GLOBUS_LOCATION/lib:$LD_LIBRARY_PATH"
    fi
    export SASL_PATH="$GLOBUS_LOCATION/lib/sasl"
    export X509_USER_KEY="${RUNTIME_CONTROL_DIR}/job.local.proxy"
    export X509_USER_CERT="${RUNTIME_CONTROL_DIR}/job.local.proxy"
    export X509_USER_PROXY="${RUNTIME_CONTROL_DIR}/job.local.proxy"
    unset X509_RUN_AS_SERVER
    $DOWNLOADER -p -c 'local' "$RUNTIME_CONTROL_DIR" "$RUNTIME_JOB_DIR" 2>>${RUNTIME_CONTROL_DIR}/job.local.errors
    if [ $? -ne '0' ] ; then
      echo 'ERROR: Downloader failed.' 1>&2
      exit 1
    fi
EOSCR
  fi
}

##############################################################
# Add std... to job arguments
##############################################################
include_std_streams () {
  input_redirect=
  output_redirect=
  if [ ! -z "$joboption_stdin" ] ; then
    input_redirect="<\$RUNTIME_JOB_STDIN"
  fi
  if [ ! -z "$joboption_stdout" ] ; then
    output_redirect="1>\$RUNTIME_JOB_STDOUT"
  fi
  if [ ! -z "$joboption_stderr" ] ; then
    if [ "$joboption_stderr" = "$joboption_stdout" ] ; then
      output_redirect="$output_redirect 2>&1"
    else
      output_redirect="$output_redirect 2>\$RUNTIME_JOB_STDERR"
    fi
  fi
}

##############################################################
# Runtime configuration
##############################################################
configure_runtime () {
  i=0
  eval "var_is_set=\${joboption_runtime_$i+yes}"
  while [ ! -z "${var_is_set}" ] ; do
    if [ "$i" = '0' ] ; then
      echo "if [ ! -z \"\$RUNTIME_CONFIG_DIR\" ] ; then" >> $LRMS_JOB_SCRIPT
    fi
    eval "var_value=\"\${joboption_runtime_$i}\""
    echo "  if [ -r \"\${RUNTIME_CONFIG_DIR}/${var_value}\" ] ; then" >> $LRMS_JOB_SCRIPT
    echo "    . \${RUNTIME_CONFIG_DIR}/${var_value} 2 " >> $LRMS_JOB_SCRIPT
    echo "  fi" >> $LRMS_JOB_SCRIPT
    i=$(( $i + 1 ))
    eval "var_is_set=\${joboption_runtime_$i+yes}"
  done
  if [ ! "$i" = '0' ] ; then
    echo "fi" >> $LRMS_JOB_SCRIPT
  fi
  echo "" >> $LRMS_JOB_SCRIPT
}
##############################################################
# move files to node
##############################################################
move_files_to_node () {
  if [ "$joboption_count" -eq 1 ] ; then
    echo "RUNTIME_LOCAL_SCRATCH_DIR=\${RUNTIME_LOCAL_SCRATCH_DIR:-$RUNTIME_LOCAL_SCRATCH_DIR}" >> $LRMS_JOB_SCRIPT
  else
    echo "RUNTIME_LOCAL_SCRATCH_DIR=\${RUNTIME_LOCAL_SCRATCH_DIR:-}" >> $LRMS_JOB_SCRIPT
  fi
  echo "RUNTIME_FRONTEND_SEES_NODE=\${RUNTIME_FRONTEND_SEES_NODE:-$RUNTIME_FRONTEND_SEES_NODE}" >> $LRMS_JOB_SCRIPT
  echo "RUNTIME_NODE_SEES_FRONTEND=\${RUNTIME_NODE_SEES_FRONTEND:-$RUNTIME_NODE_SEES_FRONTEND}" >> $LRMS_JOB_SCRIPT
  cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
  if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ]; then
    RUNTIME_NODE_JOB_DIR="$RUNTIME_LOCAL_SCRATCH_DIR"/`basename "$RUNTIME_JOB_DIR"`
    rm -rf "$RUNTIME_NODE_JOB_DIR"
    mkdir -p "$RUNTIME_NODE_JOB_DIR"
    # move directory contents
    for f in "$RUNTIME_JOB_DIR"/.* "$RUNTIME_JOB_DIR"/*; do 
      [ "$f" = "$RUNTIME_JOB_DIR/*" ] && continue # glob failed, no files
      [ "$f" = "$RUNTIME_JOB_DIR/." ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.." ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.diag" ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.comment" ] && continue
      if ! mv "$f" "$RUNTIME_NODE_JOB_DIR"; then
        echo "Failed to move '$f' to '$RUNTIME_NODE_JOB_DIR'" 1>&2
        exit 1
      fi
    done
    if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
      # creating link for whole directory
       ln -s "$RUNTIME_FRONTEND_SEES_NODE"/`basename "$RUNTIME_JOB_DIR"` "$RUNTIME_JOB_DIR"
    else
      # keep stdout, stderr and control directory on frontend
      # recreate job directory
      mkdir -p "$RUNTIME_JOB_DIR"
      # make those files
      mkdir -p `dirname "$RUNTIME_JOB_STDOUT"`
      mkdir -p `dirname "$RUNTIME_JOB_STDERR"`
      touch "$RUNTIME_JOB_STDOUT"
      touch "$RUNTIME_JOB_STDERR"
      RUNTIME_JOB_STDOUT__=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
      RUNTIME_JOB_STDERR__=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
      rm "$RUNTIME_JOB_STDOUT__" 2>/dev/null
      rm "$RUNTIME_JOB_STDERR__" 2>/dev/null
      if [ ! -z "$RUNTIME_JOB_STDOUT__" ] && [ "$RUNTIME_JOB_STDOUT" != "$RUNTIME_JOB_STDOUT__" ]; then
        ln -s "$RUNTIME_JOB_STDOUT" "$RUNTIME_JOB_STDOUT__"
      fi
      if [ "$RUNTIME_JOB_STDOUT__" != "$RUNTIME_JOB_STDERR__" ] ; then
        if [ ! -z "$RUNTIME_JOB_STDERR__" ] && [ "$RUNTIME_JOB_STDERR" != "$RUNTIME_JOB_STDERR__" ]; then
          ln -s "$RUNTIME_JOB_STDERR" "$RUNTIME_JOB_STDERR__"
        fi
      fi
      if [ ! -z "$RUNTIME_CONTROL_DIR" ] ; then
        # move control directory back to frontend
        RUNTIME_CONTROL_DIR__=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
        mv "$RUNTIME_CONTROL_DIR__" "$RUNTIME_CONTROL_DIR"
      fi
    fi
    # adjust stdin,stdout & stderr pointers
    RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_FRONTEND_JOB_DIR="$RUNTIME_JOB_DIR"
    RUNTIME_JOB_DIR="$RUNTIME_NODE_JOB_DIR"
  fi
  if [ -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
    mkdir -p "$RUNTIME_JOB_DIR"
  fi
EOSCR
}

##############################################################
# move files back to frontend
##############################################################
move_files_to_frontend () {
  cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
  if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ]; then 
    if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
      # just move it
      rm -rf "$RUNTIME_FRONTEND_JOB_DIR"
      destdir=`dirname "$RUNTIME_FRONTEND_JOB_DIR"`
      if ! mv "$RUNTIME_NODE_JOB_DIR" "$destdir"; then
        echo "Failed to move '$RUNTIME_NODE_JOB_DIR' to '$destdir'" 1>&2
        RESULT=1
      fi
    else
      # remove links
      rm -f "$RUNTIME_JOB_STDOUT" 2>/dev/null
      rm -f "$RUNTIME_JOB_STDERR" 2>/dev/null
      # move directory contents
      for f in "$RUNTIME_NODE_JOB_DIR"/.* "$RUNTIME_NODE_JOB_DIR"/*; do 
        [ "$f" = "$RUNTIME_NODE_JOB_DIR/*" ] && continue # glob failed, no files
        [ "$f" = "$RUNTIME_NODE_JOB_DIR/." ] && continue
        [ "$f" = "$RUNTIME_NODE_JOB_DIR/.." ] && continue
        [ "$f" = "$RUNTIME_NODE_JOB_DIR/.diag" ] && continue
        [ "$f" = "$RUNTIME_NODE_JOB_DIR/.comment" ] && continue
        if ! mv "$f" "$RUNTIME_FRONTEND_JOB_DIR"; then
          echo "Failed to move '$f' to '$RUNTIME_FRONTEND_JOB_DIR'" 1>&2
          RESULT=1
        fi
      done
      rm -rf "$RUNTIME_NODE_JOB_DIR"
    fi
  fi
  echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"
  exit $RESULT
EOSCR
}

##############################################################
# copy runtime settings to jobscript
##############################################################
setup_runtime_env () {
  echo "RUNTIME_JOB_DIR=$joboption_directory" >> $LRMS_JOB_SCRIPT
  echo "RUNTIME_JOB_STDIN=$joboption_stdin" >> $LRMS_JOB_SCRIPT
  echo "RUNTIME_JOB_STDOUT=$joboption_stdout" >> $LRMS_JOB_SCRIPT
  echo "RUNTIME_JOB_STDERR=$joboption_stderr" >> $LRMS_JOB_SCRIPT
  echo "RUNTIME_JOB_DIAG=${joboption_directory}.diag" >> $LRMS_JOB_SCRIPT
  # Adjust working directory for tweaky nodes
  # RUNTIME_GRIDAREA_DIR should be defined by external means on nodes
  echo "if [ ! -z \"\$RUNTIME_GRIDAREA_DIR\" ] ; then" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_JOB_DIR=\$RUNTIME_GRIDAREA_DIR/\`basename \$RUNTIME_JOB_DIR\`" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDIN=\`echo \"\$RUNTIME_JOB_STDIN\" | sed \"s#^\$RUNTIME_JOB_DIR#\$RUNTIME_GRIDAREA_DIR#\"\`" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDOUT=\`echo \"\$RUNTIME_JOB_STDOUT\" | sed \"s#^\$RUNTIME_JOB_DIR#\$RUNTIME_GRIDAREA_DIR#\"\`" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDERR=\`echo \"\$RUNTIME_JOB_STDERR\" | sed \"s#^\$RUNTIME_JOB_DIR#\$RUNTIME_GRIDAREA_DIR#\"\`" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_JOB_DIAG=\`echo \"\$RUNTIME_JOB_DIAG\" | sed \"s#^\$RUNTIME_JOB_DIR#\$RUNTIME_GRIDAREA_DIR#\"\`" >> $LRMS_JOB_SCRIPT
  echo "  RUNTIME_CONTROL_DIR=\`echo \"\$RUNTIME_CONTROL_DIR\" | sed \"s#^\$RUNTIME_JOB_DIR#\$RUNTIME_GRIDAREA_DIR#\"\`" >> $LRMS_JOB_SCRIPT
  echo "fi" >> $LRMS_JOB_SCRIPT
}


##############################################################
# change to runtime dir and setup timed run
##############################################################
cd_and_run () {

  cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
  # Changing to session directory
  HOME=$RUNTIME_JOB_DIR
  export HOME
  if ! cd "$RUNTIME_JOB_DIR"; then
    echo "Failed to switch to '$RUNTIME_JOB_DIR'" 1>&2
    RESULT=1
  fi
  if [ ! -z "$RESULT" ] && [ "$RESULT" != 0 ]; then
    exit $RESULT
  fi
EOSCR

  if [ ! -z "$NODENAME" ] ; then
    echo "nodename=\`$NODENAME\`" >> $LRMS_JOB_SCRIPT
    echo "echo \"nodename=\$nodename\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT
  fi
  #TODO this should probably be done on headnode instead
  echo "echo \"ExecutionUnits=${joboption_count}\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT

  # In case the job executable is written in a scripting language and the
  # interpreter is not found, the error message printed by GNU_TIME is
  # misleading.  This will print a more appropriate error message.
  echo "executable='$joboption_arg_0'" >> $LRMS_JOB_SCRIPT
  cat >> $LRMS_JOB_SCRIPT <<'EOSCR'
# See if executable is a script, and extract the name of the interpreter
line1=`dd if="$executable" count=1 2>/dev/null | head -n 1`
command=`echo $line1 | sed -n 's/^#! *//p'`
interpreter=`echo $command | awk '{print $1}'`
if [ "$interpreter" = /usr/bin/env ]; then interpreter=`echo $command | awk '{print $2}'`; fi
# If it's a script and the interpreter is not found ...
[ "x$interpreter" = x ] || type "$interpreter" > /dev/null 2>&1 || {

  echo "Cannot run $executable: $interpreter: not found" "$output_redirect 1>&2"
  exit 1; }
EOSCR
    # Check that gnu_time works
    cat >> $LRMS_JOB_SCRIPT <<EOSCR
GNU_TIME='$GNU_TIME'
if [ ! -z "\$GNU_TIME" ] && ! "\$GNU_TIME" --version >/dev/null 2>&1; then
  echo "WARNING: GNU time not found at: \$GNU_TIME" 2>&1;
  GNU_TIME=
fi 

if [ -z "\$GNU_TIME" ] ; then
  $joboption_args $input_redirect $output_redirect
else
  \$GNU_TIME -o "\$RUNTIME_JOB_DIAG" -a -f '\
WallTime=%es\nKernelTime=%Ss\nUserTime=%Us\nCPUUsage=%P\n\
MaxResidentMemory=%MkB\nAverageResidentMemory=%tkB\n\
AverageTotalMemory=%KkB\nAverageUnsharedMemory=%DkB\n\
AverageUnsharedStack=%pkB\nAverageSharedMemory=%XkB\n\
PageSize=%ZB\nMajorPageFaults=%F\nMinorPageFaults=%R\n\
Swaps=%W\nForcedSwitches=%c\nWaitSwitches=%w\n\
Inputs=%I\nOutputs=%O\nSocketReceived=%r\nSocketSent=%s\n\
Signals=%k\n' \
$joboption_args $input_redirect $output_redirect

fi
RESULT=\$?

EOSCR
}

if [ "submit_common.sh" = `basename $0` ]; then
  parse_arg_file $*
fi