nordugrid-arc-arex 5.4.2-1build1 / usr / share / arc / submit-boinc-job

#!/bin/bash
# set -x
#
#  Input: path to grami file (same as Globus).
#  This script creates a temporary job script and runs it
DEBUG=0
echo "----- starting submit_boinc_job -----" 1>&2
joboption_lrms=boinc

# ARC1 passes first the config file.
if [ "$1" = "--config" ]; then shift; ARC_CONFIG=$1; shift; fi

basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?

pkgdatadir="$basedir"

. ${pkgdatadir}/submit_common.sh || exit $?

##############################################################
# Parse grami file, read arc config
##############################################################
init $1
joboption_directory_orig=$joboption_directory
joboption_directory='`pwd`'
joboption_stdout='`pwd`/'`basename $joboption_stdout`
joboption_stderr='`pwd`/'`basename $joboption_stderr`

read_arc_conf

# make sure session is world-writable
chmod 777 $joboption_directory_orig

RUNTIME_NODE_SEES_FRONTEND=yes

##############################################################
# Zero stage of runtime environments
##############################################################
RTE_stage0
echo "project_root=$PROJECT_ROOT" 1>&2
cd $PROJECT_ROOT

##############################################################
# create job script
##############################################################
mktempscript
LRMS_JOB_BOINC="${LRMS_JOB_SCRIPT}.boinc"
touch $LRMS_JOB_BOINC
chmod u+x ${LRMS_JOB_SCRIPT}


##############################################################
# Start job script
##############################################################

N=0
x=$joboption_directory_orig
while [ "$x" != "/" ]
do
 x=`dirname $x`
N=$((N+1))
done

echo '#!/bin/sh' > $LRMS_JOB_SCRIPT

echo "#job script built by grid-manager and input file for BOINC job" >> $LRMS_JOB_SCRIPT
echo "" >> $LRMS_JOB_SCRIPT
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
set -x 
export RUNTIME_CONFIG_DIR=`pwd`/
#tar zxvf RTE.tar.gz
cp ../shared/ATLAS.root ./
#rename root file
#unzip RTE file
FMARK
echo tar --strip-components=$N -xvf *input.tar.gz >> $LRMS_JOB_SCRIPT

##############################################################
# non-parallel jobs
##############################################################

set_count
sourcewithargs_jobscript

##############################################################
# Override umask
##############################################################
echo "" >> $LRMS_JOB_SCRIPT
echo "# Overide umask of execution node (sometime values are really strange)" >> $LRMS_JOB_SCRIPT
echo "umask 077" >> $LRMS_JOB_SCRIPT

##############################################################
# Add environment variables
##############################################################
add_user_env

##############################################################
# Check for existance of executable,
##############################################################
if [ -z "${joboption_arg_0}" ] ; then
  echo 'Executable is not specified' 1>&2
  exit 1
fi

#######################################################################
# copy information useful for transfering files to/from node directly
#######################################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
   setup_local_transfer
fi

setup_runtime_env

##############################################################
# Add std... to job arguments
##############################################################
include_std_streams

##############################################################
#  Move files to local working directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
move_files_to_node 

##############################################################
#  Runtime configuration
##############################################################
echo RUNTIME_JOB_DIAG='`pwd`/'`basename $joboption_directory_orig`.diag >>$LRMS_JOB_SCRIPT
RTE_stage1 
echo "echo \"runtimeenvironments=\$runtimeenvironments\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT

#####################################################
#  Go to working dir and start job
#####################################################
cd_and_run

## print nodename
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
sed -i -e '/nodename=/d' $RUNTIME_JOB_DIAG
hostname=` grep domain_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
username=` grep user_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
nodename=$username@$hostname
echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"

FMARK

#####################################################
#  Upload output files
#####################################################
upload_output_files
echo "" >> $LRMS_JOB_SCRIPT

##############################################################
#  Runtime (post)configuration at computing node
##############################################################
configure_runtime


##############################################################
#  Move files back to session directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
#move_files_to_frontend

echo 'echo exitcode=$RESULT >> $RUNTIME_JOB_DIAG'  >> $LRMS_JOB_SCRIPT
#############################################################
# zip the result files into 1 file zip.tar.gz
#############################################################
notnull ()
{
   if [ -z $1 ];then
	echo 0
   else
	echo 1
   fi
}
result_list=
i=0
eval opt=\${joboption_outputfile_$i}
ret=`notnull $opt`
while [ $ret = "1" ]
do
   output_file=$(echo $opt|sed -e "s#^/#./#")
   output_file=$(echo $output_file|sed -e "s#@##")
   result_list=$result_list" "$output_file
   i=$((i+1))
   eval opt=\${joboption_outputfile_$i}
   ret=`notnull $opt`
   echo "ret="$ret
done

files=$(echo $result_list|tr " " "\n")

cat >> $LRMS_JOB_SCRIPT <<'EOF'
echo "zip all output files"
flist="*.diag "
EOF
echo for f in $files >>$LRMS_JOB_SCRIPT

cat >> $LRMS_JOB_SCRIPT <<'EOF'
do
 if [  -e $f ];then
 flist=$flist" "$f
fi
done
EOF
#echo $flist
cat <<'EOF' >>$LRMS_JOB_SCRIPT
if [ -f output.list ];then
ol=$(awk '{print $1}' output.list)
for i in $ol
do
 if [ -f $i ];then
 flist=$flist" "$i
 fi
done
fi
EOF

echo 'tar czvf result.tar.gz $flist' >>$LRMS_JOB_SCRIPT

chmod a+r $LRMS_JOB_SCRIPT


#######################################
#  Submit the job
#######################################
## generate RTE file on the fly
joboption_num=0
rflist=$RUNTIME_CONFIG_DIR/APPS/HEP/ATLAS-SITE
eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
while [ ! -z "${var_is_set}" ] ; do
   eval "var_value=\${joboption_runtime_$joboption_num}"
   if [ -r "$RUNTIME_CONFIG_DIR/${var_value}" ] ; then
      rflist=$rflist" $RUNTIME_CONFIG_DIR/${var_value}"
   fi
   joboption_num=$(( $joboption_num + 1 ))
   eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
 done
 
RN=0
x=$RUNTIME_CONFIG_DIR
while [ "$x" != "/" ]
do
 x=`dirname $x`
 RN=$((RN+1))
done
sed -i -e "s/#unzip RTE file/tar --strip-components=$RN -zxvf RTE.tar.gz/g" $LRMS_JOB_SCRIPT

echo "job script ${LRMS_JOB_SCRIPT} built" 1>&2
JobId=`basename $joboption_directory_orig`
JobInput=$joboption_directory_orig/$JobId"_input.tar.gz"
wu=$JobId
RTE_FILE="rte_"$wu".tar.gz"
RTE_LOCATION="$joboption_directory_orig/RTE.tar.gz"

echo "#!/bin/bash" >> $LRMS_JOB_BOINC
echo "set -x" >> $LRMS_JOB_BOINC

tflist=""
Root_basename=()
RootFile=()
## RootFile keeps the orginal path of the root files
## Root_basename keeps the basename of the root files, with adding JobID to make them unique
i=0
for file in `ls $joboption_directory_orig`
  do
    echo $file|grep ".root" > /dev/null
    ret=$?
    if [ $ret -eq 0 ]; then
      echo skip root file $file
      Root_basename[$i]=$JobId"_"$file
      RootFile[$i]=$joboption_directory_orig/$file 
      sed -i -e "/#rename root file/a\mv ATLAS.root_$i $file" $LRMS_JOB_SCRIPT
      let i=$i+1
      continue
    else
      tflist=$tflist" "$joboption_directory_orig/$file
    fi
  done

echo tar zhcvf $JobInput $tflist
echo "tar zhcvf $JobInput $tflist" >> $LRMS_JOB_BOINC
echo tar zhcvf $RTE_LOCATION $rflist
echo "tar zhcvf $RTE_LOCATION $rflist" >> $LRMS_JOB_BOINC

echo "cd $PROJECT_ROOT " >>$LRMS_JOB_BOINC

JobInput_basename=`basename $JobInput`
Script_basename=`basename $LRMS_JOB_SCRIPT` 

echo "cp $JobInput "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "chmod a+r "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "cp $LRMS_JOB_SCRIPT " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC
echo "chmod a+r " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC
echo "cp $RTE_LOCATION " '`bin/dir_hier_path' $RTE_FILE '`' >> $LRMS_JOB_BOINC
echo "chmod a+r " '`bin/dir_hier_path' $RTE_FILE '`' >> $LRMS_JOB_BOINC

[ -n $PROJECT_DOWNLOAD_ROOT ] && echo "cd $PROJECT_DOWNLOAD_ROOT" >> $LRMS_JOB_BOINC

## process the root files as remote files
cd $PROJECT_ROOT
echo "current directory is the project_root: "$PWD
remote_url=()
fsize=()
md5=()
i=0
while [ $i -lt ${#RootFile[@]} ]
do
  [ -L ${RootFile[$i]} ] && RootFile[$i]=`ls -l ${RootFile[$i]}|awk '{print $11}'`
  echo "ln  ${RootFile[$i]} "'`bin/dir_hier_path' ${Root_basename[$i]} '`' >> $LRMS_JOB_BOINC
  echo "chmod a+r "'`bin/dir_hier_path ' ${Root_basename[$i]} '`' >> $LRMS_JOB_BOINC
  download_dir=`bin/dir_hier_path ${Root_basename[$i]} | awk -F/ '{print $(NF-1)}'`
  remote_url[$i]="${PROJECT_DOWNLOAD_URL}/${download_dir}/${Root_basename[$i]}"
  fsize[$i]=`stat -c %s ${RootFile[$i]}`
  md5[$i]=`md5sum ${RootFile[$i]} | awk '{print $1}'`
  echo "Using remote file ${remote_url[$i]} ${fsize[$i]} ${md5[$i]}" 1>&2
  let i=$i+1
done

[ -n $PROJECT_DOWNLOAD_ROOT ] && echo "cd $PROJECT_ROOT" >> $LRMS_JOB_BOINC

## generate the input template file
let ifileno=3+${#remote_url[@]}
i=0
intmp=""
while [ $i -lt $ifileno ]
do
  intmp="$intmp
<file_info>
   <number>$i</number>
</file_info>"

  let i=$i+1
done
intmp="$intmp
<workunit>"
i=0
while [ $i -lt ${#remote_url[@]} ]
do
  intmp="$intmp
 <file_ref>
     <file_number>$i</file_number>
     <open_name>shared/ATLAS.root_$i</open_name>
     <copy_file/>
   </file_ref>"
  
  let i=$i+1
done

intmp="$intmp
  <file_ref>
     <file_number>$i</file_number>
     <open_name>shared/input.tar.gz</open_name>
     <copy_file/>
   </file_ref>
  <file_ref>"

let i=$i+1
intmp="$intmp
     <file_number>$i</file_number>
     <open_name>shared/RTE.tar.gz</open_name>
     <copy_file/>
   </file_ref>
"
let i=$i+1
intmp="$intmp
   <file_ref>
        <file_number>$i</file_number>
        <open_name>shared/start_atlas.sh</open_name>
        <copy_file/>
    </file_ref>"

intmp_res=$(cat $WU_TEMPLATE)
intmp="$intmp
$intmp_res
"

intmp="$intmp
   </workunit>
"

WU_TEMPLATE_tmp=$(mktemp /tmp/${BOINC_APP}_XXXXXX)
cat << EOF  > $WU_TEMPLATE_tmp
$intmp
EOF

#######################################
if [ -z $joboption_memory ];then
  memreq=2000000000
else
  memreq=$((joboption_memory*1000000))
fi

if [ -z $joboption_cputime ];then
  maxcputime=$((2*3600*3000000000))
else
  maxcputime=$((joboption_cputime*3000000000))
fi

priority=
if [ ! -z "$joboption_priority" ]; then
  priority="--priority $joboption_priority"
fi

cmd="bin/create_work \
      --appname $BOINC_APP \
      --wu_name $wu \
      --wu_template $WU_TEMPLATE_tmp \
      --result_template $RESULT_TEMPLATE \
      --rsc_memory_bound $memreq \
      --rsc_fpops_est $maxcputime \
      $priority"
j=0
while [ $j -lt ${#remote_url[@]} ] 
do
  cmd="$cmd \
  --remote_file ${remote_url[$j]} ${fsize[$j]} ${md5[$j]}"
  let j=$j+1
done

cmd="$cmd \
      $(basename $JobInput) \
      $RTE_FILE \
      $(basename $LRMS_JOB_SCRIPT)"
echo $cmd >> $LRMS_JOB_BOINC
echo 'ret=$?' >>$LRMS_JOB_BOINC

echo 'exit $ret' >>$LRMS_JOB_BOINC
if [ $DEBUG -eq 2 ];then
  cat $LRMS_JOB_BOINC
else
   sh $LRMS_JOB_BOINC 1>&2 >/tmp/log.boinc
fi
rc=$?

if [ $rc -eq 0 ];then
  echo "job $wu submitted successfully!" 1>&2
  echo "joboption_jobid=$wu" >> $arg_file
fi
echo "----- removing intermediate files ----" 1>&2
if [ $DEBUG -ne 1 ];then
  rm -fr $WU_TEMPLATE_tmp
  rm -fr $LRMS_JOB_BOINC $LRMS_JOB_ERR $LRMS_JOB_OUT $LRMS_JOB_SCRIPT
  rm -fr $JobInput
fi

echo "----- exiting submit_boinc_job -----" 1>&2
echo "" 1>&2
exit $rc