/usr/lib/python2.7/dist-packages/pyflow/pyflowConfig.py is in python-pyflow 1.1.14-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | #
# pyFlow - a lightweight parallel task engine
#
# Copyright (c) 2012-2015 Illumina, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
"""
pyflowConfig
This file defines a class instance 'siteConfig' containing pyflow components
which are the most likely to need site-specific configuration.
"""
import os
# this is the only object pyflow needs to import, it
# is defined at the end of this module:
#
siteConfig = None
# depending on network setup getfqdn() can be slow, so cache calls to this function here:
#
def _getHostName() :
import socket
return socket.getfqdn()
cachedHostName = None
def getHostName() :
global cachedHostName
if cachedHostName is None :
cachedHostName = _getHostName()
return cachedHostName
def getDomainName() :
hn = getHostName().split(".")
if len(hn) > 1 : hn = hn[1:]
return ".".join(hn)
class DefaultSiteConfig(object) :
"""
Default configuration settings are designed to work with as
many sites as technically feasible
"""
# All email notifications come from the following e-mail address
#
mailFrom = "pyflow-bot@" + getDomainName()
# Default memory (in megabytes) requested by each command task:
#
defaultTaskMemMb = 2048
# In local run mode, this is the defalt memory per thread that we
# assume is available:
#
defaultHostMemMbPerCore = 2048
# maximum number of jobs which can be submitted to sge at once:
#
# allowed values are "unlimited" or None for unlimited jobs, or
# a positive integer describing the max number of jobs
#
maxSGEJobs = 128
# both getHostName and getDomainName are used in the
# siteConfig factory, so these are not designed to be
# overridden at present:
getHostName = staticmethod(getHostName)
getDomainName = staticmethod(getDomainName)
@classmethod
def qsubResourceArg(cls, nCores, memMb) :
"""
When a task is launched using qsub in sge mode, it will call this
function to specify the requested number of threads and megabytes
of memory. The returned argument list will be appended to the qsub
arguments.
nCores -- number of threads requested
memMb -- memory requested (in megabytes)
"""
nCores = int(nCores)
memMb = int(memMb)
return cls._qsubResourceArgConfig(nCores, memMb)
@classmethod
def _qsubResourceArgConfig(cls, nCores, memMb) :
"""
The default function is designed for maximum
portability -- it just provides more memory
via more threads.
"""
# this is the memory we assume is available per
# thread on the cluster:
#
class Constants(object) : megsPerCore = 4096
memCores = 1 + ((memMb - 1) / Constants.megsPerCore)
qsubCores = max(nCores, memCores)
if qsubCores <= 1 : return []
return ["-pe", "threaded", str(qsubCores)]
@classmethod
def getSgeMakePrefix(cls, nCores, memMb, schedulerArgList) :
"""
This prefix will be added to ' -C directory', and run from
a local process to handle sge make jobs.
Note that memMb hasn't been well defined for make jobs yet,
is it the per task memory limit? The first application to
accually make use of this will have to setup the convention,
it is ignored right now...
"""
nCores = int(nCores)
memMb = int(memMb)
retval = ["qmake",
"-V",
"-now", "n",
"-cwd",
"-N", "pyflowMakeTask"]
# user arguments to run() (usually q specification:
retval.extend(schedulerArgList)
#### use qmake parallel environment:
# retval.extend(["-pe","make",str(nCores),"--"])
#### ...OR use 'dynamic' sge make environment:
retval.extend(["--", "-j", str(nCores)])
return retval
def getEnvVar(key) :
if key in os.environ : return os.environ[key]
return None
class hvmemSGEConfig(DefaultSiteConfig) :
"""
This config assumes 'h_vmem' is defined on the SGE instance
"""
@classmethod
def _qsubResourceArgConfig(cls, nCores, memMb) :
retval = []
# specify memory requirements
memGb = 1 + ((memMb - 1) / 1024)
reqArg = "h_vmem=%iG" % (memGb)
retval.extend(["-l", reqArg])
if nCores > 1 :
retval.extend(["-pe", "threaded", str(nCores)])
return retval
#
# final step is the selection of this run's siteConfig object:
#
def siteConfigFactory() :
# TODO: add an import time test to determine if h_vmem is valid
return hvmemSGEConfig
siteConfig = siteConfigFactory()
|