/usr/lib/condor/libexec/linux_kernel_tuning is in htcondor 8.6.8~dfsg.1-2.
This file is owned by root:root, with mode 0o755.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | #!/bin/sh
##
## This script tunes Linux kernel parameters useful for scaling HTCondor.
## Some values may be inappropriate for machines not dedicated to HTCondor.
## This script must be run as root, and we assume that (and support only)
## HTCondor is running as root for large installations.
##
## The next '##' sections marks the beginning of the implementation section
## of this script, and you should not need to change anything past that mark.
##
#
# This script logs its actions to syslog. If you'd rather it didn't,
# change the line below to something like '/bin/true'.
#
LOGGER='/usr/bin/logger -t htcondor'
#
# Increase the global number of file descriptors. For example, each
# dynamically-linked shadow uses approximately 13 just to open its
# libraries.
#
# We don't set the per-process maximum, because HTCondor will do that
# appropriately for each different subsystem (daemon) when run as root.
# If necessary, you can change those values using the configuration variables
# MAX_FILE_DESCRIPTORS and <SUBSYS>_MAX_FILE_DESCRIPTORS.
#
GLOBAL_MAX_FDS=32768
#
# Increase the maximum process ID. By default, Linux process IDs wrap at
# 32768, which isn't a lot of processes when you have one per running job.
#
GLOBAL_MAX_PROCESSES=4194303
#
# Unless otherwise specified, an outbound connection uses a port within
# this range. On some systems, this default to 1024-4999, which may not
# be enough. You can also force HTCondor to specify a particular
# outbound port by setting the configuration variables OUT_LOWPORT and
# OUT_HIGHPORT appropriately, but this will be slower under load, as
# HTCondor has to search for an open port.
#
LOCAL_PORT_RANGE="1024 65535"
#
# Increase the length of the TCP listen queue. This allows more connections
# to pile up while HTCondor is otherwise busy.
#
TCP_LISTEN_QUEUE=1024
#
# Likewise, the central manager (collector) needs have large UDP buffers.
# Increase the maximum allowed size of networks receive buffers.
#
MAX_RECEIVE_BUFFER=10485760
# Maximum amount of dirty filesystem bytes to buffer in the kernel
# before processes writing to the filesystem are blocked, and made
# to do their own i/o synchronously. Set to 0 to undo.
# If we are running with cgroups on, there's a Linux kernel bug
# that causes spurious OOM events sent to a cgroup with a hard memory limit
# if it writes a lot of data to the filesystem quickly. Limitting
# the buffering to 100M works around the problem
DIRTY_BYTES=100000000
# Set root user quota for max number of kernel session keys and
# bytes. Kernel session keys are used by Kerberos, AFS, ecryptfs, and
# other services. The condor_master creates a new session keyring by
# default on startup (via config knob DISCARD_SESSION_KEYRING_ON_STARTUP)
# so that Kerberos tokens and the like are not leaked to user jobs, and
# also to support encrypted execute directories. The default quota
# starting with RHEL 6.4 is 1M keys and 25M bytes, which is plenty
# big. But older kernels (like what shipped in RHEL 6.0) have a crazy
# small quota of 200 keys. So here we just set these params to what
# RHEL 6.4+ use by default just in case people try to use HTCondor on
# an earlier kernel.
ROOT_MAXKEYS=1000000
ROOT_MAXKEYS_BYTES=25000000
##
## Implementation. You shouldn't need to change anything below here.
##
LOG=/dev/null
if [ -d /etc/sysctl.d ]; then
LOG=/etc/sysctl.d/99-htcondor.conf
(
echo "#"
echo "# This file was written by $0"
echo "# when the condor_master started up."
echo "#"
echo "# This script tunes kernel parameters to support HTCondor at"
echo "# larger scales. A list of the changes follows. You can set"
echo "# ENABLE_KERNEL_TUNING = FALSE"
echo "# in your HTCondor configuration to disable this entirely, or"
echo "# set LINUX_KERNEL_TUNING_SCRIPT to some other file to change"
echo "# which script is run when the condor_master daemon starts."
echo "#"
) > ${LOG}
fi
increaseKernelParameter() {
PARAMETER=$1
FILE=$2
NEW=$3
if [ -n "${NEW}" ]; then
OLD=`cat ${FILE}`
if [ ${NEW} -gt ${OLD} ]; then
echo "Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}"
echo "# Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}" >> ${LOG}
echo "Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}" | ${LOGGER}
echo ${NEW} > ${FILE}
else
echo "Not changing ${PARAMETER} (${FILE}): new value (${NEW}) <= old value (${OLD})."
echo "# Not changing ${PARAMETER} (${FILE}): new value (${NEW}) <= old value (${OLD})." >> ${LOG}
echo "Not changing ${PARAMETER} (${FILE}): new value (${NEW}) <= old value (${OLD})." | ${LOGGER}
fi
fi
}
setKernelParameter() {
PARAMETER=$1
FILE=$2
NEW=$3
if [ -n "${NEW}" ]; then
OLD=`cat ${FILE}`
echo "Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}"
echo "# Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}" >> ${LOG}
echo "Changing ${PARAMETER} (${FILE}) from ${OLD} to ${NEW}" | ${LOGGER}
echo ${NEW} > ${FILE}
fi
}
increaseKernelParameter "GLOBAL_MAX_FDS" "/proc/sys/fs/file-max" ${GLOBAL_MAX_FDS}
# Set GLOBAL_MAX_PROCESSES and LOCAL_PORT_TRANGE only for schedd machines.
daemonList=`condor_config_val DAEMON_LIST | sed -e's/,/ /g'`
for daemon in $daemonList; do
daemonFile=`condor_config_val ${daemon}`
if [ -n "${daemonFile}" ]; then
baseName=`basename ${daemonFile}`
if [ ${baseName} = "condor_schedd" ]; then
increaseKernelParameter "GLOBAL_MAX_PROCESSES" "/proc/sys/kernel/pid_max" ${GLOBAL_MAX_PROCESSES}
setKernelParameter "LOCAL_PORT_RANGE" "/proc/sys/net/ipv4/ip_local_port_range" "${LOCAL_PORT_RANGE}"
fi
fi
done
increaseKernelParameter "TCP_LISTEN_QUEUE" "/proc/sys/net/core/somaxconn" ${TCP_LISTEN_QUEUE}
increaseKernelParameter "ROOT_MAXKEYS" "/proc/sys/kernel/keys/root_maxkeys" ${ROOT_MAXKEYS}
increaseKernelParameter "ROOT_MAXKEYS_BYTES" "/proc/sys/kernel/keys/root_maxbytes" ${ROOT_MAXKEYS_BYTES}
if condor_config_val BASE_CGROUP > /dev/null 2>&1
then
setKernelParameter "FS_CACHE_DIRTY_BYTES" "/proc/sys/vm/dirty_bytes" "${DIRTY_BYTES}"
fi
# FIXME: Only on the collector.
increaseKernelParameter "MAX_RECEIVE_BUFFER" "/proc/sys/net/core/rmem_max" ${MAX_RECEIVE_BUFFER}
# Mount cgroups if not already mounted
# Check to see if admin has enabled cgroups in condor
if condor_config_val BASE_CGROUP > /dev/null 2>&1
then
# Admin has requested them (or left the default on)
# Check to see if they are already mounted, assume if memory is, they all are
if ! grep -q memory /proc/self/cgroup
then
mkdir -p /cgroup/memory
mount -t cgroup -o memory memory /cgroup/memory
mkdir -p /cgroup/cpu
mount -t cgroup -o cpu cpu /cgroup/cpu
mkdir -p /cgroup/cpuacct
mount -t cgroup -o cpuacct cpuacct /cgroup/cpuacct
mkdir -p /cgroup/blkio
mount -t cgroup -o blkio blkio /cgroup/blkio
mkdir -p /cgroup/freezer
mount -t cgroup -o freezer freezer /cgroup/freezer
fi
fi
exit 0
|