openstack-pkg-tools 54 / usr / bin / pkgos-parse-requirements

#!/bin/sh

set -e

if ! [ -r /etc/pkgos/pkgos.conf ] ; then
	echo "Could not read /etc/pkgos/pkgos.conf"
	exit 1
else
	. /etc/pkgos/pkgos.conf
fi

if [ "${1}" = "-h" ] ; then
	echo "This utility attemps to parse an OpenStack requirements.txt file"
	echo "as input, and produce a list of Debian dependencies as output."
	echo "Note that this is far from perfect, and that you *WILL* need to"
	echo "manually check for the dependencies. This is only a helper in"
	echo "order to gain some precious development time."
	echo ""
	echo "If this utility is called without a parameter, it will attempt"
	echo "to read the requirements.txt and test-requirements.txt file."
	echo "Otherwise, it takes the first argument as the file to parse."
	exit 0
fi

# Some packages should never be in the dependencies in Debian,
# as they are included in Python 2.7. If you find one that is
# missing, just add it to the list it here.
BLACK_LIST="discover argparse ordereddict doc8 pylint flake8 pyflakes pep8"
is_blacklisted () {
	ISBLACKLISTED="no"
	for i in $BLACK_LIST ; do
		if [ "${i}" = "${1}" ] ; then
			ISBLACKLISTED="yes"
		fi
	done
}

BLACK_LIST_PY3="python3-testrepository python3-oslosphinx python3-coverage python3-concurrent.futures"
is_py3_blacklisted () {
	IS_PY3BLACKLISTED="no"
	for i in $BLACK_LIST_PY3 ; do
		if [ "${i}" = "${1}" ] ; then
			IS_PY3BLACKLISTED="yes"
		fi
	done
}

# Some packages should never make it into the Build-Depends-Indep:,
# because they are already in Build-Depends:. Here's the list for it.
BUILD_DEPENDS_LIST="sphinx pbr"
is_build_depends () {
	ISBUILDDEPENDS="no"
	for i in ${BUILD_DEPENDS_LIST} ; do
		if [ "${i}" = ${1} ] ; then
			ISBUILDDEPENDS="yes"
		fi
	done
}

EPOC_1_IN="python-cinderclient python-keystoneclient python-glanceclient python-swiftclient python-neutronclient python-oslo.config"
has_1_in_epoc () {
	HAS_1_IN_EPOC="no"
	for i in ${EPOC_1_IN} ; do
		if [ "${i}" = ${1} ] ; then
			HAS_1_IN_EPOC="yes"
		fi
	done
}

EPOC_2_IN="python-novaclient"
has_2_in_epoc () {
	HAS_2_IN_EPOC="no"
	for i in ${EPOC_2_IN} ; do
		if [ "${i}" = ${1} ] ; then
			HAS_2_IN_EPOC="yes"
		fi
	done
}

NO_PYTHON_PREFIX="alembic testrepository subunit websockify cliff-tablib"
is_python_prefixed () {
	PY_PREFIX="yes"
	for i in ${NO_PYTHON_PREFIX} ; do
		if [ "${i}" = "${1}" ] ; then
			PY_PREFIX="no"
		fi
	done
}

# Param: $1: input file
#        $2: if set, then the Build-Depends: programs will be removed (for example: sphinx, pbr, etc.)
parse_and_print () {
	INPUT_FILE=$1
	REMOVE_BUILD_DEPENDS="no"
	if [ "${2}" = "build-depends" ] ; then
		REMOVE_BUILD_DEPENDS="yes"
	fi
	PYVERS=${3}
	DEP_LIST=""
#	echo `cat ${INPUT_FILE} | grep -v '^#' | grep -v '^[ \t]*$' | awk '{print $1}' | tr '[:upper:]' '[:lower:]' | sed $EXP`
	for i in `cat ${INPUT_FILE} | grep -v '^#' | grep -v '^[ \t]*$' | awk '{print $1}' | tr '[:upper:]' '[:lower:]' | sed $EXP` ; do
#		echo "Line ---> $i"
		if echo $i | grep -q -e '^http://' ; then
			i=`echo $i | cut -d'=' -f2`
			VERS=""
		else
			TRIM_VERS=`echo $i | sed -e 's/^[-a-zA-Z0-9._]*//'`
			VERS=`echo $TRIM_VERS | sed -e 's/^[-a-zA-Z0-9._]*//' | tr ',|;' '\n' | sort | tr '\n' ',' | sed -e 's/,$//'`
		fi
		if [ -n "$VERS" ] ; then
			PKG=`echo $i | sed -e "s/${TRIM_VERS}//" | sed -e s/python-//`
		else
			PKG=`echo $i | sed -e s/python-//`
		fi
		PKG=`echo ${PKG} | sed -e s/_/-/g`
		is_blacklisted ${PKG}
		ISBUILDDEPENDS="no"
		if [ REMOVE_BUILD_DEPENDS="yes" ] ; then
			is_build_depends ${PKG}
		fi
		if [ $ISBLACKLISTED = "no" ] && [ ${ISBUILDDEPENDS} = "no" ]; then
			is_python_prefixed ${PKG}
			if [ ${PY_PREFIX} = "yes" ] ; then
				PKG=python-${PKG}
			fi
			# Convert the package name into lowercase, as Debian
			# doesn't have any upper case in package names
			PKG=`echo ${PKG} | tr '[:upper:]' '[:lower:]'`
			if [ -n "$VERS" ] && [ ${PKG} != "python-hacking" ] ; then
				# If there's a a version-depends, convert the pip style
				# of dependency to the Debian one (ie: >> instead of >)
				FIRST_CONSTR=`echo $VERS | cut -d, -f1`
				FIRST_NUMS=`echo $FIRST_CONSTR | sed -e 's/[<>=\!]*//'`
				FIRST_SIGN=`echo $FIRST_CONSTR | sed -e "s/${FIRST_NUMS}//"`
				if [ "${FIRST_SIGN}" = '<' ] ; then
					FIRST_SIGN='<<'
				fi
				if [ "${FIRST_SIGN}" = '>' ] ; then
					FIRST_SIGN='>>'
				fi
				has_1_in_epoc ${PKG}
				if [ "${HAS_1_IN_EPOC}" = "yes" ] ; then
					FIRST_NUMS="1:${FIRST_NUMS}"
				fi
				has_2_in_epoc ${PKG}
				if [ "${HAS_2_IN_EPOC}" = "yes" ] ; then
					FIRST_NUMS="2:${FIRST_NUMS}"
				fi
				# If there's a fake-jessie-mirror folder in /etc/pkgos
				# use that one with madison-lite to check if the version
				# of the package is already in Jessie.
				if [ -d /etc/pkgos/fake-jessie-mirror ] ; then
					STABLE_VERSION=`madison-lite -a all,amd64 --mirror /etc/pkgos/fake-${TARGET_DISTRO}-mirror ${PKG} | awk '{print $3}'`
					# Make sure that the package is in the stable repo
					if [ -z "${STABLE_VERSION}" ] ; then
						VERSION_TO_DEPEND_ON=" (${FIRST_SIGN} ${FIRST_NUMS})"
					else
						#echo "Comparing for ${PKG}: dpkg --compare-versions ${STABLE_VERSION} gt ${FIRST_NUMS}"
						if dpkg --compare-versions ${STABLE_VERSION} gt ${FIRST_NUMS} ; then
							VERSION_TO_DEPEND_ON=""
						else
							VERSION_TO_DEPEND_ON=" (${FIRST_SIGN} ${FIRST_NUMS})"
						fi
					fi
				else
					VERSION_TO_DEPEND_ON=" (${FIRST_SIGN} ${FIRST_NUMS})"
				fi
				if [ "${PYVERS}" = 3 ] ; then
					PKG=`echo ${PKG} | sed s/python-/python3-/`
					is_py3_blacklisted ${PKG}
					if [ ${IS_PY3BLACKLISTED} = "yes" ] ; then
						continue
					fi
				fi
				if [ -z "${DEP_LIST}" ] ; then
					DEP_LIST="${PKG}${VERSION_TO_DEPEND_ON}"
					#echo " ${PKG} (${FIRST_SIGN} ${FIRST_NUMS}),"
				else
					DEP_LIST="${DEP_LIST}\n${PKG}${VERSION_TO_DEPEND_ON}"
				fi
			else
				if [ -z "${DEP_LIST}" ] ; then
					DEP_LIST="${PKG}"
				else
					#echo " ${PKG},"
					DEP_LIST="${DEP_LIST}\n${PKG}"
				fi
			fi
			#echo "Package: ${PKG}\t\tFirst sign: ${FIRST_SIGN}\t\tFirst num: ${FIRST_NUMS}..."
		fi
	done
}

# Param: $DEPS: the dependency list, one package per line
#        $1: the word for the package dep (ie: Depends: or Build-Depends-Indep:)
format_output () {
#	set -x
	SPACES_IN_FRONT=`echo "${1} " | sed -e 's/[a-zBDI:-]/ /g'`
	CNT="0"
	echo $DEPS | LC_COLLATE=C sort -u | while read i ; do
		if [ "${CNT}" = "0" ] ; then
			echo "${1} ${i},"
		else
			echo -n "${1}" | sed -e 's/[a-zBDI:-]/ /g'
			echo " ${i},"
		fi
		CNT=$(($CNT + 1))
	done
	if [ $1 = "Depends:" ] ; then
		echo "         \${misc:Depends},"
		echo "         \${python:Depends},"
	fi
}

calc_substitue_list () {
	if [ -r /etc/pkgos/substitute ] ; then
		while read i ; do
			SOURCE=`echo $i | cut -d" " -f1`
			DEST=`echo $i | cut -d" " -f2`
			EXP="$EXP -e s/$SOURCE/$DEST/"
		done </etc/pkgos/substitute
	fi
#	echo \'$EXP\'
}

calc_substitue_list

if [ "${1}" ] ; then
	parse_and_print ${1}
	format_output "Build-Depends:"
else
	# Check if we declared Python3 support in debian/control
	HAS_PY3_SUPPORT="no"
	if [ -r debian/control ] ; then
		HAS_PYTHON3_PKG_IN_CTRL=`cat debian/control | grep "Package: python3" || true`
		if [ -n "${HAS_PYTHON3_PKG_IN_CTRL}" ] ; then
			HAS_PY3_SUPPORT="yes"
		fi
	fi
	# If there's a debian/*.init.in, then we want dh-systemd as build-depends
	INIT=`find debian/ -iname '*init.in'`
	if [ -n "${INIT}" ] ; then
		DH_SYSTEMD="\n               dh-systemd,"
		OSTACK_PKG_TOOLS_VERS=" (>= 23~)"
	else
		DH_SYSTEMD=""
		OSTACK_PKG_TOOLS_VERS=""
	fi
	TEMPLATES=`find debian/ -iname '*.templates'`
	if [ -n "${TEMPLATES}" ] ; then
		PO_DEBCONF="\n               po-debconf,"
	else
		PO_DEBCONF=""
	fi
	echo "Build-Depends: debhelper (>= 9),
               dh-python,${DH_SYSTEMD}
               openstack-pkg-tools${OSTACK_PKG_TOOLS_VERS},${PO_DEBCONF}
               python-all,
               python-pbr,
               python-setuptools,
               python-sphinx,"
	# Adds python3 support packages
	if [ "${HAS_PY3_SUPPORT}" = "yes" ] ; then
		echo "               python3-all,
               python3-pbr,
               python3-setuptools,"
	fi

	# Gather the dependencies.
	if [ -e test-requirements-py2.txt ] ; then
		parse_and_print test-requirements-py2.txt build-depends 2
	else
		parse_and_print test-requirements.txt build-depends 2
	fi
	DEP_LIST_TESTS=${DEP_LIST}
	if [ -r requirements-py2.txt ] ; then
		parse_and_print requirements-py2.txt depends
	else
		parse_and_print requirements.txt depends
	fi
	DEP_LIST_RUNTIME=${DEP_LIST}

	# If the package has python3 support, let's try to also print python3 build-depends
	if [ "${HAS_PY3_SUPPORT}" = "yes" ] ; then
		parse_and_print requirements.txt depends 3
		DEP_LIST_TESTS="${DEP_LIST_TESTS}\n${DEP_LIST}"
		if [ -e test-requirements-py3.txt ] ; then
			parse_and_print test-requirements-py3.txt build-depends 3
		else
			parse_and_print test-requirements.txt build-depends 3
		fi
		DEP_LIST_TESTS="${DEP_LIST_TESTS}\n${DEP_LIST}"
	fi

	if [ -r .testr.conf ] ; then
		DEP_LIST_TESTS="${DEP_LIST_TESTS}\ntestrepository\nsubunit"
		if [ -r debian/control ] && [ -n "${HAS_PYTHON3}" ] ; then
			DEP_LIST_TESTS="${DEP_LIST_TESTS}\npython3-subunit"
		fi
	fi

	# Format the output
	DEPS="${DEP_LIST_RUNTIME}\n${DEP_LIST_TESTS}"
	format_output "Build-Depends-Indep:"

	DEPS=${DEP_LIST_RUNTIME}
	format_output "Depends:"
fi