python-support 1.0.15 / usr / sbin / update-python-modules

#! /usr/bin/python
#
# copyright (c) 2006 Josselin Mouette <joss@debian.org>
# Licensed under the GNU Lesser General Public License, version 2.1
# See COPYING for details

# Everything prefixed by old_ is compatibility code with older versions
# Modules used to lie in /usr/{lib,share}/python-support/$package
# They now lie in /usr/{lib,share}/pyshared

import sys,os,shutil
from optparse import OptionParser
from subprocess import call
from py_compile import compile, PyCompileError
sys.path.append("/usr/share/python-support/private/")
import pysupport
from pysupport import py_supported,py_installed,py_oldversions

basepath='/usr/lib/pymodules'
sourcepath='/usr/share/python-support'
old_extensionpath='/usr/lib/python-support'
shared_path='/usr/share/pyshared'
shared_extensionpath='/usr/lib/pyshared'

parser = OptionParser(usage="usage: %prog [-v] [-c] package_directory [...]\n"+
                            "       %prog [-v] [-c] package.dirs [...]\n"+
                            "       %prog [-v] [-a|-f|-p]")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
                  help="verbose output", default=False)
parser.add_option("-c", "--clean", action="store_true", dest="clean_mode",
                  help="clean modules instead of compiling them",
                  default=False)
parser.add_option("-a", "--rebuild-all", action="store_true",
                  dest="rebuild_all", default=False,
                  help="rebuild all private modules for a new default python version")
parser.add_option("-f", "--force-rebuild-all", action="store_true",
                  dest="rebuild_everything", default=False,
                  help="rebuild all modules, including public modules for all python versions")
parser.add_option("-p", "--post-install", action="store_true", dest="post_install",
                  help="run post-installation operations, common to many packages",
                  default=False)
parser.add_option("-b", "--bytecompile", action="store_true", dest="force_private",
                  help="[deprecated] byte-compilation mode: only handle private modules",
                  default=False)
parser.add_option("-i", "--install", action="store_true", dest="force_public",
                  help="[deprecated] installation mode: only handle public modules",
                  default=False)
(options, args) = parser.parse_args()

def debug(x):
    if(options.verbose):
        print x

def warning(x):
    sys.stderr.write("WARNING: %s\n"%x)

def isect(l1,l2):
    return [i for i in l1 if i in l2]

def concat(l1,l2):
    return l1 + [i for i in l2 if i not in l1]


# Abstract class implementing the methods related to public modules
class _PublicList (list):
    pyversions = py_supported
    def install (self, versions):
        versions = isect (self.pyversions, versions)
        for filename in self:
            version = None
            rng = versions
            try:
                if filename.startswith (shared_path+"/"):
                    # New layout, module
                    relname = filename[len(shared_path)+1:]
                elif filename.startswith (shared_extensionpath+"/python"):
                    # New layout, extension
                    [ version, relname ] = filename[len(shared_extensionpath)+1:].split("/", 1)
                elif filename.startswith (sourcepath+"/"):
                    [ package, relname ] = filename[len(sourcepath)+1:].split("/",1)
                elif filename.startswith (old_extensionpath+"/"):
                    [ package, version, relname ] = filename[len(old_extensionpath)+1:].split("/",2)
                else:
                    raise ValueError
            except ValueError:
                warning ("%s contains an invalid filename (%s)"%(self.name, filename))
                continue
            if version:
                if version not in versions:
                    continue
                rng = [version]
            for pyversion in rng:
                destpath = os.path.join (basepath, pyversion, relname)
                try:
                    os.makedirs(os.path.dirname(destpath))
                except OSError:
                    pass
                if filename[-4:] not in ['.pyc', '.pyo']:
                    debug("link "+destpath)
                    # os.path.exists returns False for broken symbolic links
                    if os.path.exists(destpath) or os.path.islink(destpath):
                        if file!="__init__.py" or (os.path.exists(destpath) and os.path.getsize(destpath)):
                            # The file is already here, probably from the previous version. 
                            # No need to check for conflicts, dpkg catches them earlier now
                            debug("overwrite "+destpath)
                        else:
                            debug("overwrite namespace "+destpath)
                        if os.path.isdir(destpath):
                            shutil.rmtree(destpath)
                        else:
                            os.remove(destpath)
                    os.symlink(filename,destpath)


# Abstract class implementing the methods related to private modules
class _PrivateList (list):
    pyversion = None
    def bytecompile (self):
        if self.pyversion:
            debug("Byte-compilation of whole %s with python%s..."%(self.name,self.pyversion))
            call(['/usr/bin/python'+self.pyversion, 
                  os.path.join('/usr/lib','python'+self.pyversion,'py_compile.py')]
                 + self)
        else:
            for filename in self:
                debug("compile "+filename+'c')
                try:
                    # Note that compile doesn't raise PyCompileError by default
                    compile(filename, doraise=True)
                except IOError, (errno, strerror):
                    warning("I/O error while trying to byte-compile %s (%s): %s" % (filename, errno, strerror))
                except PyCompileError, inst:
                    warning("compile error while trying to byte-compile %s: %s" % (filename, inst.msg))
                except:
                    warning("unexpected error while trying to byte-compile %s: %s" % (filename, sys.exc_info()[0]))
    def clean(self):
        for filename in self:
            for ext in ['c', 'o']:
                fullpath=filename+ext
                if os.path.exists(fullpath):
                    debug("remove "+fullpath)
                    os.remove(fullpath)


# Abstract class for PrivateFileList and SharedFileList
class _FileList(list):
    def __init__ (self, path):
        self.name = path
        for line in file(path):
            line = line.strip()
            if (not line) or line.startswith('#'):
                continue
            if line.startswith('/'):
                self.append(line)
                continue
            line = [x.strip() for x in line.split('=',1)]
            if len(line) != 2:
                warning("Parse error in %s"%path)
                continue
            self.parse_option(*line)

# This class represents a file list as provided in the /usr/share/python-support/$package.public
# Useful for public modules and extensions
class SharedFileList(_FileList, _PublicList):
    def parse_option (self, arg, value):
        if arg=='pyversions':
            self.pyversions = pysupport.version_list(value)
        # Ignore unknown arguments for extensivity

# This class represents a file list as provided in the /usr/share/python-support/$package.private
# Useful for private modules
class PrivateFileList(_FileList, _PrivateList):
    def parse_option (self, arg, value):
        if arg=='pyversion':
            self.pyversion = value

# This is a helper generator that goes through files of interest in a given directory
def allfiles(path, onlypy=False):
    for root, dirs, files in os.walk(path):
        for f in files:
            if (onlypy and not f.endswith(".py")) or f== ".version":
                continue
            yield os.path.join(root,f)
        if not onlypy:
            for d in dirs:
                d = os.path.join(root, d)
                if os.path.islink(d):
                    yield d

# This class emulates the file listing as provided by /usr/share/python-support/$package.public
# with the deprecated layout /usr/{lib,share}/python-support/$package/
class SharedDirList(_PublicList):
    def __init__ (self, path):
        self.name = path
        # Add all files to the file listing
        self.extend(allfiles(path))
        verfile=os.path.join(path,'.version')
        extdir=path.replace(sourcepath,old_extensionpath,1)
        if os.path.isfile(verfile):
            # If we have a .version, use it
            self.pyversions = pysupport.version_list(file(verfile).readline())
        elif os.path.isdir(extdir):
            # Try to obtain the list of supported versions
            # from the extensions in /usr/lib
            self.pyversions = isect(py_supported,os.listdir(extdir))
        else:
            # Otherwise, support all versions
            pass

        if os.path.isdir(extdir):
            # Add the extensions to the file listing
            for version in self.pyversions:
                self.extend(allfiles(os.path.join(extdir,version)))

# This class emulates the file listing as provided by /usr/share/python-support/$package.private
# with the deprecated layout /usr/share/python-support/$package.dirs
class PrivateDirList(_PrivateList):
    def __init__ (self, path):
        self.name = path
        self.extend(allfiles(path, onlypy=True))
        versionfile = os.path.join(path, ".pyversion")
        if os.path.isfile(versionfile):
            self.pyversion = file(versionfile).readline().strip()


class CachedFileList(dict):
    def __getitem__ (self, name):
        if name in self and dict.__getitem__(self, name) == None:
            if name.startswith("/"):
                # The case of old-style private directories
                self[name] = PrivateDirList (name)
            else:
                path = os.path.join (sourcepath, name)
                if name.endswith(".public"):
                    self[name] = SharedFileList (path)
                elif name.endswith(".private"):
                    self[name] = PrivateFileList (path)
                elif os.path.isdir(path):
                    self[name] = SharedDirList (path)
                else:
                    raise Exception("[Internal Error] I don't know what to do with this path: %s"%path)
        return dict.__getitem__(self, name)


def bytecompile_all(py,path=None):
    if not path:
        path=os.path.join(basepath,py)
    if not os.path.isdir(path):
        return
    debug("Byte-compilation of whole %s..."%path)
    os.spawnl(os.P_WAIT, '/usr/bin/'+py, py,
              os.path.join('/usr/lib/',py,'compileall.py'), '-q', path)

# A function to create the ".path" at the root of the installed directory
# Returns the list of affected directories
def create_dotpath(py):
  path=os.path.join(basepath,py)
  if not os.path.isdir(path):
    return
  pathfile=os.path.join(path,".path")
  debug("Generation of %s..."%pathfile)
  pathlist=[path]
  ret=[]
  for f in os.listdir(path):
    f=os.path.join(path,f)
    if f.endswith(".pth") and os.path.isfile(f):
      for l in file(f):
        l=l.rstrip('\n')
        if l.startswith('import'):
          # Do not ship lines starting with "import", they are executed! (complete WTF)
          continue
        pathlist.append(l)
        l2=os.path.join(path,l)
        pathlist.append(l2)
        ret.append(l2)
  fd=file(pathfile,"w")
  fd.writelines([l+'\n' for l in pathlist])
  fd.close()
  return ret

def post_change_stuff(py):
  # All the changes that need to be done after anything has changed
  # in a /usr/lib/pymodules/pythonX.Y directory
  # * Cleanup of all dangling symlinks that are left out after a package
  #   is upgraded/removed.
  # * The namespace packages are here because python doesn't consider a
  #   directory to be able to contain packages if there is no __init__.py
  #   file (yes, this is completely stupid).
  # * The .path file must be created by concatenating all those .pth
  #   files that extend sys.path (this also badly sucks).
  # * Byte-compilation of all .py files that haven't already been
  path=os.path.join(basepath,py)
  if not os.path.isdir(path):
    return
  # First, remove any dangling symlinks.
  # In the same loop, we find which directories may need a namespace package
  dirhash={}
  for dir, dirs, files in os.walk(path):
    dirhash[dir]=False
    files.sort() # We need the .py to appear before the .pyc
    for f in files+dirs:
      # We also examine dirs as some symlinks are dirs
      abspath=os.path.join(dir,f)
      islink=os.path.islink(abspath)
      if islink:
        if not os.path.exists(abspath):
          # We refer to a file that was removed
          debug("remove "+abspath)
          os.remove(abspath)
          continue
        srcfile = os.readlink (abspath)
        # Remove links left here after a change in the supported python versions for a package
        removed = False
        for package in public_packages:
          if srcfile in public_packages[package]:
            if py not in public_packages[package].pyversions:
              debug("remove "+abspath)
              os.remove(abspath)
              removed = True
            break
        else:
          # Remove files provided by packages that do not use python-support anymore
          debug("remove "+abspath)
          os.remove(abspath)
          removed = True
        if removed:
          # Do not go further, the file was removed
          continue
      if f[-4:] in ['.pyc', '.pyo']:
        if not os.path.exists(abspath[:-1]):
          debug("remove "+abspath)
          os.remove(abspath)
          continue
      elif f[-3:] in ['.py', '.so']:
        if islink or f!='__init__.py':
          # List the directory as maybe needing a namespace packages
          d=dir
          while dirhash.has_key(d) and not dirhash[d]:
            dirhash[d]=True
            d=os.path.dirname(d)
    # Remove the directory if it is empty after our crazy removals
    try:
      os.removedirs(dir)
    except OSError:
      pass
  dirhash[path]=False
  # Then, find which directories belong in a .pth file
  # These directories don't need a namespace package, so we
  # set them to False in dirhash
  for p in create_dotpath (py):
    dirhash[p] = False
  # Finally, create/remove namespace packages
  for dir in dirhash:
    initfile=os.path.join(dir,"__init__.py")
    noinitfile=os.path.join(dir,".noinit")
    if dirhash[dir] and not os.path.exists(noinitfile):
      if not os.path.exists(initfile):
        debug("create namespace "+initfile)
        file(initfile,"w").close()
    else:
      for e in ['','c','o']:
        if os.path.exists(initfile+e):
          debug('remove namespace '+initfile+e)
          os.remove(initfile+e)
      try:
        os.removedirs(dir)
      except OSError:
        pass
  bytecompile_all(py)


# A helper function for older $package.dirs files
def dirlist_file(f):
    return [ l.rstrip('\n') for l in file(f) if len(l)>1 ]

# End of function definitions - Start of the script itself

# Ensure that the umask is sane
os.umask(022)

# Read all modules listing
public_packages = CachedFileList()
private_packages = CachedFileList()
dirlisting = os.listdir(sourcepath)
for name in dirlisting:
    path=os.path.join(sourcepath,name)
    if name == "private":
        continue
    ext = name.split(".")[-1]
    if os.path.isdir(path):
        if ext in ["public", "private", "dirs"]:
            # Presumably a bogus directory, see #528130
            warning("%s is a directory"%name)
        else:
            public_packages[name] = None
        continue
    if not os.path.isfile(path):
        # Ignore whatever is not a file, like dangling symlinks
        continue
    if ext == "public":
        public_packages[name] = None
    elif ext == "private":
        private_packages[name] = None
    elif ext == "dirs":
        for dirname in dirlist_file (path):
            private_packages[dirname] = None
    # Just ignore all other files

# Parse arguments
do_public=[]
do_private=[]
for arg in args:
    if arg.startswith(sourcepath):
        arg = arg[len(sourcepath):].lstrip("/")
    if arg.endswith(".dirs") and arg in dirlisting:
        for dirname in dirlist_file(os.path.join(sourcepath, arg)):
            do_private.append(private_packages[dirname])
    elif arg in public_packages:
        do_public.append(public_packages[arg])
    elif arg in private_packages:
        do_private.append(private_packages[arg])
    else:
        if options.clean_mode:
            warning("%s does not exist.\n         Some bytecompiled files may be left behind."%arg)
        else:
            parser.error("%s is not a recognized python-support module."%arg)

# Check consistency options (although these ones should not exist anymore)
if do_private and options.force_public:
    parser.error("Option -i cannot be used with a .private module file.")
if do_public and options.force_private:
    parser.error("Option -b cannot be used with a .public module file.")

if options.rebuild_everything:
    options.rebuild_all = True
    for pyver in py_supported:
        dir = os.path.join(basepath,pyver)
        if os.path.isdir(dir):
            shutil.rmtree(dir)

# Check for changes in installed python versions
need_postinstall = []
for pyver in py_oldversions+py_supported:
    dir = os.path.join(basepath,pyver)
    # Check for ".path" because sometimes the directory already exists 
    # while the python version isn't installed, because of some .so's.
    if pyver not in py_installed and os.path.isdir(dir):
        debug("Removing obsolete directory %s..."%(dir))
        shutil.rmtree(dir)
    if pyver in py_installed and not os.path.isfile(os.path.join(dir,".path")):
        need_postinstall.append(pyver)
if need_postinstall:
    debug("Building all modules for %s..."%(" ".join(need_postinstall)))
    for package in public_packages:
        public_packages[package].install(need_postinstall)
    for pyver in need_postinstall:
        # Here we need to launch create_dotpath because otherwise we could
        # end up without the .path file that is checked 6 lines earlier
        create_dotpath(pyver)

if options.rebuild_all:
    for package in private_packages:
        private_packages[package].bytecompile()


# Now for the processing of what was handed on the command line
for package in do_private:
    if not options.clean_mode:
        package.bytecompile()
    else:
        package.clean()

need_dotpath = False
for package in do_public:
    need_postinstall = concat (need_postinstall, isect(package.pyversions,py_installed))
    if options.clean_mode:
        continue
    package.install(py_installed)
    for f in package:
        if f.endswith(".pth"):
            need_dotpath = True

# Only do the funny and time-consuming things when the -p option is
# given, e.g when python-support is triggered.
if need_postinstall and 'DPKG_RUNNING_VERSION' in os.environ and not options.post_install:
    ret = os.spawnlp(os.P_WAIT, 'dpkg-trigger', 'dpkg-trigger', '--no-await', 'pysupport')
    if ret:
        sys.stderr.write("ERROR: dpkg-trigger failed\n")
        sys.exit(1)
    if need_dotpath:
        for py in need_postinstall:
            create_dotpath (py)
    need_postinstall = []

if options.post_install:
    # The trigger has been activated; do it for all installed versions
    need_postinstall = py_installed
if need_postinstall:
    need_dotpath = False
    for py in need_postinstall:
        post_change_stuff(py)