python-popcon 1.3 / usr / lib / python2.7 / dist-packages / popcon.py

#!/usr/bin/env python

# popcon.py -
# Copyright (C) 2010-2015  Bastian Venthur
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

# relevant specifications:
# * BASEDIRSPEC: http://standards.freedesktop.org/basedir-spec/basedir-spec-0.6.html


"""Get Debian popcon values for given packages.

The usage of this module is easy:

    >>> import popcon
    >>> popcon.package('reportbug-ng')
    {'reportbug-ng': 323}
    >>> popcon.package('reportbug-ng', 'reportbug')
    {'reportbug-ng': 323, 'reportbug': 75065}

The raw data (vote, old, recent, no-file) is also available, the sum of
the raw numbers is the number of installations as reported by
`popcon.package`.

    >>> popcon.package_raw('reportbug-ng', 'reportbug')
    {'reportbug-ng': Package(vote=50, old=187, recent=86, no_files=0),
            'reportbug': Package(vote=5279, old=59652, recent=10118,
            no_files=16)}

Behind the scences popcon will try to use cached infomation saved in
`DUMPFILE`. If that file is not available, or older than `EXPIRY`
seconds (default is 7 days) it will download fresh data and save that
into `DUMPFILE`.

The cached data will be kept in memory unless `KEEP_DATA` is set to
False.

"""

from __future__ import division, print_function

import warnings
import time
try:
    # python2
    from urllib2 import Request, urlopen
except ImportError:
    # python3
    from urllib.request import Request, urlopen
import gzip
try:
    # python2
    import StringIO as io
except ImportError:
    # python3
    import io
import tempfile
try:
    # python2
    import cPickle as pickle
except ImportError:
    # python3
    import pickle
import os
import collections

import xdg.BaseDirectory


__author__ = 'Bastian Venthur <venthur@debian.org>'


Package = collections.namedtuple(
    "Package", ["vote", "old", "recent", "no_files"])


# week in seconds
EXPIRY = 86400 * 7
DUMPFILE = os.path.join(
    xdg.BaseDirectory.xdg_cache_home,
     'popcon',
     'debian')  # implements BASEDIRSPEC
RESULTS_URL = "http://popcon.debian.org/all-popcon-results.txt.gz"
KEEP_DATA = True
cached_data = None
cached_timestamp = None


def _fetch():
    """Fetch all popcon results and return unparsed data."""
    request = Request(RESULTS_URL)
    response = urlopen(request)
    txt = response.read()
    response.close()
    txt = _decompress(txt)
    return txt


def _parse(results):
    """Parse all-popcon-results file."""
    ans = dict()
    results = results.splitlines()
    for line in results:
        elems = line.split()
        if elems[0] != "Package:":
            continue
        ans[elems[1]] = Package(*(int(i) for i in elems[2:]))
    return ans


def _decompress(compressed):
    """Decompress a gzipped string."""
    try:
        # python2
        gzippedstream = io.StringIO(compressed)
    except TypeError:
        # python3
        gzippedstream = io.BytesIO(compressed)
    gzipper = gzip.GzipFile(fileobj=gzippedstream)
    data = gzipper.read()
    data = data.decode()
    return data


def package(*packages):
    """Return the number of installations.

    The return value is a dict where the keys are the packages and the
    values the number of installations. If a package was not found it is
    not in the dict.
    """
    raw = package_raw(*packages)
    ans = dict()
    for pkg, values in list(raw.items()):
        ans[pkg] = sum(values)
    return ans


def package_raw(*packages):
    """Return the raw popcon values for the given packages.

    The return value is a dict where the keys are the packages and the
    values a named tuple of integers: (vote, old, recent, no-files)

        vote: number of people who use this package regulary

        old: is the number of people who installed, but don't use this
            package regularly

        recent: is the number of people who upgraded this package
            recently

        no-files: is the number of people whose entry didn't contain
            enough information (atime and ctime were 0)

    """
    global cached_data, cached_timestamp

    earliest_possible_mtime = max(
        time.time() - EXPIRY,
        os.stat(__file__).st_mtime)

    if cached_data is not None and cached_timestamp <= earliest_possible_mtime:
        cached_data = None

    data = cached_data
    if data is None and os.path.exists(DUMPFILE) and os.stat(DUMPFILE).st_mtime > earliest_possible_mtime:
        try:
            with open(DUMPFILE, 'rb') as fh:
                data = pickle.load(fh)
            cached_timestamp = os.stat(DUMPFILE).st_mtime
        except:
            warnings.warn("Problems loading cache file: %s" % DUMPFILE)

    if data is None:
        data = _fetch()
        data = _parse(data)
        if not os.path.isdir(os.path.dirname(DUMPFILE)):  # i still think that makedirs should behave like mkdir -p
            os.makedirs(
                os.path.dirname(DUMPFILE),
                mode=0o700)  # mode according to BASEDIRSPEC
        # as soon as python2.6 is in stable, we can use delete=False
        # here and replace the flush/rename/try:close sequence with the
        # cleaner close/rename.
        temp = tempfile.NamedTemporaryFile(dir=os.path.dirname(DUMPFILE))
        pickle.dump(data, temp)
        temp.flush()
        os.rename(temp.name, DUMPFILE)
        try:
            temp.close()
        except OSError:
            pass
        cached_timestamp = time.time()
    ans = dict()
    for pkg in packages:
        if pkg in data:
            ans[pkg] = data[pkg]
    if KEEP_DATA:
        cached_data = data
    return ans


if __name__ ==  "__main__":
    print(package('reportbug-ng'))
    print(package('reportbug-ng', 'reportbug'))