pyzor 1:1.0.0-2 / usr / bin / pyzor

#!/usr/bin/python

"""Pyzor client."""

from __future__ import print_function

import os
import sys
import email
import random
import mailbox
import hashlib
import getpass
import logging
import optparse
import tempfile
import threading

try:
    import ConfigParser
except ImportError:
    import configparser as ConfigParser

import pyzor.digest
import pyzor.client
import pyzor.config


def load_configuration():
    """Load the configuration for the server.

    The configuration comes from three sources: the default values, the
    configuration file, and command-line options."""
    # Work out the default directory for configuration files.
    # If $HOME is defined, then use $HOME/.pyzor, otherwise use /etc/pyzor.
    userhome = os.getenv("HOME")
    if userhome:
        homedir = os.path.join(userhome, '.pyzor')
    else:
        homedir = os.path.join("/etc", "pyzor")

    # Configuration defaults.  The configuration file overrides these, and
    # then the command-line options override those.
    defaults = {
        "ServersFile": "servers",
        "AccountsFile": "accounts",
        "LocalWhitelist": "whitelist",
        "LogFile": "",
        "Timeout": "5",  # seconds
        "Style": "msg",
        "ReportThreshold": "0",
        "WhitelistThreshold": "0",
    }

    # Process any command line options.
    description = ("Read data from stdin and execute the requested command "
                   "(one of 'check', 'report', 'ping', 'pong', 'digest', "
                   "'predigest', 'genkey', 'local_whitelist', "
                   "'local_unwhitelist').")
    opt = optparse.OptionParser(description=description)
    opt.add_option("-n", "--nice", dest="nice", type="int",
                   help="'nice' level", default=0)
    opt.add_option("-d", "--debug", action="store_true", default=False,
                   dest="debug", help="enable debugging output")
    opt.add_option("--homedir", action="store", default=homedir,
                   dest="homedir", help="configuration directory")
    opt.add_option("-s", "--style", action="store",
                   dest="Style", default=None,
                   help="input style: 'msg' (individual RFC5321 message), "
                        "'mbox' (mbox file of messages), 'digests' (Pyzor "
                        "digests, one per line).")
    opt.add_option("--log-file", action="store", default=None,
                   dest="LogFile", help="name of log file")
    opt.add_option("--servers-file", action="store", default=None,
                   dest="ServersFile", help="name of servers file")
    opt.add_option("--accounts-file", action="store", default=None,
                   dest="AccountsFile", help="name of accounts file")
    opt.add_option("--local-whitelist", action="store", default=None,
                   dest="LocalWhitelist", help="name of the local whitelist "
                   "file")
    opt.add_option("-t", "--timeout", dest="Timeout", type="int",
                   help="timeout (in seconds)", default=None)
    opt.add_option("-r", "--report-threshold", dest="ReportThreshold",
                   type="int", default=None,
                   help="threshold for number of reports")
    opt.add_option("-w", "--whitelist-threshold", dest="WhitelistThreshold",
                   type="int", default=None,
                   help="threshold for number of whitelist")
    opt.add_option("-V", "--version", action="store_true", default=False,
                   dest="version", help="print version and exit")
    options, args = opt.parse_args()

    if options.version:
        print("%s %s" % (sys.argv[0], pyzor.__version__))
        sys.exit(0)

    if not len(args):
        opt.print_help()
        sys.exit()
    try:
        os.nice(options.nice)
    except AttributeError:
        pass

    # Create the configuration directory if it doesn't already exist.
    if not os.path.exists(options.homedir):
        os.mkdir(options.homedir)

    # Load the configuration.
    config = ConfigParser.ConfigParser()
    # Set the defaults.
    config.add_section("client")
    for key, value in defaults.iteritems():
        config.set("client", key, value)
    # Override with the configuration.
    config.read(os.path.join(options.homedir, "config"))
    # Override with the command-line options.
    for key in defaults:
        value = getattr(options, key)
        if value is not None:
            config.set("client", key, str(value))
    return config, options, args


def main():
    """Execute any requested actions."""
    # Set umask - this restricts this process from granting any world access
    # to files/directories created by this process.
    os.umask(0o0077)

    config, options, args = load_configuration()

    homefiles = ["LogFile", "ServersFile", "AccountsFile", "LocalWhitelist"]
    pyzor.config.expand_homefiles(homefiles, "client", options.homedir, config)

    logger = pyzor.config.setup_logging("pyzor",
                                        config.get("client", "LogFile"),
                                        options.debug)
    servers = pyzor.config.load_servers(config.get("client", "ServersFile"))
    accounts = pyzor.config.load_accounts(config.get("client", "AccountsFile"))

    # Run the specified commands.
    client = pyzor.client.Client(accounts,
                                 int(config.get("client", "Timeout")))
    for command in args:
        try:
            dispatch = DISPATCHES[command]
        except KeyError:
            logger.critical("Unknown command: %s", command)
        else:
            try:
                if not dispatch(client, servers, config):
                    sys.exit(1)
            except pyzor.TimeoutError:
                # Note that most of the methods will trap their own timeout
                # error.
                logger.error("Timeout from server in %s", command)


def get_input_handler(style="msg", digester=pyzor.digest.DataDigester):
    """Return an object that can be iterated over to get all the digests."""
    try:
        return INPUT_HANDLERS[style](digester)
    except KeyError:
        raise ValueError("Unknown input style.")


def _get_input_digests(dummy):
    for line in sys.stdin:
        yield line.strip()


def _get_input_msg(digester):
    msg = email.message_from_file(sys.stdin)
    digested = digester(msg).value
    yield digested


def _get_input_mbox(digester):
    tfile = tempfile.NamedTemporaryFile()
    tfile.write(sys.stdin.read().encode("utf8"))
    tfile.seek(0)
    mbox = mailbox.mbox(tfile.name)
    for msg in mbox:
        digested = digester(msg).value
        yield digested
    tfile.close()


def ping(client, servers, config):
    """Check that the server is reachable."""
    # pylint: disable-msg=W0613
    runner = pyzor.client.ClientRunner(client.ping)
    for server in servers:
        runner.run(server, (server,))
    sys.stdout.writelines(runner.results)
    return runner.all_ok


def pong(client, servers, config):
    """Used to test pyzor."""
    rt = int(config.get("client", "ReportThreshold"))
    wt = int(config.get("client", "WhitelistThreshold"))
    style = config.get("client", "Style")
    runner = pyzor.client.CheckClientRunner(client.pong, rt, wt)
    for digested in get_input_handler(style):
        send_digest(digested, runner, servers)
    sys.stdout.writelines(runner.results)

    return runner.all_ok and runner.found_hit and not runner.whitelisted


def info(client, servers, config):
    """Get information about each message."""
    style = config.get("client", "Style")
    runner = pyzor.client.InfoClientRunner(client.info)
    for digested in get_input_handler(style):
        send_digest(digested, runner, servers)
    sys.stdout.writelines(runner.results)

    return runner.all_ok


def check(client, servers, config):
    """Check each message against each server.

    The return value is 'failure' if there is a positive spam count and
    *zero* whitelisted count; otherwise 'success'.
    """
    rt = int(config.get("client", "ReportThreshold"))
    wt = int(config.get("client", "WhitelistThreshold"))
    style = config.get("client", "Style")
    lwhitelist_fp = config.get("client", "LocalWhitelist")
    lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
    runner = pyzor.client.CheckClientRunner(client.check, rt, wt)
    mock_runner = pyzor.client.CheckClientRunner(client._mock_check, rt, wt)
    for digested in get_input_handler(style):
        if digested in lwhitelist:
            send_digest(digested, mock_runner, servers)
        else:
            send_digest(digested, runner, servers)
    sys.stdout.writelines(mock_runner.results)
    sys.stdout.writelines(runner.results)

    return runner.all_ok and runner.found_hit and not runner.whitelisted


def _send_digest(runner, server, digested, spec=None):
    """Send these digests to one server."""
    if spec:
        runner.run(server, (digested, server, spec))
    else:
        runner.run(server, (digested, server))


def send_digest(digested, runner, servers):
    """Send these digests to each server."""
    if not digested:
        return

    if len(servers) == 1:
        _send_digest(runner, servers[0], digested)
        return runner.all_ok

    threads = []
    for server in servers:
        args = (runner, server, digested)
        thread = threading.Thread(target=_send_digest, args=args)
        threads.append(thread)
        thread.start()

    for thread in threads:
        thread.join()

    return runner.all_ok


def report(client, servers, config):
    """Report each message as spam."""
    style = config.get("client", "Style")
    all_ok = True
    for digested in get_input_handler(style):
        runner = pyzor.client.ClientRunner(client.report)
        if digested and not send_digest(digested, runner, servers):
            all_ok = False
        sys.stdout.writelines(runner.results)
    return all_ok


def whitelist(client, servers, config):
    """Report each message as ham."""
    style = config.get("client", "Style")
    all_ok = True
    for digested in get_input_handler(style):
        runner = pyzor.client.ClientRunner(client.whitelist)
        if digested and not send_digest(digested, runner, servers):
            all_ok = False
        sys.stdout.writelines(runner.results)
    return all_ok


def digest(client, servers, config):
    """Generate a digest for each message.

    This method can be used to look up digests in the database when
    diagnosing, or to report digests in a two-stage operation (digest,
    then report with --digests)."""
    style = config.get("client", "Style")
    for digested in get_input_handler(style):
        if digested:
            print(digested)
    return True


def local_whitelist(client, servers, config):
    """Add to the local whitelist."""
    logger = logging.getLogger("pyzor")
    lwhitelist_fp = config.get("client", "LocalWhitelist")
    lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
    style = config.get("client", "Style")
    for digested in get_input_handler(style):
        if digested in lwhitelist:
            logger.critical("Digest %s already whitelisted locally", digested)
        lwhitelist.add(digested)
    with open(lwhitelist_fp, "w") as lwhitelist_f:
        lwhitelist_f.write("\n".join(lwhitelist))
    return True


def local_unwhitelist(client, servers, config):
    """Remove from the local whitelist."""
    logger = logging.getLogger("pyzor")
    lwhitelist_fp = config.get("client", "LocalWhitelist")
    lwhitelist = pyzor.config.load_local_whitelist(lwhitelist_fp)
    style = config.get("client", "Style")
    for digested in get_input_handler(style):
        if digested not in lwhitelist:
            logger.critical("Digest %s is not whitelisted.", digested)
            continue
        lwhitelist.remove(digested)
    with open(lwhitelist_fp, "w") as lwhitelist_f:
        lwhitelist_f.write("\n".join(lwhitelist))
    return True


def predigest(client, servers, config):
    """Output the normalised version of each message, which is used to
    create the digest.

    This method can be used to diagnose which parts of the message are
    used to determine uniqueness."""
    for unused in get_input_handler(
            "msg", digester=pyzor.digest.PrintingDataDigester):
        pass
    return True


def genkey(client, servers, config, hash_func=hashlib.sha1):
    """Generate a key to use to authenticate pyzor requests.  This method
    will prompt for a password (and confirmation).

    A random salt is generated (which makes it extremely difficult to
    reverse the generated key to get the original password) and combined
    with the entered password to provide a key.  This key (but not the salt)
    should be provided to the pyzord administrator, along with a username.
    """
    # pylint: disable-msg=W0613
    password = getpass.getpass(prompt="Enter passphrase: ")
    if getpass.getpass(prompt="Enter passphrase again: ") != password:
        log = logging.getLogger("pyzor")
        log.error("Passwords do not match.")
        return False
    # pylint: disable-msg=W0612
    salt = "".join([chr(random.randint(0, 255))
                    for unused in xrange(hash_func(b"").digest_size)])
    if sys.version_info >= (3, 0):
        salt = salt.encode("utf8")
    salt_digest = hash_func(salt)
    pass_digest = hash_func(salt_digest.digest())
    pass_digest.update(password.encode("utf8"))
    print("salt,key:")
    print("%s,%s" % (salt_digest.hexdigest(), pass_digest.hexdigest()))
    return True


DISPATCHES = {
    "ping": ping,
    "pong": pong,
    "info": info,
    "check": check,
    "report": report,
    "whitelist": whitelist,
    "digest": digest,
    "predigest": predigest,
    "genkey": genkey,
    "local_whitelist": local_whitelist,
    "local_unwhitelist": local_unwhitelist,
}


INPUT_HANDLERS = {
    "msg": _get_input_msg,
    "mbox": _get_input_mbox,
    "digests": _get_input_digests,
}

if __name__ == "__main__":
    main()