/usr/share/pyshared/scrapyd/environ.py is in python-scrapy 0.14.4-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import os
from zope.interface import implements
from .interfaces import IEnvironment
class Environment(object):
implements(IEnvironment)
def __init__(self, config, initenv=os.environ):
self.dbs_dir = config.get('dbs_dir', 'dbs')
self.logs_dir = config.get('logs_dir', 'logs')
self.logs_to_keep = config.getint('logs_to_keep', 5)
if config.cp.has_section('settings'):
self.settings = dict(config.cp.items('settings'))
else:
self.settings = {}
self.initenv = initenv
def get_environment(self, message, slot):
project = message['_project']
env = self.initenv.copy()
env['SCRAPY_SLOT'] = str(slot)
env['SCRAPY_PROJECT'] = project
env['SCRAPY_SPIDER'] = message['_spider']
env['SCRAPY_JOB'] = message['_job']
if project in self.settings:
env['SCRAPY_SETTINGS_MODULE'] = self.settings[project]
env['SCRAPY_LOG_FILE'] = self._get_log_file(message)
return env
def _get_log_file(self, message):
logsdir = os.path.join(self.logs_dir, message['_project'], \
message['_spider'])
if not os.path.exists(logsdir):
os.makedirs(logsdir)
to_delete = sorted(os.listdir(logsdir), reverse=True)[:-self.logs_to_keep]
for x in to_delete:
os.remove(os.path.join(logsdir, x))
return os.path.join(logsdir, "%s.log" % message['_job'])
|