/usr/share/pyshared/scrapyd/poller.py is in python-scrapy 0.14.4-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | from zope.interface import implements
from twisted.internet.defer import DeferredQueue, inlineCallbacks, maybeDeferred, returnValue
from .utils import get_spider_queues
from .interfaces import IPoller
class QueuePoller(object):
implements(IPoller)
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
|