/usr/lib/python2.7/dist-packages/scrapy/spidermiddlewares/referer.py is in python-scrapy 1.0.3-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | """
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from scrapy.http import Request
from scrapy.exceptions import NotConfigured
class RefererMiddleware(object):
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('REFERER_ENABLED'):
raise NotConfigured
return cls()
def process_spider_output(self, response, result, spider):
def _set_referer(r):
if isinstance(r, Request):
r.headers.setdefault('Referer', response.url)
return r
return (_set_referer(r) for r in result or ())
|