/usr/lib/python2.7/dist-packages/requestbuilder/service.py is in python-requestbuilder 0.1.0~beta2-1build1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 | # Copyright (c) 2012-2013, Eucalyptus Systems, Inc.
#
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
from __future__ import absolute_import
import copy
import datetime
import functools
import logging
import os.path
import random
from requestbuilder import SERVICE
from requestbuilder.exceptions import (ClientError, ServerError,
ServiceInitError)
from requestbuilder.util import (add_default_routes, aggregate_subclass_fields,
set_userregion)
import requests.exceptions
import time
import urlparse
import weakref
class BaseService(object):
NAME = None
DESCRIPTION = ''
API_VERSION = ''
MAX_RETRIES = 2
TIMEOUT = 30 # socket timeout in seconds
AUTH_CLASS = None
REGION_ENVVAR = None
URL_ENVVAR = None
ARGS = []
DEFAULT_ROUTES = (SERVICE,)
def __init__(self, config, auth=None, loglevel=None, max_retries=None,
timeout=None, **kwargs):
self.args = kwargs
self.config = config
self.endpoint = None
self.log = logging.getLogger(self.__class__.__name__)
if loglevel is not None:
self.log.level = loglevel
self.max_retries = max_retries
self.session_args = {}
self.timeout = timeout
self._session = None
if auth is not None:
self.auth = auth
self.auth.service = weakref.proxy(self)
elif self.AUTH_CLASS is not None:
self.auth = self.AUTH_CLASS(self.config, loglevel=self.log.level)
self.auth.service = weakref.proxy(self)
else:
self.auth = None
@property
def region_name(self):
# FIXME: this makes it impossible for services in different regions
# to share configuration.
return self.config.get_region()
def collect_arg_objs(self):
service_args = aggregate_subclass_fields(self.__class__, 'ARGS')
add_default_routes(service_args, self.DEFAULT_ROUTES)
if self.auth is not None:
auth_args = self.auth.collect_arg_objs()
else:
auth_args = []
return service_args + auth_args
def preprocess_arg_objs(self, arg_objs):
if self.auth is not None:
self.auth.preprocess_arg_objs(arg_objs)
def configure(self):
# self.args gets highest precedence for self.endpoint and user/region
self.process_url(self.args.get('url'))
set_userregion(self.config, self.args.get('userregion'))
# Environment comes next
set_userregion(self.config, os.getenv(self.REGION_ENVVAR))
self.process_url(os.getenv(self.URL_ENVVAR))
# Finally, try the config file
if self.NAME is not None:
self.process_url(self.config.get_region_option(self.NAME + '-url'))
# Configure timeout and retry handlers
if self.max_retries is None:
config_max_retries = self.config.get_global_option('max-retries')
if config_max_retries is not None:
self.max_retries = int(config_max_retries)
else:
self.max_retries = self.MAX_RETRIES
if self.timeout is None:
config_timeout = self.config.get_global_option('timeout')
if config_timeout is not None:
self.timeout = float(config_timeout)
else:
self.timeout = self.TIMEOUT
# SSL cert verification is opt-in
self.session_args['verify'] = self.config.get_region_option_bool(
'verify-ssl', default=False)
# Ensure everything is okay and finish up
self.validate_config()
if self.auth is not None:
self.auth.configure()
@property
def session(self):
if self._session is None:
if requests.__version__ >= '1.0':
self._session = requests.session()
for key, val in self.session_args.iteritems():
setattr(self._session, key, val)
else:
self._session = requests.session(**self.session_args)
return self._session
def validate_config(self):
if self.endpoint is None:
if self.NAME is not None:
url_opt = '{0}-url'.format(self.NAME)
available_regions = []
for rname, rconfig in self.config.regions.iteritems():
if url_opt in rconfig and '*' not in rname:
available_regions.append(rname)
if len(available_regions) > 0:
msg = ('No {0} endpoint to connect to was given. '
'Configured regions with {0} endpoints are: '
'{1}').format(self.NAME,
', '.join(sorted(available_regions)))
else:
msg = ('No {0} endpoint to connect to was given. {0} '
'endpoints may be specified in a config file with '
'"{1}".').format(self.NAME, url_opt)
else:
msg = 'No endpoint to connect to was given'
raise ServiceInitError(msg)
def process_url(self, url):
if url:
if '::' in url:
userregion, endpoint = url.split('::', 1)
else:
endpoint = url
userregion = None
if self.endpoint is None:
self.endpoint = endpoint
set_userregion(self.config, userregion)
def send_request(self, method='GET', path=None, params=None, headers=None,
data=None):
## TODO: test url-encoding
if path:
# We can't simply use urljoin because a path might start with '/'
# like it could for S3 keys that start with that character.
if self.endpoint.endswith('/'):
url = self.endpoint + path
else:
url = self.endpoint + '/' + path
else:
url = self.endpoint
headers = dict(headers)
if 'host' not in map(str.lower, headers.iterkeys()):
headers['Host'] = urlparse.urlparse(self.endpoint).netloc
try:
max_tries = self.max_retries + 1
assert max_tries >= 1
redirects_left = 5
if isinstance(data, file) and hasattr(data, 'seek'):
# If we're redirected we need to be able to reset
data_file_offset = data.tell()
else:
data_file_offset = None
while True:
for attempt_no, delay in enumerate(
_generate_delays(max_tries), 1):
# Use exponential backoff if this is a retry
if delay > 0:
self.log.debug('will retry after %.3f seconds', delay)
time.sleep(delay)
self.log.info('sending request (attempt %i of %i)',
attempt_no, max_tries)
response = self.__log_and_send_request(method, url, params,
data, headers)
if response.status_code not in (500, 503):
break
# If it *was* in that list, retry
if (response.status_code in (301, 302, 307, 308) and
redirects_left > 0 and 'Location' in response.headers):
# Standard redirect -- we need to handle this ourselves
# because we have to re-sign requests when their URLs
# change.
redirects_left -= 1
parsed_rdr = urlparse.urlparse(response.headers['Location'])
parsed_url = urlparse.urlparse(url)
new_url_bits = []
for rdr_bit, url_bit in zip(parsed_rdr, parsed_url):
new_url_bits.append(rdr_bit or url_bit)
if 'Host' in headers:
headers['Host'] = new_url_bits[1] # netloc
url = urlparse.urlunparse(new_url_bits)
self.log.debug('redirecting to %s (%i redirect(s) '
'remaining)', url, redirects_left)
if data_file_offset is not None:
self.log.debug('re-seeking body to beginning of file')
# pylint: disable=E1101
data.seek(data_file_offset)
# pylint: enable=E1101
continue
elif response.status_code >= 300:
# We include 30x because we've handled the standard method
# of redirecting, but the server might still be trying to
# redirect another way for some reason.
self.handle_http_error(response)
return response
except requests.exceptions.ConnectionError as exc:
if len(exc.args) > 0 and hasattr(exc.args[0], 'reason'):
raise ClientError(exc.args[0].reason)
else:
raise ClientError('connection error')
except requests.exceptions.HTTPError as exc:
return self.handle_http_error(response)
except requests.exceptions.RequestException as exc:
raise ClientError(exc)
def handle_http_error(self, response):
raise ServerError(response)
def __log_and_send_request(self, method, url, params, data, headers):
# Requests 1 gives auth handlers PreparedRequests instead of the
# original Requests like version 0 does. Since most of our auth
# handlers inspect and/or modify things that aren't headers, we
# manually apply auth to it in this method to make things less painful.
#
# The pre_send hook only works on requests 0. We replicate that for
# requests 1 just below.
hooks = {'pre_send': functools.partial(_log_request_data, self.log),
'response': functools.partial(_log_response_data, self.log)}
if requests.__version__ >= '1.0':
request = requests.Request(method=method, url=url,
params=params, data=data,
headers=headers)
if self.auth is not None:
self.auth(request)
# A prepared request gives us extra info we want to log
p_request = request.prepare()
p_request.hooks = {'response': hooks['response']}
self.log.debug('request method: %s', request.method)
self.log.debug('request url: %s', p_request.url)
if isinstance(p_request.headers, dict):
for key, val in sorted(p_request.headers.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
self.log.debug('request header: %s: %s', key, val)
if isinstance(request.params, dict):
for key, val in sorted(request.params.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
self.log.debug('request param: %s: %s', key, val)
if isinstance(request.data, dict):
for key, val in sorted(request.data.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
self.log.debug('request data: %s: %s', key, val)
p_request.start_time = datetime.datetime.now()
return self.session.send(p_request, stream=True,
timeout=self.timeout)
else:
request = requests.Request(method=method, url=url, params=params,
data=data, headers=headers,
timeout=self.timeout)
if self.auth is not None:
self.auth(request)
request.session = self.session
# A hook lets us log all the info that requests adds right
# before sending
request.hooks = hooks
request.start_time = datetime.datetime.now()
request.send()
return request.response
# Note that the hook this is meant to run as was removed from requests 1.
def _log_request_data(logger, request, **kwargs):
logger.debug('request method: %s', request.method)
logger.debug('request url: %s', request.full_url)
if isinstance(request.headers, dict):
for key, val in sorted(request.headers.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
logger.debug('request header: %s: %s', key, val)
if isinstance(request.params, dict):
for key, val in sorted(request.params.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
logger.debug('request param: %s: %s', key, val)
if isinstance(request.data, dict):
for key, val in sorted(request.data.iteritems()):
if key.lower().endswith('password'):
val = '<redacted>'
logger.debug('request data: %s: %s', key, val)
def _log_response_data(logger, response, **kwargs):
duration = datetime.datetime.now() - response.request.start_time
logger.debug('response time: %i.%03i seconds', duration.seconds,
duration.microseconds // 1000)
if response.status_code >= 400:
logger.error('response status: %i', response.status_code)
else:
logger.info('response status: %i', response.status_code)
if isinstance(response.headers, dict):
for key, val in sorted(response.headers.items()):
logger.debug('response header: %s: %s', key, val)
def _generate_delays(max_tries):
if max_tries >= 1:
yield 0
for retry_no in range(1, max_tries):
next_delay = (random.random() + 1) * 2 ** (retry_no - 1)
yield min((next_delay, 15))
|