/usr/share/pyshared/weboob/tools/misc.py is in python-weboob-core 0.g-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | # -*- coding: utf-8 -*-
# Copyright(C) 2010-2013 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from logging import warning
from time import time, sleep
from tempfile import gettempdir
import os
import sys
import traceback
import types
# keep compatibility
from .date import local2utc, utc2local
__all__ = ['get_backtrace', 'get_bytes_size', 'html2text', 'iter_fields',
'local2utc', 'to_unicode', 'utc2local', 'limit']
def get_backtrace(empty="Empty backtrace."):
"""
Try to get backtrace as string.
Returns "Error while trying to get backtrace" on failure.
"""
try:
info = sys.exc_info()
trace = traceback.format_exception(*info)
sys.exc_clear()
if trace[0] != "None\n":
return "".join(trace)
except:
# No i18n here (imagine if i18n function calls error...)
return "Error while trying to get backtrace"
return empty
def get_bytes_size(size, unit_name):
unit_data = {
'bytes': 1,
'KB': 1024,
'MB': 1024 * 1024,
'GB': 1024 * 1024 * 1024,
'TB': 1024 * 1024 * 1024 * 1024,
}
return float(size * unit_data.get(unit_name, 1))
try:
import html2text as h2t
h2t.UNICODE_SNOB = 1
h2t.SKIP_INTERNAL_LINKS = True
h2t.INLINE_LINKS = False
h2t.LINKS_EACH_PARAGRAPH = True
html2text = h2t.html2text
except ImportError:
warning('python-html2text is not present. HTML pages will not be converted into text.')
def html2text(html):
return html
def iter_fields(obj):
for attribute_name in dir(obj):
if attribute_name.startswith('_'):
continue
attribute = getattr(obj, attribute_name)
if not isinstance(attribute, types.MethodType):
yield attribute_name, attribute
def to_unicode(text):
r"""
>>> to_unicode('ascii')
u'ascii'
>>> to_unicode(u'utf\xe9'.encode('UTF-8'))
u'utf\xe9'
>>> to_unicode(u'unicode')
u'unicode'
"""
if isinstance(text, unicode):
return text
if not isinstance(text, str):
try:
text = str(text)
except UnicodeError:
return unicode(text)
try:
return unicode(text, 'utf-8')
except UnicodeError:
try:
return unicode(text, 'iso-8859-15')
except UnicodeError:
return unicode(text, 'windows-1252', 'replace')
def limit(iterator, lim):
count = 0
iterator = iter(iterator)
while count < lim:
yield iterator.next()
count += 1
raise StopIteration()
def ratelimit(group, delay):
"""
Simple rate limiting.
Waits if the last call of lastlimit with this group name was less than
delay seconds ago. The rate limiting is global, shared between any instance
of the application and any call to this function sharing the same group
name. The same group name should not be used with different delays.
This function is intended to be called just before the code that should be
rate-limited.
This function is not thread-safe. For reasonably non-critical rate
limiting (like accessing a website), it should be sufficient nevertheless.
@param group [string] rate limiting group name, alphanumeric
@param delay [int] delay in seconds between each call
"""
path = os.path.join(gettempdir(), 'weboob_ratelimit.%s' % group)
while True:
try:
offset = time() - os.stat(path).st_mtime
except OSError:
with open(path, 'w'):
pass
offset = 0
if delay < offset:
break
sleep(delay - offset)
os.utime(path, None)
|