/usr/lib/python3.7/test/test_unicodedata.py is in libpython3.7-testsuite 3.7.0~b3-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 | """ Test script for the unicodedata module.
Written by Marc-Andre Lemburg (mal@lemburg.com).
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
"""
import sys
import unittest
import hashlib
from test.support import script_helper
encoding = 'utf-8'
errors = 'surrogatepass'
### Run tests
class UnicodeMethodsTest(unittest.TestCase):
# update this, if the database changes
expectedchecksum = '727091e0fd5807eb41c72912ae95cdd74c795e27'
def test_method_checksum(self):
h = hashlib.sha1()
for i in range(0x10000):
char = chr(i)
data = [
# Predicates (single char)
"01"[char.isalnum()],
"01"[char.isalpha()],
"01"[char.isdecimal()],
"01"[char.isdigit()],
"01"[char.islower()],
"01"[char.isnumeric()],
"01"[char.isspace()],
"01"[char.istitle()],
"01"[char.isupper()],
# Predicates (multiple chars)
"01"[(char + 'abc').isalnum()],
"01"[(char + 'abc').isalpha()],
"01"[(char + '123').isdecimal()],
"01"[(char + '123').isdigit()],
"01"[(char + 'abc').islower()],
"01"[(char + '123').isnumeric()],
"01"[(char + ' \t').isspace()],
"01"[(char + 'abc').istitle()],
"01"[(char + 'ABC').isupper()],
# Mappings (single char)
char.lower(),
char.upper(),
char.title(),
# Mappings (multiple chars)
(char + 'abc').lower(),
(char + 'ABC').upper(),
(char + 'abc').title(),
(char + 'ABC').title(),
]
h.update(''.join(data).encode(encoding, errors))
result = h.hexdigest()
self.assertEqual(result, self.expectedchecksum)
class UnicodeDatabaseTest(unittest.TestCase):
def setUp(self):
# In case unicodedata is not available, this will raise an ImportError,
# but the other test cases will still be run
import unicodedata
self.db = unicodedata
def tearDown(self):
del self.db
class UnicodeFunctionsTest(UnicodeDatabaseTest):
# Update this if the database changes. Make sure to do a full rebuild
# (e.g. 'make distclean && make') to get the correct checksum.
expectedchecksum = 'db6f92bb5010f8e85000634b08e77233355ab37a'
def test_function_checksum(self):
data = []
h = hashlib.sha1()
for i in range(0x10000):
char = chr(i)
data = [
# Properties
format(self.db.digit(char, -1), '.12g'),
format(self.db.numeric(char, -1), '.12g'),
format(self.db.decimal(char, -1), '.12g'),
self.db.category(char),
self.db.bidirectional(char),
self.db.decomposition(char),
str(self.db.mirrored(char)),
str(self.db.combining(char)),
]
h.update(''.join(data).encode("ascii"))
result = h.hexdigest()
self.assertEqual(result, self.expectedchecksum)
def test_digit(self):
self.assertEqual(self.db.digit('A', None), None)
self.assertEqual(self.db.digit('9'), 9)
self.assertEqual(self.db.digit('\u215b', None), None)
self.assertEqual(self.db.digit('\u2468'), 9)
self.assertEqual(self.db.digit('\U00020000', None), None)
self.assertEqual(self.db.digit('\U0001D7FD'), 7)
self.assertRaises(TypeError, self.db.digit)
self.assertRaises(TypeError, self.db.digit, 'xx')
self.assertRaises(ValueError, self.db.digit, 'x')
def test_numeric(self):
self.assertEqual(self.db.numeric('A',None), None)
self.assertEqual(self.db.numeric('9'), 9)
self.assertEqual(self.db.numeric('\u215b'), 0.125)
self.assertEqual(self.db.numeric('\u2468'), 9.0)
self.assertEqual(self.db.numeric('\ua627'), 7.0)
self.assertEqual(self.db.numeric('\U00020000', None), None)
self.assertEqual(self.db.numeric('\U0001012A'), 9000)
self.assertRaises(TypeError, self.db.numeric)
self.assertRaises(TypeError, self.db.numeric, 'xx')
self.assertRaises(ValueError, self.db.numeric, 'x')
def test_decimal(self):
self.assertEqual(self.db.decimal('A',None), None)
self.assertEqual(self.db.decimal('9'), 9)
self.assertEqual(self.db.decimal('\u215b', None), None)
self.assertEqual(self.db.decimal('\u2468', None), None)
self.assertEqual(self.db.decimal('\U00020000', None), None)
self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
self.assertRaises(TypeError, self.db.decimal)
self.assertRaises(TypeError, self.db.decimal, 'xx')
self.assertRaises(ValueError, self.db.decimal, 'x')
def test_category(self):
self.assertEqual(self.db.category('\uFFFE'), 'Cn')
self.assertEqual(self.db.category('a'), 'Ll')
self.assertEqual(self.db.category('A'), 'Lu')
self.assertEqual(self.db.category('\U00020000'), 'Lo')
self.assertEqual(self.db.category('\U0001012A'), 'No')
self.assertRaises(TypeError, self.db.category)
self.assertRaises(TypeError, self.db.category, 'xx')
def test_bidirectional(self):
self.assertEqual(self.db.bidirectional('\uFFFE'), '')
self.assertEqual(self.db.bidirectional(' '), 'WS')
self.assertEqual(self.db.bidirectional('A'), 'L')
self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
self.assertRaises(TypeError, self.db.bidirectional)
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
def test_decomposition(self):
self.assertEqual(self.db.decomposition('\uFFFE'),'')
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
self.assertRaises(TypeError, self.db.decomposition)
self.assertRaises(TypeError, self.db.decomposition, 'xx')
def test_mirrored(self):
self.assertEqual(self.db.mirrored('\uFFFE'), 0)
self.assertEqual(self.db.mirrored('a'), 0)
self.assertEqual(self.db.mirrored('\u2201'), 1)
self.assertEqual(self.db.mirrored('\U00020000'), 0)
self.assertRaises(TypeError, self.db.mirrored)
self.assertRaises(TypeError, self.db.mirrored, 'xx')
def test_combining(self):
self.assertEqual(self.db.combining('\uFFFE'), 0)
self.assertEqual(self.db.combining('a'), 0)
self.assertEqual(self.db.combining('\u20e1'), 230)
self.assertEqual(self.db.combining('\U00020000'), 0)
self.assertRaises(TypeError, self.db.combining)
self.assertRaises(TypeError, self.db.combining, 'xx')
def test_normalize(self):
self.assertRaises(TypeError, self.db.normalize)
self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
self.assertEqual(self.db.normalize('NFKC', ''), '')
# The rest can be found in test_normalization.py
# which requires an external file.
def test_pr29(self):
# http://www.unicode.org/review/pr-29.html
# See issues #1054943 and #10254.
composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
'Li\u030dt-s\u1e73\u0301',
'\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
+ '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
'\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
+ '\u0938\u094d\u0924\u093e\u0928')
for text in composed:
self.assertEqual(self.db.normalize('NFC', text), text)
def test_issue10254(self):
# Crash reported in #10254
a = 'C\u0338' * 20 + 'C\u0327'
b = 'C\u0338' * 20 + '\xC7'
self.assertEqual(self.db.normalize('NFC', a), b)
def test_east_asian_width(self):
eaw = self.db.east_asian_width
self.assertRaises(TypeError, eaw, b'a')
self.assertRaises(TypeError, eaw, bytearray())
self.assertRaises(TypeError, eaw, '')
self.assertRaises(TypeError, eaw, 'ra')
self.assertEqual(eaw('\x1e'), 'N')
self.assertEqual(eaw('\x20'), 'Na')
self.assertEqual(eaw('\uC894'), 'W')
self.assertEqual(eaw('\uFF66'), 'H')
self.assertEqual(eaw('\uFF1F'), 'F')
self.assertEqual(eaw('\u2010'), 'A')
self.assertEqual(eaw('\U00020000'), 'W')
def test_east_asian_width_9_0_changes(self):
self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
class UnicodeMiscTest(UnicodeDatabaseTest):
def test_failed_import_during_compiling(self):
# Issue 4367
# Decoding \N escapes requires the unicodedata module. If it can't be
# imported, we shouldn't segfault.
# This program should raise a SyntaxError in the eval.
code = "import sys;" \
"sys.modules['unicodedata'] = None;" \
"""eval("'\\\\N{SOFT HYPHEN}'")"""
# We use a separate process because the unicodedata module may already
# have been loaded in this process.
result = script_helper.assert_python_failure("-c", code)
error = "SyntaxError: (unicode error) \\N escapes not supported " \
"(can't load unicodedata module)"
self.assertIn(error, result.err.decode("ascii"))
def test_decimal_numeric_consistent(self):
# Test that decimal and numeric are consistent,
# i.e. if a character has a decimal value,
# its numeric value should be the same.
count = 0
for i in range(0x10000):
c = chr(i)
dec = self.db.decimal(c, -1)
if dec != -1:
self.assertEqual(dec, self.db.numeric(c))
count += 1
self.assertTrue(count >= 10) # should have tested at least the ASCII digits
def test_digit_numeric_consistent(self):
# Test that digit and numeric are consistent,
# i.e. if a character has a digit value,
# its numeric value should be the same.
count = 0
for i in range(0x10000):
c = chr(i)
dec = self.db.digit(c, -1)
if dec != -1:
self.assertEqual(dec, self.db.numeric(c))
count += 1
self.assertTrue(count >= 10) # should have tested at least the ASCII digits
def test_bug_1704793(self):
self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
def test_ucd_510(self):
import unicodedata
# In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
self.assertTrue(unicodedata.mirrored("\u0f3a"))
self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
# Also, we now have two ways of representing
# the upper-case mapping: as delta, or as absolute value
self.assertTrue("a".upper()=='A')
self.assertTrue("\u1d79".upper()=='\ua77d')
self.assertTrue(".".upper()=='.')
def test_bug_5828(self):
self.assertEqual("\u1d79".lower(), "\u1d79")
# Only U+0000 should have U+0000 as its upper/lower/titlecase variant
self.assertEqual(
[
c for c in range(sys.maxunicode+1)
if "\x00" in chr(c).lower()+chr(c).upper()+chr(c).title()
],
[0]
)
def test_bug_4971(self):
# LETTER DZ WITH CARON: DZ, Dz, dz
self.assertEqual("\u01c4".title(), "\u01c5")
self.assertEqual("\u01c5".title(), "\u01c5")
self.assertEqual("\u01c6".title(), "\u01c5")
def test_linebreak_7643(self):
for i in range(0x10000):
lines = (chr(i) + 'A').splitlines()
if i in (0x0a, 0x0b, 0x0c, 0x0d, 0x85,
0x1c, 0x1d, 0x1e, 0x2028, 0x2029):
self.assertEqual(len(lines), 2,
r"\u%.4x should be a linebreak" % i)
else:
self.assertEqual(len(lines), 1,
r"\u%.4x should not be a linebreak" % i)
if __name__ == "__main__":
unittest.main()
|