/usr/lib/python3/dist-packages/tables/attributeset.py is in python3-tables 3.4.2-4.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 | # -*- coding: utf-8 -*-
########################################################################
#
# License: BSD
# Created: May 26, 2003
# Author: Francesc Alted - faltet@pytables.com
#
# $Id$
#
########################################################################
"""Here is defined the AttributeSet class."""
from __future__ import absolute_import
import re
import sys
import warnings
import six.moves.cPickle
import numpy
from . import hdf5extension
from .utils import SizeType
from .registry import class_name_dict
from .exceptions import ClosedNodeError, PerformanceWarning
from .path import check_attribute_name
from .undoredo import attr_to_shadow
from .filters import Filters
from six.moves import map
import six
# System attributes
SYS_ATTRS = ["CLASS", "VERSION", "TITLE", "NROWS", "EXTDIM",
"ENCODING", "PYTABLES_FORMAT_VERSION",
"FLAVOR", "FILTERS", "AUTO_INDEX",
"DIRTY", "NODE_TYPE", "NODE_TYPE_VERSION",
"PSEUDOATOM"]
# Prefixes of other system attributes
SYS_ATTRS_PREFIXES = ["FIELD_"]
# RO_ATTRS will be disabled and let the user modify them if they
# want to. The user is still not allowed to remove or rename
# system attributes. Francesc Alted 2004-12-19
# Read-only attributes:
# RO_ATTRS = ["CLASS", "FLAVOR", "VERSION", "NROWS", "EXTDIM",
# "PYTABLES_FORMAT_VERSION", "FILTERS",
# "NODE_TYPE", "NODE_TYPE_VERSION"]
# RO_ATTRS = []
# The next attributes are not meant to be copied during a Node copy process
SYS_ATTRS_NOTTOBECOPIED = ["CLASS", "VERSION", "TITLE", "NROWS", "EXTDIM",
"PYTABLES_FORMAT_VERSION", "FILTERS", "ENCODING"]
# Attributes forced to be copied during node copies
FORCE_COPY_CLASS = ['CLASS', 'VERSION']
# Regular expression for column default values.
_field_fill_re = re.compile('^FIELD_[0-9]+_FILL$')
# Regular expression for fixing old pickled filters.
_old_filters_re = re.compile(br'\(([ic])tables\.Leaf\n')
# Fixed version of the previous string.
_new_filters_sub = br'(\1tables.filters\n'
def issysattrname(name):
"Check if a name is a system attribute or not"
if (name in SYS_ATTRS or
numpy.prod([name.startswith(prefix)
for prefix in SYS_ATTRS_PREFIXES])):
return True
else:
return False
@six.python_2_unicode_compatible
class AttributeSet(hdf5extension.AttributeSet, object):
"""Container for the HDF5 attributes of a Node.
This class provides methods to create new HDF5 node attributes,
and to get, rename or delete existing ones.
Like in Group instances (see :ref:`GroupClassDescr`), AttributeSet
instances make use of the *natural naming* convention, i.e. you can
access the attributes on disk as if they were normal Python
attributes of the AttributeSet instance.
This offers the user a very convenient way to access HDF5 node
attributes. However, for this reason and in order not to pollute the
object namespace, one can not assign *normal* attributes to
AttributeSet instances, and their members use names which start by
special prefixes as happens with Group objects.
.. rubric:: Notes on native and pickled attributes
The values of most basic types are saved as HDF5 native data in the
HDF5 file. This includes Python bool, int, float, complex and str
(but not long nor unicode) values, as well as their NumPy scalar
versions and homogeneous or *structured* NumPy arrays of them. When
read, these values are always loaded as NumPy scalar or array
objects, as needed.
For that reason, attributes in native HDF5 files will be always
mapped into NumPy objects. Specifically, a multidimensional
attribute will be mapped into a multidimensional ndarray and a
scalar will be mapped into a NumPy scalar object (for example, a
scalar H5T_NATIVE_LLONG will be read and returned as a numpy.int64
scalar).
However, other kinds of values are serialized using pickle, so you
only will be able to correctly retrieve them using a Python-aware
HDF5 library. Thus, if you want to save Python scalar values and
make sure you are able to read them with generic HDF5 tools, you
should make use of *scalar or homogeneous/structured array NumPy
objects* (for example, numpy.int64(1) or numpy.array([1, 2, 3],
dtype='int16')).
One more advice: because of the various potential difficulties in
restoring a Python object stored in an attribute, you may end up
getting a pickle string where a Python object is expected. If this
is the case, you may wish to run pickle.loads() on that string to
get an idea of where things went wrong, as shown in this example::
>>> import os, tempfile
>>> import tables
>>>
>>> class MyClass(object):
... foo = 'bar'
...
>>> myObject = MyClass() # save object of custom class in HDF5 attr
>>> h5fname = tempfile.mktemp(suffix='.h5')
>>> h5f = tables.open_file(h5fname, 'w')
>>> h5f.root._v_attrs.obj = myObject # store the object
>>> print(h5f.root._v_attrs.obj.foo) # retrieve it
bar
>>> h5f.close()
>>>
>>> del MyClass, myObject # delete class of object and reopen file
>>> h5f = tables.open_file(h5fname, 'r')
>>> print(repr(h5f.root._v_attrs.obj))
'ccopy_reg\\n_reconstructor...
>>> import pickle # let's unpickle that to see what went wrong
>>> pickle.loads(h5f.root._v_attrs.obj)
Traceback (most recent call last):
...
AttributeError: 'module' object has no attribute 'MyClass'
>>> # So the problem was not in the stored object,
... # but in the *environment* where it was restored.
... h5f.close()
>>> os.remove(h5fname)
.. rubric:: Notes on AttributeSet methods
Note that this class overrides the __getattr__(), __setattr__() and
__delattr__() special methods. This allows you to read, assign or
delete attributes on disk by just using the next constructs::
leaf.attrs.myattr = 'str attr' # set a string (native support)
leaf.attrs.myattr2 = 3 # set an integer (native support)
leaf.attrs.myattr3 = [3, (1, 2)] # a generic object (Pickled)
attrib = leaf.attrs.myattr # get the attribute ``myattr``
del leaf.attrs.myattr # delete the attribute ``myattr``
In addition, the dictionary-like __getitem__(), __setitem__() and
__delitem__() methods are available, so you may write things like
this::
for name in node._v_attrs._f_list():
print("name: %s, value: %s" % (name, node._v_attrs[name]))
Use whatever idiom you prefer to access the attributes.
If an attribute is set on a target node that already has a large
number of attributes, a PerformanceWarning will be issued.
.. rubric:: AttributeSet attributes
.. attribute:: _v_attrnames
A list with all attribute names.
.. attribute:: _v_attrnamessys
A list with system attribute names.
.. attribute:: _v_attrnamesuser
A list with user attribute names.
.. attribute:: _v_unimplemented
A list of attribute names with unimplemented native HDF5 types.
"""
def _g_getnode(self):
return self._v__nodefile._get_node(self._v__nodepath)
@property
def _v_node(self):
"""The :class:`Node` instance this attribute set is associated with."""
return self._g_getnode()
def __init__(self, node):
"""Create the basic structures to keep the attribute information.
Reads all the HDF5 attributes (if any) on disk for the node "node".
Parameters
----------
node
The parent node
"""
# Refuse to create an instance of an already closed node
if not node._v_isopen:
raise ClosedNodeError("the node for attribute set is closed")
dict_ = self.__dict__
self._g_new(node)
dict_["_v__nodefile"] = node._v_file
dict_["_v__nodepath"] = node._v_pathname
dict_["_v_attrnames"] = self._g_list_attr(node)
# The list of unimplemented attribute names
dict_["_v_unimplemented"] = []
# Get the file version format. This is an optimization
# in order to avoid accessing it too much.
try:
format_version = node._v_file.format_version
except AttributeError:
parsed_version = None
else:
if format_version == 'unknown':
parsed_version = None
else:
parsed_version = tuple(map(int, format_version.split('.')))
dict_["_v__format_version"] = parsed_version
# Split the attribute list in system and user lists
dict_["_v_attrnamessys"] = []
dict_["_v_attrnamesuser"] = []
for attr in self._v_attrnames:
# put the attributes on the local dictionary to allow
# tab-completion
self.__getattr__(attr)
if issysattrname(attr):
self._v_attrnamessys.append(attr)
else:
self._v_attrnamesuser.append(attr)
# Sort the attributes
self._v_attrnames.sort()
self._v_attrnamessys.sort()
self._v_attrnamesuser.sort()
def _g_update_node_location(self, node):
"""Updates the location information about the associated `node`."""
dict_ = self.__dict__
dict_['_v__nodefile'] = node._v_file
dict_['_v__nodepath'] = node._v_pathname
# hdf5extension operations:
self._g_new(node)
def _f_list(self, attrset='user'):
"""Get a list of attribute names.
The attrset string selects the attribute set to be used. A
'user' value returns only user attributes (this is the default).
A 'sys' value returns only system attributes. Finally, 'all'
returns both system and user attributes.
"""
if attrset == "user":
return self._v_attrnamesuser[:]
elif attrset == "sys":
return self._v_attrnamessys[:]
elif attrset == "all":
return self._v_attrnames[:]
def __getattr__(self, name):
"""Get the attribute named "name"."""
# If attribute does not exist, raise AttributeError
if not name in self._v_attrnames:
raise AttributeError("Attribute '%s' does not exist in node: "
"'%s'" % (name, self._v__nodepath))
# Read the attribute from disk. This is an optimization to read
# quickly system attributes that are _string_ values, but it
# takes care of other types as well as for example NROWS for
# Tables and EXTDIM for EArrays
format_version = self._v__format_version
value = self._g_getattr(self._v_node, name)
# Check whether the value is pickled
# Pickled values always seems to end with a "."
maybe_pickled = (
isinstance(value, numpy.generic) and # NumPy scalar?
value.dtype.type == numpy.bytes_ and # string type?
value.itemsize > 0 and value.endswith(b'.'))
if (maybe_pickled and value in [b"0", b"0."]):
# Workaround for a bug in many versions of Python (starting
# somewhere after Python 2.6.1). See ticket #253.
retval = value
elif (maybe_pickled and _field_fill_re.match(name)
and format_version == (1, 5)):
# This format was used during the first 1.2 releases, just
# for string defaults.
try:
retval = six.moves.cPickle.loads(value)
retval = numpy.array(retval)
except ImportError:
retval = None # signal error avoiding exception
elif maybe_pickled and name == 'FILTERS' and format_version < (2, 0):
# This is a big hack, but we don't have other way to recognize
# pickled filters of PyTables 1.x files.
value = _old_filters_re.sub(_new_filters_sub, value, 1)
retval = six.moves.cPickle.loads(value) # pass unpickling errors through
elif maybe_pickled:
try:
retval = six.moves.cPickle.loads(value)
# except cPickle.UnpicklingError:
# It seems that pickle may raise other errors than UnpicklingError
# Perhaps it would be better just an "except:" clause?
# except (cPickle.UnpicklingError, ImportError):
# Definitely (see SF bug #1254636)
except UnicodeDecodeError:
# Object maybe pickled on python 2 and unpickled on python 3.
# encoding='bytes' was added in python 3.4 to resolve this.
# However 'bytes' mangles class attributes as they are
# unplicked as bytestrings. Hence try 'latin1' first.
# Ref: http://bugs.python.org/issue6784
try:
retval = six.moves.cPickle.loads(value, encoding='latin1')
except TypeError:
try:
retval = six.moves.cPickle.loads(value, encoding='bytes')
except:
retval = value
except:
retval = value
except:
# catch other unpickling errors:
# ivb (2005-09-07): It is too hard to tell
# whether the unpickling failed
# because of the string not being a pickle one at all,
# because of a malformed pickle string,
# or because of some other problem in object reconstruction,
# thus making inconvenient even the issuing of a warning here.
# The documentation contains a note on this issue,
# explaining how the user can tell where the problem was.
retval = value
# Additional check for allowing a workaround for #307
if isinstance(retval, six.text_type) and retval == u'':
retval = numpy.array(retval)[()]
elif name == 'FILTERS' and format_version >= (2, 0):
retval = Filters._unpack(value)
elif name == 'TITLE' and not isinstance(value, str):
if sys.version_info[0] < 3:
# unicode is OK for TITLE
retval = value
else:
retval = value.decode('utf-8')
elif (issysattrname(name) and isinstance(value, (bytes, six.text_type)) and
not isinstance(value, str) and not _field_fill_re.match(name)):
# system attributes should always be str
if sys.version_info[0] < 3:
retval = value.encode()
else:
# python 3, bytes and not "FIELD_[0-9]+_FILL"
retval = value.decode('utf-8')
else:
retval = value
# Put this value in local directory
self.__dict__[name] = retval
return retval
def _g__setattr(self, name, value):
"""Set a PyTables attribute.
Sets a (maybe new) PyTables attribute with the specified `name`
and `value`. If the attribute already exists, it is simply
replaced.
It does not log the change.
"""
# Save this attribute to disk
# (overwriting an existing one if needed)
stvalue = value
if issysattrname(name):
if name in ["EXTDIM", "AUTO_INDEX", "DIRTY", "NODE_TYPE_VERSION"]:
stvalue = numpy.array(value, dtype=numpy.int32)
value = stvalue[()]
elif name == "NROWS":
stvalue = numpy.array(value, dtype=SizeType)
value = stvalue[()]
elif name == "FILTERS" and self._v__format_version >= (2, 0):
stvalue = value._pack()
# value will remain as a Filters instance here
# Convert value from a Python scalar into a NumPy scalar
# (only in case it has not been converted yet)
# Fixes ticket #59
if (stvalue is value and
type(value) in (bool, bytes, int, float, complex, six.text_type,
numpy.unicode_)):
# Additional check for allowing a workaround for #307
if isinstance(value, six.text_type) and len(value) == 0:
stvalue = numpy.array(u'')
else:
stvalue = numpy.array(value)
value = stvalue[()]
self._g_setattr(self._v_node, name, stvalue)
# New attribute or value. Introduce it into the local
# directory
self.__dict__[name] = value
# Finally, add this attribute to the list if not present
attrnames = self._v_attrnames
if not name in attrnames:
attrnames.append(name)
attrnames.sort()
if issysattrname(name):
attrnamessys = self._v_attrnamessys
attrnamessys.append(name)
attrnamessys.sort()
else:
attrnamesuser = self._v_attrnamesuser
attrnamesuser.append(name)
attrnamesuser.sort()
def __setattr__(self, name, value):
"""Set a PyTables attribute.
Sets a (maybe new) PyTables attribute with the specified `name`
and `value`. If the attribute already exists, it is simply
replaced.
A ``ValueError`` is raised when the name starts with a reserved
prefix or contains a ``/``. A `NaturalNameWarning` is issued if
the name is not a valid Python identifier. A
`PerformanceWarning` is issued when the recommended maximum
number of attributes in a node is going to be exceeded.
"""
nodefile = self._v__nodefile
attrnames = self._v_attrnames
# Check for name validity
check_attribute_name(name)
nodefile._check_writable()
# Check if there are too many attributes.
max_node_attrs = nodefile.params['MAX_NODE_ATTRS']
if len(attrnames) >= max_node_attrs:
warnings.warn("""\
node ``%s`` is exceeding the recommended maximum number of attributes (%d);\
be ready to see PyTables asking for *lots* of memory and possibly slow I/O"""
% (self._v__nodepath, max_node_attrs),
PerformanceWarning)
undo_enabled = nodefile.is_undo_enabled()
# Log old attribute removal (if any).
if undo_enabled and (name in attrnames):
self._g_del_and_log(name)
# Set the attribute.
self._g__setattr(name, value)
# Log new attribute addition.
if undo_enabled:
self._g_log_add(name)
def _g_log_add(self, name):
self._v__nodefile._log('ADDATTR', self._v__nodepath, name)
def _g_del_and_log(self, name):
nodefile = self._v__nodefile
node_pathname = self._v__nodepath
# Log *before* moving to use the right shadow name.
nodefile._log('DELATTR', node_pathname, name)
attr_to_shadow(nodefile, node_pathname, name)
def _g__delattr(self, name):
"""Delete a PyTables attribute.
Deletes the specified existing PyTables attribute.
It does not log the change.
"""
# Delete the attribute from disk
self._g_remove(self._v_node, name)
# Delete the attribute from local lists
self._v_attrnames.remove(name)
if name in self._v_attrnamessys:
self._v_attrnamessys.remove(name)
else:
self._v_attrnamesuser.remove(name)
# Delete the attribute from the local directory
# closes (#1049285)
del self.__dict__[name]
def __delattr__(self, name):
"""Delete a PyTables attribute.
Deletes the specified existing PyTables attribute from the
attribute set. If a nonexistent or system attribute is
specified, an ``AttributeError`` is raised.
"""
nodefile = self._v__nodefile
# Check if attribute exists
if name not in self._v_attrnames:
raise AttributeError(
"Attribute ('%s') does not exist in node '%s'"
% (name, self._v__nodepath))
nodefile._check_writable()
# Remove the PyTables attribute or move it to shadow.
if nodefile.is_undo_enabled():
self._g_del_and_log(name)
else:
self._g__delattr(name)
def __getitem__(self, name):
"""The dictionary like interface for __getattr__()."""
try:
return self.__getattr__(name)
except AttributeError:
# Capture the AttributeError an re-raise a KeyError one
raise KeyError(
"Attribute ('%s') does not exist in node '%s'"
% (name, self._v__nodepath))
def __setitem__(self, name, value):
"""The dictionary like interface for __setattr__()."""
self.__setattr__(name, value)
def __delitem__(self, name):
"""The dictionary like interface for __delattr__()."""
try:
self.__delattr__(name)
except AttributeError:
# Capture the AttributeError an re-raise a KeyError one
raise KeyError(
"Attribute ('%s') does not exist in node '%s'"
% (name, self._v__nodepath))
def __contains__(self, name):
"""Is there an attribute with that name?
A true value is returned if the attribute set has an attribute
with the given name, false otherwise.
"""
return name in self._v_attrnames
def _f_rename(self, oldattrname, newattrname):
"""Rename an attribute from oldattrname to newattrname."""
if oldattrname == newattrname:
# Do nothing
return
# First, fetch the value of the oldattrname
attrvalue = getattr(self, oldattrname)
# Now, create the new attribute
setattr(self, newattrname, attrvalue)
# Finally, remove the old attribute
delattr(self, oldattrname)
def _g_copy(self, newset, set_attr=None, copyclass=False):
"""Copy set attributes.
Copies all user and allowed system PyTables attributes to the
given attribute set, replacing the existing ones.
You can specify a *bound* method of the destination set that
will be used to set its attributes. Else, its `_g__setattr`
method will be used.
Changes are logged depending on the chosen setting method. The
default setting method does not log anything.
.. versionchanged:: 3.0
The *newSet* parameter has been renamed into *newset*.
.. versionchanged:: 3.0
The *copyClass* parameter has been renamed into *copyclass*.
"""
copysysattrs = newset._v__nodefile.params['PYTABLES_SYS_ATTRS']
if set_attr is None:
set_attr = newset._g__setattr
for attrname in self._v_attrnamesuser:
# Do not copy the unimplemented attributes.
if attrname not in self._v_unimplemented:
set_attr(attrname, getattr(self, attrname))
# Copy the system attributes that we are allowed to.
if copysysattrs:
for attrname in self._v_attrnamessys:
if ((attrname not in SYS_ATTRS_NOTTOBECOPIED) and
# Do not copy the FIELD_ attributes in tables as this can
# be really *slow* (don't know exactly the reason).
# See #304.
not attrname.startswith("FIELD_")):
set_attr(attrname, getattr(self, attrname))
# Copy CLASS and VERSION attributes if requested
if copyclass:
for attrname in FORCE_COPY_CLASS:
if attrname in self._v_attrnamessys:
set_attr(attrname, getattr(self, attrname))
def _f_copy(self, where):
"""Copy attributes to the where node.
Copies all user and certain system attributes to the given where
node (a Node instance - see :ref:`NodeClassDescr`), replacing
the existing ones.
"""
# AttributeSet must be defined in order to define a Node.
# However, we need to know Node here.
# Using class_name_dict avoids a circular import.
if not isinstance(where, class_name_dict['Node']):
raise TypeError("destination object is not a node: %r" % (where,))
self._g_copy(where._v_attrs, where._v_attrs.__setattr__)
def _g_close(self):
# Nothing will be done here, as the existing instance is completely
# operative now.
pass
def __str__(self):
"""The string representation for this object."""
# The pathname
pathname = self._v__nodepath
# Get this class name
classname = self.__class__.__name__
# The attribute names
attrnumber = len([n for n in self._v_attrnames])
return "%s._v_attrs (%s), %s attributes" % \
(pathname, classname, attrnumber)
def __repr__(self):
"""A detailed string representation for this object."""
# print additional info only if there are attributes to show
attrnames = [n for n in self._v_attrnames]
if len(attrnames):
rep = ['%s := %r' % (attr, getattr(self, attr))
for attr in attrnames]
attrlist = '[%s]' % (',\n '.join(rep))
return "%s:\n %s" % (str(self), attrlist)
else:
return str(self)
class NotLoggedAttributeSet(AttributeSet):
def _g_log_add(self, name):
pass
def _g_del_and_log(self, name):
self._g__delattr(name)
## Local Variables:
## mode: python
## py-indent-offset: 4
## tab-width: 4
## fill-column: 72
## End:
|