/usr/lib/python3/dist-packages/pytools/diskdict.py is in python3-pytools 2014.3-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | # see end of file for sqlite import
from pytools import memoize
@memoize
def get_disk_dict(name, version, **kwargs):
import sys
import os
from os.path import join
from tempfile import gettempdir
import getpass
cache_dir = join(gettempdir(),
"%s-v%s-uid%s-py%s" % (
name, version,
getpass.getuser(), ".".join(str(i) for i in sys.version_info)))
# {{{ ensure cache directory exists
try:
os.mkdir(cache_dir)
except OSError as e:
from errno import EEXIST
if e.errno != EEXIST:
raise
# }}}
return DiskDict(join(cache_dir, "database.sqlite"), **kwargs)
class DiskDict(object):
"""Provides a disk-backed dictionary. Unlike :mod:`shelve`, this class allows
arbitrary values for keys, at a slight performance penalty.
Note that this is a dangerous game: The :func:`hash` of many objects
changes between runs. In particular, ``hash(None)`` changes between runs.
:class:`str`, :class:`unicode`, :class:`int`, :class:`tuple` and
:class:`long` seem to be constant for a given Python executable, but they
may change for a new version.
So don't use this class for data that you absolutely *have* to be able
to retrieve. It's fine for caches and the like, though.
"""
def __init__(self, dbfilename, version_base=(), dep_modules=[],
commit_interval=1):
self.db_conn = sqlite.connect(dbfilename, timeout=30)
try:
self.db_conn.execute("select * from data;")
except sqlite.OperationalError:
self.db_conn.execute("""
create table data (
id integer primary key autoincrement,
key_hash integer,
key_pickle blob,
version_hash integer,
version_pickle blob,
when_inserted timestamp default current_timestamp,
result_pickle blob)""")
def mtime(file):
if not isinstance(file, str):
# assume file names a module
file = file.__file__
import os
return os.stat(file).st_mtime
from pickle import dumps
self.version = (version_base,) + tuple(
mtime(dm) for dm in dep_modules)
self.version_pickle = dumps(self.version)
self.version_hash = hash(self.version)
self.cache = {}
self.commit_interval = commit_interval
self.commit_countdown = self.commit_interval
def __contains__(self, key):
if key in self.cache:
return True
else:
from pickle import loads
for key_pickle, version_pickle, result_pickle in self.db_conn.execute(
"select key_pickle, version_pickle, result_pickle from data"
" where key_hash = ? and version_hash = ?",
(hash(key), self.version_hash)):
if loads(str(key_pickle)) == key \
and loads(str(version_pickle)) == self.version:
result = loads(str(result_pickle))
self.cache[key] = result
return True
return False
def __getitem__(self, key):
try:
return self.cache[key]
except KeyError:
from pickle import loads
for key_pickle, version_pickle, result_pickle in self.db_conn.execute(
"select key_pickle, version_pickle, result_pickle from data"
" where key_hash = ? and version_hash = ?",
(hash(key), self.version_hash)):
if loads(str(key_pickle)) == key \
and loads(str(version_pickle)) == self.version:
result = loads(str(result_pickle))
self.cache[key] = result
return result
raise KeyError(key)
def __delitem__(self, key):
if key in self.cache:
del self.cache[key]
from pickle import loads
for item_id, key_pickle, version_pickle in self.db_conn.execute(
"select id, key_pickle, version_pickle from data"
" where key_hash = ? and version_hash = ?",
(hash(key), self.version_hash)):
if loads(key_pickle) == key and loads(version_pickle) == self.version:
self.db_conn.execute("delete from data where id = ?", (item_id,))
self.commit_countdown -= 1
if self.commit_countdown <= 0:
self.commit_countdown = self.commit_interval
self.db_conn.commit()
def __setitem__(self, key, value):
del self[key]
self.cache[key] = value
from pickle import dumps
self.db_conn.execute("insert into data"
" (key_hash, key_pickle, version_hash, "
" version_pickle, result_pickle)"
" values (?,?,?,?,?)",
(hash(key), sqlite.Binary(dumps(key)),
self.version_hash, self.version_pickle,
sqlite.Binary(dumps(value))))
self.commit_countdown -= 1
if self.commit_countdown <= 0:
self.commit_countdown = self.commit_interval
self.db_conn.commit()
try:
import sqlite3 as sqlite
except ImportError:
try:
from pysqlite2 import dbapi2 as sqlite
except ImportError:
import warnings
warnings.warn("DiskDict will be memory-only: "
"a usable version of sqlite was not found.")
DiskDict = dict # noqa
|