/usr/lib/python2.7/dist-packages/barman/compression.py is in barman 2.3-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 | # Copyright (C) 2011-2017 2ndQuadrant Limited
#
# This file is part of Barman.
#
# Barman is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Barman is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Barman. If not, see <http://www.gnu.org/licenses/>.
"""
This module is responsible to manage the compression features of Barman
"""
import bz2
import gzip
import logging
import shutil
from abc import ABCMeta, abstractmethod
from contextlib import closing
from barman.command_wrappers import Command
from barman.exceptions import (CommandFailedException,
CompressionIncompatibility)
from barman.utils import with_metaclass
_logger = logging.getLogger(__name__)
class CompressionManager(object):
def __init__(self, config, path):
"""
Compression manager
"""
self.config = config
self.path = path
def check(self, compression=None):
"""
This method returns True if the compression specified in the
configuration file is present in the register, otherwise False
"""
if not compression:
compression = self.config.compression
if compression not in compression_registry:
return False
return True
def get_compressor(self, compression=None):
"""
Returns a new compressor instance
:param str compression: Compression name
"""
if not compression:
compression = self.config.compression
# Check if the requested compression mechanism is allowed
if self.check(compression):
return compression_registry[compression](
config=self.config, compression=compression, path=self.path)
else:
return None
def identify_compression(filename):
"""
Try to guess the compression algorithm of a file
:param filename: the pat of the file to identify
:rtype: str
"""
# TODO: manage multiple decompression methods for the same
# compression algorithm (e.g. what to do when gzip is detected?
# should we use gzip or pigz?)
with open(filename, 'rb') as f:
file_start = f.read(MAGIC_MAX_LENGTH)
for file_type, cls in sorted(compression_registry.items()):
if cls.validate(file_start):
return file_type
return None
class Compressor(with_metaclass(ABCMeta, object)):
"""
Base class for all the compressors
"""
MAGIC = None
def __init__(self, config, compression, path=None):
self.config = config
self.compression = compression
self.path = path
@classmethod
def validate(cls, file_start):
"""
Guess if the first bytes of a file are compatible with the compression
implemented by this class
:param file_start: a binary string representing the first few
bytes of a file
:rtype: bool
"""
return cls.MAGIC and file_start.startswith(cls.MAGIC)
@abstractmethod
def compress(self, src, dst):
"""
Abstract Method for compression method
:param str src: source file path
:param str dst: destination file path
"""
@abstractmethod
def decompress(self, src, dst):
"""
Abstract method for decompression method
:param str src: source file path
:param str dst: destination file path
"""
class CommandCompressor(Compressor):
"""
Base class for compressors built on external commands
"""
def __init__(self, config, compression, path=None):
super(CommandCompressor, self).__init__(
config, compression, path)
self._compress = None
self._decompress = None
def compress(self, src, dst):
"""
Compress using the specific command defined in the sublcass
:param src: source file to compress
:param dst: destination of the decompression
"""
return self._compress(src, dst)
def decompress(self, src, dst):
"""
Decompress using the specific command defined in the sublcass
:param src: source file to decompress
:param dst: destination of the decompression
"""
return self._decompress(src, dst)
def _build_command(self, pipe_command):
"""
Build the command string and create the actual Command object
:param pipe_command: the command used to compress/decompress
:rtype: Command
"""
command = 'command(){ '
command += pipe_command
command += ' > "$2" < "$1"'
command += ';}; command'
return Command(command, shell=True, check=True, path=self.path)
class InternalCompressor(Compressor):
"""
Base class for compressors built on python libraries
"""
def compress(self, src, dst):
"""
Compress using the object defined in the sublcass
:param src: source file to compress
:param dst: destination of the decompression
"""
try:
with open(src, 'rb') as istream:
with closing(self._compressor(dst)) as ostream:
shutil.copyfileobj(istream, ostream)
except Exception as e:
# you won't get more information from the compressors anyway
raise CommandFailedException(dict(ret=None, err=str(e), out=None))
return 0
def decompress(self, src, dst):
"""
Decompress using the object defined in the sublcass
:param src: source file to decompress
:param dst: destination of the decompression
"""
try:
with closing(self._decompressor(src)) as istream:
with open(dst, 'wb') as ostream:
shutil.copyfileobj(istream, ostream)
except Exception as e:
# you won't get more information from the compressors anyway
raise CommandFailedException(dict(ret=None, err=str(e), out=None))
return 0
@abstractmethod
def _decompressor(self, src):
"""
Abstract decompressor factory method
:param src: source file path
:return: a file-like readable decompressor object
"""
@abstractmethod
def _compressor(self, dst):
"""
Abstract compressor factory method
:param dst: destination file path
:return: a file-like writable compressor object
"""
class GZipCompressor(CommandCompressor):
"""
Predefined compressor with GZip
"""
MAGIC = b'\x1f\x8b\x08'
def __init__(self, config, compression, path=None):
super(GZipCompressor, self).__init__(
config, compression, path)
self._compress = self._build_command('gzip -c')
self._decompress = self._build_command('gzip -c -d')
class PyGZipCompressor(InternalCompressor):
"""
Predefined compressor that uses GZip Python libraries
"""
MAGIC = b'\x1f\x8b\x08'
def __init__(self, config, compression, path=None):
super(PyGZipCompressor, self).__init__(
config, compression, path)
# Default compression level used in system gzip utility
self._level = -1 # Z_DEFAULT_COMPRESSION constant of zlib
def _compressor(self, name):
return gzip.GzipFile(name, mode='wb', compresslevel=self._level)
def _decompressor(self, name):
return gzip.GzipFile(name, mode='rb')
class PigzCompressor(CommandCompressor):
"""
Predefined compressor with Pigz
Note that pigz on-disk is the same as gzip, so the MAGIC value of this
class is the same
"""
MAGIC = b'\x1f\x8b\x08'
def __init__(self, config, compression, path=None):
super(PigzCompressor, self).__init__(
config, compression, path)
self._compress = self._build_command('pigz -c')
self._decompress = self._build_command('pigz -c -d')
class BZip2Compressor(CommandCompressor):
"""
Predefined compressor with BZip2
"""
MAGIC = b'\x42\x5a\x68'
def __init__(self, config, compression, path=None):
super(BZip2Compressor, self).__init__(
config, compression, path)
self._compress = self._build_command('bzip2 -c')
self._decompress = self._build_command('bzip2 -c -d')
class PyBZip2Compressor(InternalCompressor):
"""
Predefined compressor with BZip2 Python libraries
"""
MAGIC = b'\x42\x5a\x68'
def __init__(self, config, compression, path=None):
super(PyBZip2Compressor, self).__init__(
config, compression, path)
# Default compression level used in system gzip utility
self._level = 9
def _compressor(self, name):
return bz2.BZ2File(name, mode='wb', compresslevel=self._level)
def _decompressor(self, name):
return bz2.BZ2File(name, mode='rb')
class CustomCompressor(CommandCompressor):
"""
Custom compressor
"""
def __init__(self, config, compression, path=None):
if not config.custom_compression_filter:
raise CompressionIncompatibility("custom_compression_filter")
if not config.custom_decompression_filter:
raise CompressionIncompatibility("custom_decompression_filter")
super(CustomCompressor, self).__init__(
config, compression, path)
self._compress = self._build_command(
config.custom_compression_filter)
self._decompress = self._build_command(
config.custom_decompression_filter)
# a dictionary mapping all supported compression schema
# to the class implementing it
# WARNING: items in this dictionary are extracted using alphabetical order
# It's important that gzip and bzip2 are positioned before their variants
compression_registry = {
'gzip': GZipCompressor,
'pigz': PigzCompressor,
'bzip2': BZip2Compressor,
'pygzip': PyGZipCompressor,
'pybzip2': PyBZip2Compressor,
'custom': CustomCompressor,
}
#: The longest string needed to identify a compression schema
MAGIC_MAX_LENGTH = max(len(x.MAGIC or '')
for x in compression_registry.values())
|