This file is indexed.

/usr/share/pyshared/CedarBackup2/extend/mbox.py is in cedar-backup2 2.22.0-1.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
# -*- coding: iso-8859-1 -*-
# vim: set ft=python ts=3 sw=3 expandtab:
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#              C E D A R
#          S O L U T I O N S       "Software done right."
#           S O F T W A R E
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Copyright (c) 2006-2007,2010 Kenneth J. Pronovici.
# All rights reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License,
# Version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Copies of the GNU General Public License are available from
# the Free Software Foundation website, http://www.gnu.org/.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Author   : Kenneth J. Pronovici <pronovic@ieee.org>
# Language : Python (>= 2.5)
# Project  : Official Cedar Backup Extensions
# Revision : $Id: mbox.py 1006 2010-07-07 21:03:57Z pronovic $
# Purpose  : Provides an extension to back up mbox email files.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

########################################################################
# Module documentation
########################################################################

"""
Provides an extension to back up mbox email files.

Backing up email
================

   Email folders (often stored as mbox flatfiles) are not well-suited being backed
   up with an incremental backup like the one offered by Cedar Backup.  This is
   because mbox files often change on a daily basis, forcing the incremental
   backup process to back them up every day in order to avoid losing data.  This
   can result in quite a bit of wasted space when backing up large folders.  (Note
   that the alternative maildir format does not share this problem, since it
   typically uses one file per message.)

   One solution to this problem is to design a smarter incremental backup process,
   which backs up baseline content on the first day of the week, and then backs up
   only new messages added to that folder on every other day of the week.  This way,
   the backup for any single day is only as large as the messages placed into the 
   folder on that day.  The backup isn't as "perfect" as the incremental backup
   process, because it doesn't preserve information about messages deleted from
   the backed-up folder.  However, it should be much more space-efficient, and
   in a recovery situation, it seems better to restore too much data rather
   than too little.

What is this extension?
=======================

   This is a Cedar Backup extension used to back up mbox email files via the Cedar
   Backup command line.  Individual mbox files or directories containing mbox
   files can be backed up using the same collect modes allowed for filesystems in
   the standard Cedar Backup collect action: weekly, daily, incremental.  It 
   implements the "smart" incremental backup process discussed above, using 
   functionality provided by the C{grepmail} utility.

   This extension requires a new configuration section <mbox> and is intended to
   be run either immediately before or immediately after the standard collect
   action.  Aside from its own configuration, it requires the options and collect
   configuration sections in the standard Cedar Backup configuration file.

   The mbox action is conceptually similar to the standard collect action,
   except that mbox directories are not collected recursively.  This implies
   some configuration changes (i.e. there's no need for global exclusions or an
   ignore file).  If you back up a directory, all of the mbox files in that
   directory are backed up into a single tar file using the indicated
   compression method.

@author: Kenneth J. Pronovici <pronovic@ieee.org>
"""

########################################################################
# Imported modules
########################################################################

# System modules
import os
import logging
import datetime
import pickle
import tempfile
from bz2 import BZ2File
from gzip import GzipFile

# Cedar Backup modules
from CedarBackup2.filesystem import FilesystemList, BackupFileList
from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
from CedarBackup2.util import resolveCommand, executeCommand
from CedarBackup2.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership


########################################################################
# Module-wide constants and variables
########################################################################

logger = logging.getLogger("CedarBackup2.log.extend.mbox")

GREPMAIL_COMMAND = [ "grepmail", ]
REVISION_PATH_EXTENSION = "mboxlast"


########################################################################
# MboxFile class definition
########################################################################

class MboxFile(object):

   """
   Class representing mbox file configuration..

   The following restrictions exist on data in this class:

      - The absolute path must be absolute.
      - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
      - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.

   @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode, compressMode
   """

   def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
      """
      Constructor for the C{MboxFile} class.

      You should never directly instantiate this class.
      
      @param absolutePath: Absolute path to an mbox file on disk.
      @param collectMode: Overridden collect mode for this directory.
      @param compressMode: Overridden compression mode for this directory.
      """
      self._absolutePath = None
      self._collectMode = None
      self._compressMode = None
      self.absolutePath = absolutePath
      self.collectMode = collectMode
      self.compressMode = compressMode

   def __repr__(self):
      """
      Official string representation for class instance.
      """
      return "MboxFile(%s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode)

   def __str__(self):
      """
      Informal string representation for class instance.
      """
      return self.__repr__()

   def __cmp__(self, other):
      """
      Definition of equals operator for this class.
      @param other: Other object to compare to.
      @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
      """
      if other is None:
         return 1
      if self.absolutePath != other.absolutePath:
         if self.absolutePath < other.absolutePath:
            return -1
         else:
            return 1
      if self.collectMode != other.collectMode:
         if self.collectMode < other.collectMode:
            return -1
         else:
            return 1
      if self.compressMode != other.compressMode:
         if self.compressMode < other.compressMode:
            return -1
         else:
            return 1
      return 0

   def _setAbsolutePath(self, value):
      """
      Property target used to set the absolute path.
      The value must be an absolute path if it is not C{None}.
      It does not have to exist on disk at the time of assignment.
      @raise ValueError: If the value is not an absolute path.
      @raise ValueError: If the value cannot be encoded properly.
      """
      if value is not None:
         if not os.path.isabs(value):
            raise ValueError("Absolute path must be, er, an absolute path.")
      self._absolutePath = encodePath(value)

   def _getAbsolutePath(self):
      """
      Property target used to get the absolute path.
      """
      return self._absolutePath

   def _setCollectMode(self, value):
      """
      Property target used to set the collect mode.
      If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COLLECT_MODES:
            raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
      self._collectMode = value

   def _getCollectMode(self):
      """
      Property target used to get the collect mode.
      """
      return self._collectMode

   def _setCompressMode(self, value):
      """
      Property target used to set the compress mode.
      If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COMPRESS_MODES:
            raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
      self._compressMode = value

   def _getCompressMode(self):
      """
      Property target used to get the compress mode.
      """
      return self._compressMode

   absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
   collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
   compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")


########################################################################
# MboxDir class definition
########################################################################

class MboxDir(object):

   """
   Class representing mbox directory configuration..

   The following restrictions exist on data in this class:

      - The absolute path must be absolute.
      - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
      - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.

   Unlike collect directory configuration, this is the only place exclusions
   are allowed (no global exclusions at the <mbox> configuration level).  Also,
   we only allow relative exclusions and there is no configured ignore file.
   This is because mbox directory backups are not recursive.

   @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode, 
          compressMode, relativeExcludePaths, excludePatterns
   """

   def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
                relativeExcludePaths=None, excludePatterns=None):
      """
      Constructor for the C{MboxDir} class.

      You should never directly instantiate this class.
      
      @param absolutePath: Absolute path to a mbox file on disk.
      @param collectMode: Overridden collect mode for this directory.
      @param compressMode: Overridden compression mode for this directory.
      @param relativeExcludePaths: List of relative paths to exclude.
      @param excludePatterns: List of regular expression patterns to exclude
      """
      self._absolutePath = None
      self._collectMode = None
      self._compressMode = None
      self._relativeExcludePaths = None
      self._excludePatterns = None
      self.absolutePath = absolutePath
      self.collectMode = collectMode
      self.compressMode = compressMode
      self.relativeExcludePaths = relativeExcludePaths
      self.excludePatterns = excludePatterns

   def __repr__(self):
      """
      Official string representation for class instance.
      """
      return "MboxDir(%s, %s, %s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode,
                                              self.relativeExcludePaths, self.excludePatterns)

   def __str__(self):
      """
      Informal string representation for class instance.
      """
      return self.__repr__()

   def __cmp__(self, other):
      """
      Definition of equals operator for this class.
      @param other: Other object to compare to.
      @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
      """
      if other is None:
         return 1
      if self.absolutePath != other.absolutePath:
         if self.absolutePath < other.absolutePath:
            return -1
         else:
            return 1
      if self.collectMode != other.collectMode:
         if self.collectMode < other.collectMode:
            return -1
         else:
            return 1
      if self.compressMode != other.compressMode:
         if self.compressMode < other.compressMode:
            return -1
         else:
            return 1
      if self.relativeExcludePaths != other.relativeExcludePaths:
         if self.relativeExcludePaths < other.relativeExcludePaths:
            return -1
         else:
            return 1
      if self.excludePatterns != other.excludePatterns:
         if self.excludePatterns < other.excludePatterns:
            return -1
         else:
            return 1
      return 0

   def _setAbsolutePath(self, value):
      """
      Property target used to set the absolute path.
      The value must be an absolute path if it is not C{None}.
      It does not have to exist on disk at the time of assignment.
      @raise ValueError: If the value is not an absolute path.
      @raise ValueError: If the value cannot be encoded properly.
      """
      if value is not None:
         if not os.path.isabs(value):
            raise ValueError("Absolute path must be, er, an absolute path.")
      self._absolutePath = encodePath(value)

   def _getAbsolutePath(self):
      """
      Property target used to get the absolute path.
      """
      return self._absolutePath

   def _setCollectMode(self, value):
      """
      Property target used to set the collect mode.
      If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COLLECT_MODES:
            raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
      self._collectMode = value

   def _getCollectMode(self):
      """
      Property target used to get the collect mode.
      """
      return self._collectMode

   def _setCompressMode(self, value):
      """
      Property target used to set the compress mode.
      If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COMPRESS_MODES:
            raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
      self._compressMode = value

   def _getCompressMode(self):
      """
      Property target used to get the compress mode.
      """
      return self._compressMode

   def _setRelativeExcludePaths(self, value):
      """
      Property target used to set the relative exclude paths list.
      Elements do not have to exist on disk at the time of assignment.
      """
      if value is None:
         self._relativeExcludePaths = None
      else:
         try:
            saved = self._relativeExcludePaths
            self._relativeExcludePaths = UnorderedList()
            self._relativeExcludePaths.extend(value)
         except Exception, e:
            self._relativeExcludePaths = saved
            raise e

   def _getRelativeExcludePaths(self):
      """
      Property target used to get the relative exclude paths list.
      """
      return self._relativeExcludePaths

   def _setExcludePatterns(self, value):
      """
      Property target used to set the exclude patterns list.
      """
      if value is None:
         self._excludePatterns = None
      else:
         try:
            saved = self._excludePatterns
            self._excludePatterns = RegexList()
            self._excludePatterns.extend(value)
         except Exception, e:
            self._excludePatterns = saved
            raise e

   def _getExcludePatterns(self):
      """
      Property target used to get the exclude patterns list.
      """
      return self._excludePatterns

   absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
   collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
   compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
   relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
   excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")


########################################################################
# MboxConfig class definition
########################################################################

class MboxConfig(object):

   """
   Class representing mbox configuration.

   Mbox configuration is used for backing up mbox email files.

   The following restrictions exist on data in this class:

      - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
      - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
      - The C{mboxFiles} list must be a list of C{MboxFile} objects
      - The C{mboxDirs} list must be a list of C{MboxDir} objects

   For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
   through the L{util.ObjectTypeList} list implementation that overrides common
   list methods and transparently ensures that each element is of the proper
   type.

   Unlike collect configuration, no global exclusions are allowed on this
   level.  We only allow relative exclusions at the mbox directory level.
   Also, there is no configured ignore file.  This is because mbox directory
   backups are not recursive.

   @note: Lists within this class are "unordered" for equality comparisons.

   @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, mboxFiles, mboxDirs
   """

   def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
      """
      Constructor for the C{MboxConfig} class.

      @param collectMode: Default collect mode.
      @param compressMode: Default compress mode.
      @param mboxFiles: List of mbox files to back up
      @param mboxDirs: List of mbox directories to back up

      @raise ValueError: If one of the values is invalid.
      """
      self._collectMode = None
      self._compressMode = None
      self._mboxFiles = None
      self._mboxDirs = None
      self.collectMode = collectMode
      self.compressMode = compressMode
      self.mboxFiles = mboxFiles
      self.mboxDirs = mboxDirs

   def __repr__(self):
      """
      Official string representation for class instance.
      """
      return "MboxConfig(%s, %s, %s, %s)" % (self.collectMode, self.compressMode, self.mboxFiles, self.mboxDirs)

   def __str__(self):
      """
      Informal string representation for class instance.
      """
      return self.__repr__()

   def __cmp__(self, other):
      """
      Definition of equals operator for this class.
      Lists within this class are "unordered" for equality comparisons.
      @param other: Other object to compare to.
      @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
      """
      if other is None:
         return 1
      if self.collectMode != other.collectMode:
         if self.collectMode < other.collectMode:
            return -1
         else:
            return 1
      if self.compressMode != other.compressMode:
         if self.compressMode < other.compressMode:
            return -1
         else:
            return 1
      if self.mboxFiles != other.mboxFiles:
         if self.mboxFiles < other.mboxFiles:
            return -1
         else:
            return 1
      if self.mboxDirs != other.mboxDirs:
         if self.mboxDirs < other.mboxDirs:
            return -1
         else:
            return 1
      return 0

   def _setCollectMode(self, value):
      """
      Property target used to set the collect mode.
      If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COLLECT_MODES:
            raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
      self._collectMode = value

   def _getCollectMode(self):
      """
      Property target used to get the collect mode.
      """
      return self._collectMode

   def _setCompressMode(self, value):
      """
      Property target used to set the compress mode.
      If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
      @raise ValueError: If the value is not valid.
      """
      if value is not None:
         if value not in VALID_COMPRESS_MODES:
            raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
      self._compressMode = value

   def _getCompressMode(self):
      """
      Property target used to get the compress mode.
      """
      return self._compressMode

   def _setMboxFiles(self, value):
      """
      Property target used to set the mboxFiles list.
      Either the value must be C{None} or each element must be an C{MboxFile}.
      @raise ValueError: If the value is not an C{MboxFile}
      """
      if value is None:
         self._mboxFiles = None
      else:
         try:
            saved = self._mboxFiles
            self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
            self._mboxFiles.extend(value)
         except Exception, e:
            self._mboxFiles = saved
            raise e

   def _getMboxFiles(self):
      """
      Property target used to get the mboxFiles list.
      """
      return self._mboxFiles

   def _setMboxDirs(self, value):
      """
      Property target used to set the mboxDirs list.
      Either the value must be C{None} or each element must be an C{MboxDir}.
      @raise ValueError: If the value is not an C{MboxDir}
      """
      if value is None:
         self._mboxDirs = None
      else:
         try:
            saved = self._mboxDirs
            self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
            self._mboxDirs.extend(value)
         except Exception, e:
            self._mboxDirs = saved
            raise e

   def _getMboxDirs(self):
      """
      Property target used to get the mboxDirs list.
      """
      return self._mboxDirs

   collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
   compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
   mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
   mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")


########################################################################
# LocalConfig class definition
########################################################################

class LocalConfig(object):

   """
   Class representing this extension's configuration document.

   This is not a general-purpose configuration object like the main Cedar
   Backup configuration object.  Instead, it just knows how to parse and emit
   Mbox-specific configuration values.  Third parties who need to read and
   write configuration related to this extension should access it through the
   constructor, C{validate} and C{addConfig} methods.

   @note: Lists within this class are "unordered" for equality comparisons.

   @sort: __init__, __repr__, __str__, __cmp__, mbox, validate, addConfig
   """

   def __init__(self, xmlData=None, xmlPath=None, validate=True):
      """
      Initializes a configuration object.

      If you initialize the object without passing either C{xmlData} or
      C{xmlPath} then configuration will be empty and will be invalid until it
      is filled in properly.

      No reference to the original XML data or original path is saved off by
      this class.  Once the data has been parsed (successfully or not) this
      original information is discarded.

      Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
      method will be called (with its default arguments) against configuration
      after successfully parsing any passed-in XML.  Keep in mind that even if
      C{validate} is C{False}, it might not be possible to parse the passed-in
      XML document if lower-level validations fail.

      @note: It is strongly suggested that the C{validate} option always be set
      to C{True} (the default) unless there is a specific need to read in
      invalid configuration from disk.  

      @param xmlData: XML data representing configuration.
      @type xmlData: String data.

      @param xmlPath: Path to an XML file on disk.
      @type xmlPath: Absolute path to a file on disk.

      @param validate: Validate the document after parsing it.
      @type validate: Boolean true/false.

      @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
      @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
      @raise ValueError: If the parsed configuration document is not valid.
      """
      self._mbox = None
      self.mbox = None
      if xmlData is not None and xmlPath is not None:
         raise ValueError("Use either xmlData or xmlPath, but not both.")
      if xmlData is not None:
         self._parseXmlData(xmlData)
         if validate:
            self.validate()
      elif xmlPath is not None:
         xmlData = open(xmlPath).read()
         self._parseXmlData(xmlData)
         if validate:
            self.validate()

   def __repr__(self):
      """
      Official string representation for class instance.
      """
      return "LocalConfig(%s)" % (self.mbox)

   def __str__(self):
      """
      Informal string representation for class instance.
      """
      return self.__repr__()

   def __cmp__(self, other):
      """
      Definition of equals operator for this class.
      Lists within this class are "unordered" for equality comparisons.
      @param other: Other object to compare to.
      @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
      """
      if other is None:
         return 1
      if self.mbox != other.mbox:
         if self.mbox < other.mbox:
            return -1
         else:
            return 1
      return 0

   def _setMbox(self, value):
      """
      Property target used to set the mbox configuration value.
      If not C{None}, the value must be a C{MboxConfig} object.
      @raise ValueError: If the value is not a C{MboxConfig}
      """
      if value is None:
         self._mbox = None
      else:
         if not isinstance(value, MboxConfig):
            raise ValueError("Value must be a C{MboxConfig} object.")
         self._mbox = value

   def _getMbox(self):
      """
      Property target used to get the mbox configuration value.
      """
      return self._mbox

   mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")

   def validate(self):
      """
      Validates configuration represented by the object.

      Mbox configuration must be filled in.  Within that, the collect mode and
      compress mode are both optional, but the list of repositories must
      contain at least one entry.

      Each configured file or directory must contain an absolute path, and then
      must be either able to take collect mode and compress mode configuration
      from the parent C{MboxConfig} object, or must set each value on its own.

      @raise ValueError: If one of the validations fails.
      """
      if self.mbox is None:
         raise ValueError("Mbox section is required.")
      if ((self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
          (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1)):
         raise ValueError("At least one mbox file or directory must be configured.")
      if self.mbox.mboxFiles is not None:
         for mboxFile in self.mbox.mboxFiles:
            if mboxFile.absolutePath is None:
               raise ValueError("Each mbox file must set an absolute path.")
            if self.mbox.collectMode is None and mboxFile.collectMode is None:
               raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
            if self.mbox.compressMode is None and mboxFile.compressMode is None:
               raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
      if self.mbox.mboxDirs is not None:
         for mboxDir in self.mbox.mboxDirs:
            if mboxDir.absolutePath is None:
               raise ValueError("Each mbox directory must set an absolute path.")
            if self.mbox.collectMode is None and mboxDir.collectMode is None:
               raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
            if self.mbox.compressMode is None and mboxDir.compressMode is None:
               raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")

   def addConfig(self, xmlDom, parentNode):
      """
      Adds an <mbox> configuration section as the next child of a parent.

      Third parties should use this function to write configuration related to
      this extension.

      We add the following fields to the document::

         collectMode    //cb_config/mbox/collectMode
         compressMode   //cb_config/mbox/compressMode

      We also add groups of the following items, one list element per
      item::

         mboxFiles      //cb_config/mbox/file
         mboxDirs       //cb_config/mbox/dir

      The mbox files and mbox directories are added by L{_addMboxFile} and
      L{_addMboxDir}.

      @param xmlDom: DOM tree as from C{impl.createDocument()}.
      @param parentNode: Parent that the section should be appended to.
      """
      if self.mbox is not None:
         sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
         addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
         addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
         if self.mbox.mboxFiles is not None:
            for mboxFile in self.mbox.mboxFiles:
               LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
         if self.mbox.mboxDirs is not None:
            for mboxDir in self.mbox.mboxDirs:
               LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)

   def _parseXmlData(self, xmlData):
      """
      Internal method to parse an XML string into the object.

      This method parses the XML document into a DOM tree (C{xmlDom}) and then
      calls a static method to parse the mbox configuration section.

      @param xmlData: XML data to be parsed
      @type xmlData: String data

      @raise ValueError: If the XML cannot be successfully parsed.
      """
      (xmlDom, parentNode) = createInputDom(xmlData)
      self._mbox = LocalConfig._parseMbox(parentNode)

   @staticmethod
   def _parseMbox(parent):
      """
      Parses an mbox configuration section.
      
      We read the following individual fields::

         collectMode    //cb_config/mbox/collect_mode
         compressMode   //cb_config/mbox/compress_mode

      We also read groups of the following item, one list element per
      item::

         mboxFiles      //cb_config/mbox/file
         mboxDirs       //cb_config/mbox/dir

      The mbox files are parsed by L{_parseMboxFiles} and the mbox
      directories are parsed by L{_parseMboxDirs}.

      @param parent: Parent node to search beneath.

      @return: C{MboxConfig} object or C{None} if the section does not exist.
      @raise ValueError: If some filled-in value is invalid.
      """
      mbox = None
      section = readFirstChild(parent, "mbox")
      if section is not None:
         mbox = MboxConfig()
         mbox.collectMode = readString(section, "collect_mode")
         mbox.compressMode = readString(section, "compress_mode")
         mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
         mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
      return mbox

   @staticmethod
   def _parseMboxFiles(parent):
      """
      Reads a list of C{MboxFile} objects from immediately beneath the parent.

      We read the following individual fields::

         absolutePath            abs_path
         collectMode             collect_mode
         compressMode            compess_mode 

      @param parent: Parent node to search beneath.

      @return: List of C{MboxFile} objects or C{None} if none are found.
      @raise ValueError: If some filled-in value is invalid.
      """
      lst = []
      for entry in readChildren(parent, "file"):
         if isElement(entry):
            mboxFile = MboxFile()
            mboxFile.absolutePath = readString(entry, "abs_path")
            mboxFile.collectMode = readString(entry, "collect_mode")
            mboxFile.compressMode = readString(entry, "compress_mode")
            lst.append(mboxFile)
      if lst == []:
         lst = None
      return lst

   @staticmethod
   def _parseMboxDirs(parent):
      """
      Reads a list of C{MboxDir} objects from immediately beneath the parent.

      We read the following individual fields::

         absolutePath            abs_path
         collectMode             collect_mode
         compressMode            compess_mode 

      We also read groups of the following items, one list element per
      item::

         relativeExcludePaths    exclude/rel_path
         excludePatterns         exclude/pattern

      The exclusions are parsed by L{_parseExclusions}.

      @param parent: Parent node to search beneath.

      @return: List of C{MboxDir} objects or C{None} if none are found.
      @raise ValueError: If some filled-in value is invalid.
      """
      lst = []
      for entry in readChildren(parent, "dir"):
         if isElement(entry):
            mboxDir = MboxDir()
            mboxDir.absolutePath = readString(entry, "abs_path")
            mboxDir.collectMode = readString(entry, "collect_mode")
            mboxDir.compressMode = readString(entry, "compress_mode")
            (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
            lst.append(mboxDir)
      if lst == []:
         lst = None
      return lst

   @staticmethod
   def _parseExclusions(parentNode):
      """
      Reads exclusions data from immediately beneath the parent.

      We read groups of the following items, one list element per item::

         relative    exclude/rel_path
         patterns    exclude/pattern

      If there are none of some pattern (i.e. no relative path items) then
      C{None} will be returned for that item in the tuple.  

      @param parentNode: Parent node to search beneath.

      @return: Tuple of (relative, patterns) exclusions.
      """
      section = readFirstChild(parentNode, "exclude")
      if section is None:
         return (None, None)
      else:
         relative = readStringList(section, "rel_path")
         patterns = readStringList(section, "pattern")
         return (relative, patterns)

   @staticmethod
   def _addMboxFile(xmlDom, parentNode, mboxFile):
      """
      Adds an mbox file container as the next child of a parent.

      We add the following fields to the document::

         absolutePath            file/abs_path
         collectMode             file/collect_mode
         compressMode            file/compress_mode

      The <file> node itself is created as the next child of the parent node.
      This method only adds one mbox file node.  The parent must loop for each
      mbox file in the C{MboxConfig} object.

      If C{mboxFile} is C{None}, this method call will be a no-op.

      @param xmlDom: DOM tree as from C{impl.createDocument()}.
      @param parentNode: Parent that the section should be appended to.
      @param mboxFile: MboxFile to be added to the document.
      """
      if mboxFile is not None:
         sectionNode = addContainerNode(xmlDom, parentNode, "file")
         addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
         addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
         addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)

   @staticmethod
   def _addMboxDir(xmlDom, parentNode, mboxDir):
      """
      Adds an mbox directory container as the next child of a parent.

      We add the following fields to the document::

         absolutePath            dir/abs_path
         collectMode             dir/collect_mode
         compressMode            dir/compress_mode

      We also add groups of the following items, one list element per item::

         relativeExcludePaths    dir/exclude/rel_path
         excludePatterns         dir/exclude/pattern

      The <dir> node itself is created as the next child of the parent node.
      This method only adds one mbox directory node.  The parent must loop for
      each mbox directory in the C{MboxConfig} object.

      If C{mboxDir} is C{None}, this method call will be a no-op.

      @param xmlDom: DOM tree as from C{impl.createDocument()}.
      @param parentNode: Parent that the section should be appended to.
      @param mboxDir: MboxDir to be added to the document.
      """
      if mboxDir is not None:
         sectionNode = addContainerNode(xmlDom, parentNode, "dir")
         addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
         addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
         addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
         if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
             (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
            excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
            if mboxDir.relativeExcludePaths is not None:
               for relativePath in mboxDir.relativeExcludePaths:
                  addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
            if mboxDir.excludePatterns is not None:
               for pattern in mboxDir.excludePatterns:
                  addStringNode(xmlDom, excludeNode, "pattern", pattern)


########################################################################
# Public functions
########################################################################

###########################
# executeAction() function
###########################

def executeAction(configPath, options, config):
   """
   Executes the mbox backup action.

   @param configPath: Path to configuration file on disk.
   @type configPath: String representing a path on disk.

   @param options: Program command-line options.
   @type options: Options object.

   @param config: Program configuration.
   @type config: Config object.

   @raise ValueError: Under many generic error conditions
   @raise IOError: If a backup could not be written for some reason.
   """
   logger.debug("Executing mbox extended action.")
   newRevision = datetime.datetime.today()  # mark here so all actions are after this date/time
   if config.options is None or config.collect is None:
      raise ValueError("Cedar Backup configuration is not properly filled in.")
   local = LocalConfig(xmlPath=configPath)
   todayIsStart = isStartOfWeek(config.options.startingDay)
   fullBackup = options.full or todayIsStart
   logger.debug("Full backup flag is [%s]" % fullBackup)
   if local.mbox.mboxFiles is not None:
      for mboxFile in local.mbox.mboxFiles:
         logger.debug("Working with mbox file [%s]" % mboxFile.absolutePath)
         collectMode = _getCollectMode(local, mboxFile)
         compressMode = _getCompressMode(local, mboxFile)
         lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
         if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
            logger.debug("Mbox file meets criteria to be backed up today.")
            _backupMboxFile(config, mboxFile.absolutePath, fullBackup, 
                            collectMode, compressMode, lastRevision, newRevision)
         else:
            logger.debug("Mbox file will not be backed up, per collect mode.")
         if collectMode == 'incr':
            _writeNewRevision(config, mboxFile, newRevision)
   if local.mbox.mboxDirs is not None:
      for mboxDir in local.mbox.mboxDirs:
         logger.debug("Working with mbox directory [%s]" % mboxDir.absolutePath)
         collectMode = _getCollectMode(local, mboxDir)
         compressMode = _getCompressMode(local, mboxDir)
         lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
         (excludePaths, excludePatterns) = _getExclusions(mboxDir)
         if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
            logger.debug("Mbox directory meets criteria to be backed up today.")
            _backupMboxDir(config, mboxDir.absolutePath, 
                           fullBackup, collectMode, compressMode, 
                           lastRevision, newRevision, 
                           excludePaths, excludePatterns)
         else:
            logger.debug("Mbox directory will not be backed up, per collect mode.")
         if collectMode == 'incr':
            _writeNewRevision(config, mboxDir, newRevision)
   logger.info("Executed the mbox extended action successfully.")

def _getCollectMode(local, item):
   """
   Gets the collect mode that should be used for an mbox file or directory. 
   Use file- or directory-specific value if possible, otherwise take from mbox section.
   @param local: LocalConfig object.
   @param item: Mbox file or directory
   @return: Collect mode to use.
   """
   if item.collectMode is None:
      collectMode = local.mbox.collectMode
   else:
      collectMode = item.collectMode
   logger.debug("Collect mode is [%s]" % collectMode)
   return collectMode

def _getCompressMode(local, item):
   """
   Gets the compress mode that should be used for an mbox file or directory.
   Use file- or directory-specific value if possible, otherwise take from mbox section.
   @param local: LocalConfig object.
   @param item: Mbox file or directory
   @return: Compress mode to use.
   """
   if item.compressMode is None:
      compressMode = local.mbox.compressMode
   else:
      compressMode = item.compressMode
   logger.debug("Compress mode is [%s]" % compressMode)
   return compressMode

def _getRevisionPath(config, item):
   """
   Gets the path to the revision file associated with a repository.
   @param config: Cedar Backup configuration.
   @param item: Mbox file or directory
   @return: Absolute path to the revision file associated with the repository.
   """
   normalized = buildNormalizedPath(item.absolutePath)
   filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
   revisionPath = os.path.join(config.options.workingDir, filename)
   logger.debug("Revision file path is [%s]" % revisionPath)
   return revisionPath

def _loadLastRevision(config, item, fullBackup, collectMode):
   """
   Loads the last revision date for this item from disk and returns it.

   If this is a full backup, or if the revision file cannot be loaded for some
   reason, then C{None} is returned.  This indicates that there is no previous
   revision, so the entire mail file or directory should be backed up.

   @note: We write the actual revision object to disk via pickle, so we don't
   deal with the datetime precision or format at all.  Whatever's in the object
   is what we write.

   @param config: Cedar Backup configuration.
   @param item: Mbox file or directory
   @param fullBackup: Indicates whether this is a full backup
   @param collectMode: Indicates the collect mode for this item

   @return: Revision date as a datetime.datetime object or C{None}.
   """
   revisionPath = _getRevisionPath(config, item)
   if fullBackup:
      revisionDate = None
      logger.debug("Revision file ignored because this is a full backup.")
   elif collectMode in ['weekly', 'daily']:
      revisionDate = None
      logger.debug("No revision file based on collect mode [%s]." % collectMode)
   else:
      logger.debug("Revision file will be used for non-full incremental backup.")
      if not os.path.isfile(revisionPath):
         revisionDate = None
         logger.debug("Revision file [%s] does not exist on disk." % revisionPath)
      else:
         try:
            revisionDate = pickle.load(open(revisionPath, "r"))
            logger.debug("Loaded revision file [%s] from disk: [%s]" % (revisionPath, revisionDate))
         except:
            revisionDate = None
            logger.error("Failed loading revision file [%s] from disk." % revisionPath)
   return revisionDate

def _writeNewRevision(config, item, newRevision):
   """
   Writes new revision information to disk.

   If we can't write the revision file successfully for any reason, we'll log
   the condition but won't throw an exception.

   @note: We write the actual revision object to disk via pickle, so we don't
   deal with the datetime precision or format at all.  Whatever's in the object
   is what we write.

   @param config: Cedar Backup configuration.
   @param item: Mbox file or directory
   @param newRevision: Revision date as a datetime.datetime object.
   """
   revisionPath = _getRevisionPath(config, item)
   try:
      pickle.dump(newRevision, open(revisionPath, "w"))
      changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
      logger.debug("Wrote new revision file [%s] to disk: [%s]" % (revisionPath, newRevision))
   except:
      logger.error("Failed to write revision file [%s] to disk." % revisionPath)

def _getExclusions(mboxDir):
   """
   Gets exclusions (file and patterns) associated with an mbox directory.

   The returned files value is a list of absolute paths to be excluded from the
   backup for a given directory.  It is derived from the mbox directory's
   relative exclude paths.
   
   The returned patterns value is a list of patterns to be excluded from the
   backup for a given directory.  It is derived from the mbox directory's list
   of patterns.

   @param mboxDir: Mbox directory object.

   @return: Tuple (files, patterns) indicating what to exclude.
   """
   paths = []
   if mboxDir.relativeExcludePaths is not None:
      for relativePath in mboxDir.relativeExcludePaths:
         paths.append(os.path.join(mboxDir.absolutePath, relativePath))
   patterns = []
   if mboxDir.excludePatterns is not None:
      patterns.extend(mboxDir.excludePatterns)
   logger.debug("Exclude paths: %s" % paths)
   logger.debug("Exclude patterns: %s" % patterns)
   return(paths, patterns)

def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
   """
   Gets the backup file path (including correct extension) associated with an mbox path.

   We assume that if the target directory is passed in, that we're backing up a
   directory.  Under these circumstances, we'll just use the basename of the
   individual path as the output file.

   @note: The backup path only contains the current date in YYYYMMDD format,
   but that's OK because the index information (stored elsewhere) is the actual
   date object.

   @param config: Cedar Backup configuration.
   @param mboxPath: Path to the indicated mbox file or directory
   @param compressMode: Compress mode to use for this mbox path
   @param newRevision: Revision this backup path represents
   @param targetDir: Target directory in which the path should exist

   @return: Absolute path to the backup file associated with the repository.
   """
   if targetDir is None:
      normalizedPath = buildNormalizedPath(mboxPath)
      revisionDate = newRevision.strftime("%Y%m%d")
      filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
   else:
      filename = os.path.basename(mboxPath)
   if compressMode == 'gzip':
      filename = "%s.gz" % filename
   elif compressMode == 'bzip2':
      filename = "%s.bz2" % filename
   if targetDir is None:
      backupPath = os.path.join(config.collect.targetDir, filename)
   else:
      backupPath = os.path.join(targetDir, filename)
   logger.debug("Backup file path is [%s]" % backupPath)
   return backupPath

def _getTarfilePath(config, mboxPath, compressMode, newRevision):
   """
   Gets the tarfile backup file path (including correct extension) associated
   with an mbox path.

   Along with the path, the tar archive mode is returned in a form that can
   be used with L{BackupFileList.generateTarfile}.

   @note: The tarfile path only contains the current date in YYYYMMDD format,
   but that's OK because the index information (stored elsewhere) is the actual
   date object.

   @param config: Cedar Backup configuration.
   @param mboxPath: Path to the indicated mbox file or directory
   @param compressMode: Compress mode to use for this mbox path
   @param newRevision: Revision this backup path represents

   @return: Tuple of (absolute path to tarfile, tar archive mode)
   """
   normalizedPath = buildNormalizedPath(mboxPath)
   revisionDate = newRevision.strftime("%Y%m%d")
   filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
   if compressMode == 'gzip':
      filename = "%s.gz" % filename
      archiveMode = "targz"
   elif compressMode == 'bzip2':
      filename = "%s.bz2" % filename
      archiveMode = "tarbz2"
   else:
      archiveMode = "tar"
   tarfilePath = os.path.join(config.collect.targetDir, filename)
   logger.debug("Tarfile path is [%s]" % tarfilePath)
   return (tarfilePath, archiveMode)

def _getOutputFile(backupPath, compressMode):
   """
   Opens the output file used for saving backup information.

   If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
   compress mode is "bzip2", we'll open a C{BZ2File}.  Otherwise, we'll just
   return an object from the normal C{open()} method.

   @param backupPath: Path to file to open.
   @param compressMode: Compress mode of file ("none", "gzip", "bzip").

   @return: Output file object.
   """
   if compressMode == "gzip":
      return GzipFile(backupPath, "w")
   elif compressMode == "bzip2":
      return BZ2File(backupPath, "w")
   else:
      return open(backupPath, "w")

def _backupMboxFile(config, absolutePath, 
                    fullBackup, collectMode, compressMode, 
                    lastRevision, newRevision, targetDir=None):
   """
   Backs up an individual mbox file.

   @param config: Cedar Backup configuration.
   @param absolutePath: Path to mbox file to back up.
   @param fullBackup: Indicates whether this should be a full backup.
   @param collectMode: Indicates the collect mode for this item
   @param compressMode: Compress mode of file ("none", "gzip", "bzip")
   @param lastRevision: Date of last backup as datetime.datetime
   @param newRevision: Date of new (current) backup as datetime.datetime
   @param targetDir: Target directory to write the backed-up file into
    
   @raise ValueError: If some value is missing or invalid.
   @raise IOError: If there is a problem backing up the mbox file.
   """
   backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
   outputFile = _getOutputFile(backupPath, compressMode)
   if fullBackup or collectMode != "incr" or lastRevision is None:
      args = [ "-a", "-u", absolutePath, ]  # remove duplicates but fetch entire mailbox
   else:
      revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")  # ISO-8601 format; grepmail calls Date::Parse::str2time()
      args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
   command = resolveCommand(GREPMAIL_COMMAND)
   result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
   if result != 0:
      raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
   logger.debug("Completed backing up mailbox [%s]." % absolutePath)
   return backupPath

def _backupMboxDir(config, absolutePath, 
                   fullBackup, collectMode, compressMode,
                   lastRevision, newRevision, 
                   excludePaths, excludePatterns):
   """
   Backs up a directory containing mbox files.

   @param config: Cedar Backup configuration.
   @param absolutePath: Path to mbox directory to back up.
   @param fullBackup: Indicates whether this should be a full backup.
   @param collectMode: Indicates the collect mode for this item
   @param compressMode: Compress mode of file ("none", "gzip", "bzip")
   @param lastRevision: Date of last backup as datetime.datetime
   @param newRevision: Date of new (current) backup as datetime.datetime
   @param excludePaths: List of absolute paths to exclude.
   @param excludePatterns: List of patterns to exclude.
    
   @raise ValueError: If some value is missing or invalid.
   @raise IOError: If there is a problem backing up the mbox file.
   """
   try:
      tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
      mboxList = FilesystemList()
      mboxList.excludeDirs = True
      mboxList.excludePaths = excludePaths
      mboxList.excludePatterns = excludePatterns
      mboxList.addDirContents(absolutePath, recursive=False)
      tarList = BackupFileList()
      for item in mboxList:
         backupPath = _backupMboxFile(config, item, fullBackup, 
                                      collectMode, "none",  # no need to compress inside compressed tar
                                      lastRevision, newRevision, 
                                      targetDir=tmpdir)
         tarList.addFile(backupPath)
      (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
      tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
      changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup) 
      logger.debug("Completed backing up directory [%s]." % absolutePath)
   finally:
      try:
         for item in tarList:
            if os.path.exists(item):
               try:
                  os.remove(item)
               except: pass
      except: pass
      try:
         os.rmdir(tmpdir)
      except: pass