This file is indexed.

/usr/share/pyshared/epydoc/docstringparser.py is in python-epydoc 3.0.1+dfsg-5.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
# epydoc -- Docstring processing
#
# Copyright (C) 2005 Edward Loper
# Author: Edward Loper <edloper@loper.org>
# URL: <http://epydoc.sf.net>
#
# $Id: docstringparser.py 1689 2008-01-30 17:01:02Z edloper $

"""
Parse docstrings and handle any fields it defines, such as C{@type}
and C{@author}.  Fields are used to describe specific information
about an object.  There are two classes of fields: X{simple fields}
and X{special fields}.

Simple fields are fields that get stored directly in an C{APIDoc}'s
metadata dictionary, without any special processing.  The set of
simple fields is defined by the list L{STANDARD_FIELDS}, whose
elements are L{DocstringField}s.

Special fields are fields that perform some sort of processing on the
C{APIDoc}, or add information to attributes other than the metadata
dictionary.  Special fields are are handled by field handler
functions, which are registered using L{register_field_handler}.
"""
__docformat__ = 'epytext en'


######################################################################
## Imports
######################################################################

import re, sys
from epydoc import markup
from epydoc.markup import epytext
from epydoc.apidoc import *
from epydoc.docintrospecter import introspect_docstring_lineno
from epydoc.util import py_src_filename
from epydoc import log
import epydoc.docparser
import __builtin__, exceptions

######################################################################
# Docstring Fields
######################################################################

class DocstringField:
    """
    A simple docstring field, which can be used to describe specific
    information about an object, such as its author or its version.
    Simple docstring fields are fields that take no arguments, and
    are displayed as simple sections.

    @ivar tags: The set of tags that can be used to identify this
        field.
    @ivar singular: The label that should be used to identify this
        field in the output, if the field contains one value.
    @ivar plural: The label that should be used to identify this
        field in the output, if the field contains multiple values.
    @ivar short: If true, then multiple values should be combined
        into a single comma-delimited list.  If false, then
        multiple values should be listed separately in a bulleted
        list.
    @ivar multivalue: If true, then multiple values may be given
        for this field; if false, then this field can only take a
        single value, and a warning should be issued if it is
        redefined.
    @ivar takes_arg: If true, then this field expects an argument;
        and a separate field section will be constructed for each
        argument value.  The label (and plural label) should include
        a '%s' to mark where the argument's string rep should be
        added.
    """
    def __init__(self, tags, label, plural=None,
                 short=0, multivalue=1, takes_arg=0,
                 varnames=None):
        if type(tags) in (list, tuple):
            self.tags = tuple(tags)
        elif type(tags) is str:
            self.tags = (tags,)
        else: raise TypeError('Bad tags: %s' % tags)
        self.singular = label
        if plural is None: self.plural = label
        else: self.plural = plural
        self.multivalue = multivalue
        self.short = short
        self.takes_arg = takes_arg
        self.varnames = varnames or []

    def __cmp__(self, other):
        if not isinstance(other, DocstringField): return -1
        return cmp(self.tags, other.tags)
    
    def __hash__(self):
        return hash(self.tags)

    def __repr__(self):
        return '<Field: %s>' % self.tags[0]

STANDARD_FIELDS = [
    #: A list of the standard simple fields accepted by epydoc.  This
    #: list can be augmented at run-time by a docstring with the special
    #: C{@deffield} field.  The order in which fields are listed here
    #: determines the order in which they will be displayed in the
    #: output.
    
    # If it's deprecated, put that first.
    DocstringField(['deprecated', 'depreciated'],
             'Deprecated', multivalue=0, varnames=['__deprecated__']),

    # Status info
    DocstringField(['version'], 'Version', multivalue=0,
                   varnames=['__version__']),
    DocstringField(['date'], 'Date', multivalue=0,
                   varnames=['__date__']),
    DocstringField(['status'], 'Status', multivalue=0),
    
    # Bibliographic Info
    DocstringField(['author', 'authors'], 'Author', 'Authors', short=1,
                   varnames=['__author__', '__authors__']),
    DocstringField(['contact'], 'Contact', 'Contacts', short=1,
                   varnames=['__contact__']),
    DocstringField(['organization', 'org'],
                   'Organization', 'Organizations'),
    DocstringField(['copyright', '(c)'], 'Copyright', multivalue=0,
                   varnames=['__copyright__']),
    DocstringField(['license'], 'License', multivalue=0,
                   varnames=['__license__']),

    # Various warnings etc.
    DocstringField(['bug'], 'Bug', 'Bugs'),
    DocstringField(['warning', 'warn'], 'Warning', 'Warnings'),
    DocstringField(['attention'], 'Attention'),
    DocstringField(['note'], 'Note', 'Notes'),

    # Formal conditions
    DocstringField(['requires', 'require', 'requirement'], 'Requires'),
    DocstringField(['precondition', 'precond'],
             'Precondition', 'Preconditions'),
    DocstringField(['postcondition', 'postcond'],
             'Postcondition', 'Postconditions'),
    DocstringField(['invariant'], 'Invariant'),

    # When was it introduced (version # or date)
    DocstringField(['since'], 'Since', multivalue=0),

    # Changes made
    DocstringField(['change', 'changed'], 'Change Log'),
                   
    # Crossreferences
    DocstringField(['see', 'seealso'], 'See Also', short=1),

    # Future Work
    DocstringField(['todo'], 'To Do', takes_arg=True),

    # Permissions (used by zope-based projects)
    DocstringField(['permission', 'permissions'], 'Permission', 'Permissions')
    ]

######################################################################
#{ Docstring Parsing
######################################################################

DEFAULT_DOCFORMAT = 'epytext'
"""The name of the default markup languge used to process docstrings."""

# [xx] keep track of which ones we've already done, in case we're
# asked to process one twice?  e.g., for @include we might have to
# parse the included docstring earlier than we might otherwise..??

def parse_docstring(api_doc, docindex, suppress_warnings=[]):
    """
    Process the given C{APIDoc}'s docstring.  In particular, populate
    the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
    information provided by fields in the docstring.
    
    @param docindex: A DocIndex, used to find the containing
        module (to look up the docformat); and to find any
        user docfields defined by containing objects.
    @param suppress_warnings: A set of objects for which docstring
        warnings should be suppressed.
    """
    if api_doc.metadata is not UNKNOWN:
        if not (isinstance(api_doc, RoutineDoc)
                and api_doc.canonical_name[-1] == '__init__'):
            log.debug("%s's docstring processed twice" %
                      api_doc.canonical_name)
        return
        
    initialize_api_doc(api_doc)

    # If there's no docstring, then check for special variables (e.g.,
    # __version__), and then return -- there's nothing else to do.
    if (api_doc.docstring in (None, UNKNOWN)):
        if isinstance(api_doc, NamespaceDoc):
            for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
                add_metadata_from_var(api_doc, field)
        return

    # Remove leading indentation from the docstring.
    api_doc.docstring = unindent_docstring(api_doc.docstring)

    # Decide which docformat is used by this module.
    docformat = get_docformat(api_doc, docindex)

    # A list of markup errors from parsing.
    parse_errors = []
    
    # Extract a signature from the docstring, if it has one.  This
    # overrides any signature we got via introspection/parsing.
    if isinstance(api_doc, RoutineDoc):
        parse_function_signature(api_doc, None, docformat, parse_errors)

    # Parse the docstring.  Any errors encountered are stored as
    # `ParseError` objects in the errors list.
    parsed_docstring = markup.parse(api_doc.docstring, docformat,
                                    parse_errors)
        
    # Divide the docstring into a description and a list of
    # fields.
    descr, fields = parsed_docstring.split_fields(parse_errors)
    api_doc.descr = descr

    field_warnings = []

    # Handle the constructor fields that have been defined in the class
    # docstring. This code assumes that a class docstring is parsed before
    # the same class __init__ docstring.
    if isinstance(api_doc, ClassDoc):

        # Parse ahead the __init__ docstring for this class
        initvar = api_doc.variables.get('__init__')
        if initvar and isinstance(initvar.value, RoutineDoc):
            init_api_doc = initvar.value
            parse_docstring(init_api_doc, docindex, suppress_warnings)

            parse_function_signature(init_api_doc, api_doc,
                                     docformat, parse_errors)
            init_fields = split_init_fields(fields, field_warnings)

            # Process fields
            for field in init_fields:
                try:
                    process_field(init_api_doc, docindex, field.tag(),
                                    field.arg(), field.body())
                except ValueError, e: field_warnings.append(str(e))

    # Process fields
    for field in fields:
        try:
            process_field(api_doc, docindex, field.tag(),
                               field.arg(), field.body())
        except ValueError, e: field_warnings.append(str(e))

    # Check to make sure that all type parameters correspond to
    # some documented parameter.
    check_type_fields(api_doc, field_warnings)

    # Check for special variables (e.g., __version__)
    if isinstance(api_doc, NamespaceDoc):
        for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
            add_metadata_from_var(api_doc, field)

    # Extract a summary
    if api_doc.summary is None and api_doc.descr is not None:
        api_doc.summary, api_doc.other_docs = api_doc.descr.summary()

    # If the summary is empty, but the return field is not, then use
    # the return field to generate a summary description.
    if (isinstance(api_doc, RoutineDoc) and api_doc.summary is None and
        api_doc.return_descr is not None):
        s, o = api_doc.return_descr.summary()
        api_doc.summary = RETURN_PDS + s
        api_doc.other_docs = o

    # [XX] Make sure we don't have types/param descrs for unknown
    # vars/params?

    # Report any errors that occured
    if api_doc in suppress_warnings:
        if parse_errors or field_warnings:
            log.info("Suppressing docstring warnings for %s, since it "
                     "is not included in the documented set." %
                     api_doc.canonical_name)
    else:
        report_errors(api_doc, docindex, parse_errors, field_warnings)

def add_metadata_from_var(api_doc, field):
    for varname in field.varnames:
        # Check if api_doc has a variable w/ the given name.
        if varname not in api_doc.variables: continue

        # Check moved here from before the for loop because we expect to
        # reach rarely this point. The loop below is to be performed more than
        # once only for fields with more than one varname, which currently is
        # only 'author'.
        for md in api_doc.metadata:
            if field == md[0]:
                return # We already have a value for this metadata.

        var_doc = api_doc.variables[varname]
        if var_doc.value is UNKNOWN: continue
        val_doc = var_doc.value
        value = []

        # Try extracting the value from the pyval.
        ok_types = (basestring, int, float, bool, type(None))
        if val_doc.pyval is not UNKNOWN:
            if isinstance(val_doc.pyval, ok_types):
                value = [val_doc.pyval]
            elif field.multivalue:
                if isinstance(val_doc.pyval, (tuple, list)):
                    for elt in val_doc.pyval:
                        if not isinstance(elt, ok_types): break
                    else:
                        value = list(val_doc.pyval)

        # Try extracting the value from the parse tree.
        elif val_doc.toktree is not UNKNOWN:
            try: value = [epydoc.docparser.parse_string(val_doc.toktree)]
            except KeyboardInterrupt: raise
            except: pass
            if field.multivalue and not value:
                try: value = epydoc.docparser.parse_string_list(val_doc.toktree)
                except KeyboardInterrupt: raise
                except: raise
                
        # Add any values that we found.
        for elt in value:
            if isinstance(elt, str):
                elt = decode_with_backslashreplace(elt)
            else:
                elt = unicode(elt)
            elt = epytext.ParsedEpytextDocstring(
                epytext.parse_as_para(elt), inline=True)

            # Add in the metadata and remove from the variables
            api_doc.metadata.append( (field, varname, elt) )

        # Remove the variable itself (unless it's documented)
        if var_doc.docstring in (None, UNKNOWN):
            del api_doc.variables[varname]
            if api_doc.sort_spec is not UNKNOWN:
                try: api_doc.sort_spec.remove(varname)
                except ValueError: pass

def initialize_api_doc(api_doc):
    """A helper function for L{parse_docstring()} that initializes
    the attributes that C{parse_docstring()} will write to."""
    if api_doc.descr is UNKNOWN:
        api_doc.descr = None
    if api_doc.summary is UNKNOWN:
        api_doc.summary = None
    if api_doc.metadata is UNKNOWN:
        api_doc.metadata = []
    if isinstance(api_doc, RoutineDoc):
        if api_doc.arg_descrs is UNKNOWN:
            api_doc.arg_descrs = []
        if api_doc.arg_types is UNKNOWN:
            api_doc.arg_types = {}
        if api_doc.return_descr is UNKNOWN:
            api_doc.return_descr = None
        if api_doc.return_type is UNKNOWN:
            api_doc.return_type = None
        if api_doc.exception_descrs is UNKNOWN:
            api_doc.exception_descrs = []
    if isinstance(api_doc, (VariableDoc, PropertyDoc)):
        if api_doc.type_descr is UNKNOWN:
            api_doc.type_descr = None
    if isinstance(api_doc, NamespaceDoc):
        if api_doc.group_specs is UNKNOWN:
            api_doc.group_specs = []
        if api_doc.sort_spec is UNKNOWN:
            api_doc.sort_spec = []

def split_init_fields(fields, warnings):
    """
    Remove the fields related to the constructor from a class docstring
    fields list.

    @param fields: The fields to process. The list will be modified in place
    @type fields: C{list} of L{markup.Field}
    @param warnings: A list to emit processing warnings
    @type warnings: C{list}
    @return: The C{fields} items to be applied to the C{__init__} method
    @rtype: C{list} of L{markup.Field}
    """
    init_fields = []

    # Split fields in lists according to their argument, keeping order.
    arg_fields = {}
    args_order = []
    i = 0
    while i < len(fields):
        field = fields[i]

        # gather together all the fields with the same arg
        if field.arg() is not None:
            arg_fields.setdefault(field.arg(), []).append(fields.pop(i))
            args_order.append(field.arg())
        else:
            i += 1

    # Now check that for each argument there is at most a single variable
    # and a single parameter, and at most a single type for each of them.
    for arg in args_order:
        ff = arg_fields.pop(arg, None)
        if ff is None:
            continue

        var = tvar = par = tpar = None
        for field in ff:
            if field.tag() in VARIABLE_TAGS:
                if var is None:
                    var = field
                    fields.append(field)
                else:
                    warnings.append(
                        "There is more than one variable named '%s'"
                        % arg)
            elif field.tag() in PARAMETER_TAGS:
                if par is None:
                    par = field
                    init_fields.append(field)
                else:
                    warnings.append(
                        "There is more than one parameter named '%s'"
                        % arg)

            elif field.tag() == 'type':
                if var is None and par is None:
                    # type before obj
                    tvar = tpar = field
                else:
                    if var is not None and tvar is None:
                        tvar = field
                    if par is not None and tpar is None:
                        tpar = field

            elif field.tag() in EXCEPTION_TAGS:
                init_fields.append(field)

            else: # Unespected field
                fields.append(field)

        # Put selected types into the proper output lists
        if tvar is not None:
            if var is not None:
                fields.append(tvar)
            else:
                pass # [xx] warn about type w/o object?

        if tpar is not None:
            if par is not None:
                init_fields.append(tpar)
            else:
                pass # [xx] warn about type w/o object?

    return init_fields

def report_errors(api_doc, docindex, parse_errors, field_warnings):
    """A helper function for L{parse_docstring()} that reports any
    markup warnings and field warnings that we encountered while
    processing C{api_doc}'s docstring."""
    if not parse_errors and not field_warnings: return

    # Get the name of the item containing the error, and the
    # filename of its containing module.
    name = api_doc.canonical_name
    module = api_doc.defining_module
    if module is not UNKNOWN and module.filename not in (None, UNKNOWN):
        try: filename = py_src_filename(module.filename)
        except: filename = module.filename
    else:
        filename = '??'

    # [xx] Don't report markup errors for standard builtins.
    # n.b. that we must use 'is' to compare pyvals here -- if we use
    # 'in' or '==', then a user __cmp__ method might raise an
    # exception, or lie.
    if isinstance(api_doc, ValueDoc) and api_doc != module:
        if module not in (None, UNKNOWN) and module.pyval is exceptions:
            return
        for builtin_val in __builtin__.__dict__.values():
            if builtin_val is api_doc.pyval:
                return
        
    # Get the start line of the docstring containing the error.
    startline = api_doc.docstring_lineno
    if startline in (None, UNKNOWN):
        startline = introspect_docstring_lineno(api_doc)
        if startline in (None, UNKNOWN):
            startline = None

    # Display a block header.
    header = 'File %s, ' % filename
    if startline is not None:
        header += 'line %d, ' % startline
    header += 'in %s' % name
    log.start_block(header)
    

    # Display all parse errors.  But first, combine any errors
    # with duplicate description messages.
    if startline is None:
        # remove dups, but keep original order:
        dups = {}
        for error in parse_errors:
            message = error.descr()
            if message not in dups:
                log.docstring_warning(message)
                dups[message] = 1
    else:
        # Combine line number fields for dup messages:
        messages = {} # maps message -> list of linenum
        for error in parse_errors:
            error.set_linenum_offset(startline)
            message = error.descr()
            messages.setdefault(message, []).append(error.linenum())
        message_items = messages.items()
        message_items.sort(lambda a,b:cmp(min(a[1]), min(b[1])))
        for message, linenums in message_items:
            linenums = [n for n in linenums if n is not None]
            if len(linenums) == 0:
                log.docstring_warning(message)
            elif len(linenums) == 1:
                log.docstring_warning("Line %s: %s" % (linenums[0], message))
            else:
                linenums = ', '.join(['%s' % l for l in linenums])
                log.docstring_warning("Lines %s: %s" % (linenums, message))

    # Display all field warnings.
    for warning in field_warnings:
        log.docstring_warning(warning)

    # End the message block.
    log.end_block()

RETURN_PDS = markup.parse('Returns:', markup='epytext')
"""A ParsedDocstring containing the text 'Returns'.  This is used to
construct summary descriptions for routines that have empty C{descr},
but non-empty C{return_descr}."""
RETURN_PDS._tree.children[0].attribs['inline'] = True

######################################################################
#{ Field Processing Error Messages
######################################################################

UNEXPECTED_ARG = '%r did not expect an argument'
EXPECTED_ARG = '%r expected an argument'
EXPECTED_SINGLE_ARG = '%r expected a single argument'
BAD_CONTEXT = 'Invalid context for %r'
REDEFINED = 'Redefinition of %s'
UNKNOWN_TAG = 'Unknown field tag %r'
BAD_PARAM = '@%s for unknown parameter %s'

######################################################################
#{ Field Processing
######################################################################

def process_field(api_doc, docindex, tag, arg, descr):
    """
    Process a single field, and use it to update C{api_doc}.  If
    C{tag} is the name of a special field, then call its handler
    function.  If C{tag} is the name of a simple field, then use
    C{process_simple_field} to process it.  Otherwise, check if it's a
    user-defined field, defined in this docstring or the docstring of
    a containing object; and if so, process it with
    C{process_simple_field}.

    @param tag: The field's tag, such as C{'author'}
    @param arg: The field's optional argument
    @param descr: The description following the field tag and
        argument.
    @raise ValueError: If a problem was encountered while processing
        the field.  The C{ValueError}'s string argument is an
        explanation of the problem, which should be displayed as a
        warning message.
    """
    # standard special fields
    if tag in _field_dispatch_table:
        handler = _field_dispatch_table[tag]
        handler(api_doc, docindex, tag, arg, descr)
        return

    # standard simple fields & user-defined fields
    for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
        if tag in field.tags:
            # [xx] check if it's redefined if it's not multivalue??
            if not field.takes_arg:
                _check(api_doc, tag, arg, expect_arg=False)
            api_doc.metadata.append((field, arg, descr))
            return

    # If we didn't handle the field, then report a warning.
    raise ValueError(UNKNOWN_TAG % tag)

def user_docfields(api_doc, docindex):
    """
    Return a list of user defined fields that can be used for the
    given object.  This list is taken from the given C{api_doc}, and
    any of its containing C{NamepaceDoc}s.

    @note: We assume here that a parent's docstring will always be
        parsed before its childrens'.  This is indeed the case when we
        are called via L{docbuilder.build_doc_index()}.  If a child's
        docstring is parsed before its parents, then its parent won't
        yet have had its C{extra_docstring_fields} attribute
        initialized.
    """
    docfields = []
    # Get any docfields from `api_doc` itself
    if api_doc.extra_docstring_fields not in (None, UNKNOWN):
        docfields += api_doc.extra_docstring_fields
    # Get any docfields from `api_doc`'s ancestors
    for i in range(len(api_doc.canonical_name)-1, 0, -1):
        ancestor = docindex.get_valdoc(api_doc.canonical_name[:i])
        if ancestor is not None \
        and ancestor.extra_docstring_fields not in (None, UNKNOWN):
            docfields += ancestor.extra_docstring_fields
    return docfields

_field_dispatch_table = {}
def register_field_handler(handler, *field_tags):
    """
    Register the given field handler function for processing any
    of the given field tags.  Field handler functions should
    have the following signature:

        >>> def field_handler(api_doc, docindex, tag, arg, descr):
        ...     '''update api_doc in response to the field.'''

    Where C{api_doc} is the documentation object to update;
    C{docindex} is a L{DocIndex} that can be used to look up the
    documentation for related objects; C{tag} is the field tag that
    was used; C{arg} is the optional argument; and C{descr} is the
    description following the field tag and argument.
    """
    for field_tag in field_tags:
        _field_dispatch_table[field_tag] = handler

######################################################################
#{ Field Handler Functions
######################################################################

def process_summary_field(api_doc, docindex, tag, arg, descr):
    """Store C{descr} in C{api_doc.summary}"""
    _check(api_doc, tag, arg, expect_arg=False)
    if api_doc.summary is not None:
        raise ValueError(REDEFINED % tag)
    api_doc.summary = descr

def process_include_field(api_doc, docindex, tag, arg, descr):
    """Copy the docstring contents from the object named in C{descr}"""
    _check(api_doc, tag, arg, expect_arg=False)
    # options:
    #   a. just append the descr to our own
    #   b. append descr and update metadata
    #   c. append descr and process all fields.
    # in any case, mark any errors we may find as coming from an
    # imported docstring.
    
    # how does this interact with documentation inheritance??
    raise ValueError('%s not implemented yet' % tag)

def process_undocumented_field(api_doc, docindex, tag, arg, descr):
    """Remove any documentation for the variables named in C{descr}"""
    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False)
    for ident in _descr_to_identifiers(descr):
        var_name_re = re.compile('^%s$' % ident.replace('*', '(.*)'))
        for var_name, var_doc in api_doc.variables.items():
            if var_name_re.match(var_name):
                # Remove the variable from `variables`.
                api_doc.variables.pop(var_name, None)
                if api_doc.sort_spec is not UNKNOWN:
                    try: api_doc.sort_spec.remove(var_name)
                    except ValueError: pass
        # For modules, remove any submodules that match var_name_re.
        if isinstance(api_doc, ModuleDoc):
            removed = set([m for m in api_doc.submodules
                           if var_name_re.match(m.canonical_name[-1])])
            if removed:
                # Remove the indicated submodules from this module.
                api_doc.submodules = [m for m in api_doc.submodules
                                      if m not in removed]
                # Remove all ancestors of the indicated submodules
                # from the docindex root.  E.g., if module x
                # declares y to be undocumented, then x.y.z should
                # also be undocumented.
                for elt in docindex.root[:]:
                    for m in removed:
                        if m.canonical_name.dominates(elt.canonical_name):
                            docindex.root.remove(elt)

def process_group_field(api_doc, docindex, tag, arg, descr):
    """Define a group named C{arg} containing the variables whose
    names are listed in C{descr}."""
    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=True)
    api_doc.group_specs.append( (arg, _descr_to_identifiers(descr)) )
    # [xx] should this also set sort order?

def process_deffield_field(api_doc, docindex, tag, arg, descr):
    """Define a new custom field."""
    _check(api_doc, tag, arg, expect_arg=True)
    if api_doc.extra_docstring_fields is UNKNOWN:
        api_doc.extra_docstring_fields = []
    try:
        docstring_field = _descr_to_docstring_field(arg, descr)
        docstring_field.varnames.append("__%s__" % arg)
        api_doc.extra_docstring_fields.append(docstring_field)
    except ValueError, e:
        raise ValueError('Bad %s: %s' % (tag, e))

def process_raise_field(api_doc, docindex, tag, arg, descr):
    """Record the fact that C{api_doc} can raise the exception named
    C{tag} in C{api_doc.exception_descrs}."""
    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg='single')
    try: name = DottedName(arg, strict=True)
    except DottedName.InvalidDottedName: name = arg
    api_doc.exception_descrs.append( (name, descr) )

def process_sort_field(api_doc, docindex, tag, arg, descr):
    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False)
    api_doc.sort_spec = _descr_to_identifiers(descr) + api_doc.sort_spec

# [xx] should I notice when they give a type for an unknown var?
def process_type_field(api_doc, docindex, tag, arg, descr):
    # In namespace, "@type var: ..." describes the type of a var.
    if isinstance(api_doc, NamespaceDoc):
        _check(api_doc, tag, arg, expect_arg='single')
        set_var_type(api_doc, arg, descr)

    # For variables & properties, "@type: ..." describes the variable.
    elif isinstance(api_doc, (VariableDoc, PropertyDoc)):
        _check(api_doc, tag, arg, expect_arg=False)
        if api_doc.type_descr is not None:
            raise ValueError(REDEFINED % tag)
        api_doc.type_descr = descr

    # For routines, "@type param: ..." describes a parameter.
    elif isinstance(api_doc, RoutineDoc):
        _check(api_doc, tag, arg, expect_arg='single')
        if arg in api_doc.arg_types:
            raise ValueError(REDEFINED % ('type for '+arg))
        api_doc.arg_types[arg] = descr

    else:
        raise ValueError(BAD_CONTEXT % tag)
        
def process_var_field(api_doc, docindex, tag, arg, descr):
    _check(api_doc, tag, arg, context=ModuleDoc, expect_arg=True)
    for ident in re.split('[:;, ] *', arg):
        set_var_descr(api_doc, ident, descr)
        
def process_cvar_field(api_doc, docindex, tag, arg, descr):
    # If @cvar is used *within* a variable, then use it as the
    # variable's description, and treat the variable as a class var.
    if (isinstance(api_doc, VariableDoc) and
        isinstance(api_doc.container, ClassDoc)):
        _check(api_doc, tag, arg, expect_arg=False)
        api_doc.is_instvar = False
        api_doc.descr = markup.ConcatenatedDocstring(api_doc.descr, descr)
        api_doc.summary, api_doc.other_docs = descr.summary()

    # Otherwise, @cvar should be used in a class.
    else:
        _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True)
        for ident in re.split('[:;, ] *', arg):
            set_var_descr(api_doc, ident, descr)
            api_doc.variables[ident].is_instvar = False
        
def process_ivar_field(api_doc, docindex, tag, arg, descr):
    # If @ivar is used *within* a variable, then use it as the
    # variable's description, and treat the variable as an instvar.
    if (isinstance(api_doc, VariableDoc) and
        isinstance(api_doc.container, ClassDoc)):
        _check(api_doc, tag, arg, expect_arg=False)
        # require that there be no other descr?
        api_doc.is_instvar = True
        api_doc.descr = markup.ConcatenatedDocstring(api_doc.descr, descr)
        api_doc.summary, api_doc.other_docs = descr.summary()

    # Otherwise, @ivar should be used in a class.
    else:
        _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True)
        for ident in re.split('[:;, ] *', arg):
            set_var_descr(api_doc, ident, descr)
            api_doc.variables[ident].is_instvar = True

# [xx] '@return: foo' used to get used as a descr if no other
# descr was present.  is that still true?
def process_return_field(api_doc, docindex, tag, arg, descr):
    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=False)
    if api_doc.return_descr is not None:
        raise ValueError(REDEFINED % 'return value description')
    api_doc.return_descr = descr

def process_rtype_field(api_doc, docindex, tag, arg, descr):
    _check(api_doc, tag, arg,
           context=(RoutineDoc, PropertyDoc), expect_arg=False)
    if isinstance(api_doc, RoutineDoc):
        if api_doc.return_type is not None:
            raise ValueError(REDEFINED % 'return value type')
        api_doc.return_type = descr

    elif isinstance(api_doc, PropertyDoc):
        _check(api_doc, tag, arg, expect_arg=False)
        if api_doc.type_descr is not None:
            raise ValueError(REDEFINED % tag)
        api_doc.type_descr = descr

def process_arg_field(api_doc, docindex, tag, arg, descr):
    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True)
    idents = re.split('[:;, ] *', arg)
    api_doc.arg_descrs.append( (idents, descr) )
    # Check to make sure that the documented parameter(s) are
    # actually part of the function signature.
    all_args = api_doc.all_args()
    if all_args not in (['...'], UNKNOWN):
        bad_params = ['"%s"' % i for i in idents if i not in all_args]
        if bad_params:
            raise ValueError(BAD_PARAM % (tag, ', '.join(bad_params)))

def process_kwarg_field(api_doc, docindex, tag, arg, descr):
    # [xx] these should -not- be checked if they exist..
    # and listed separately or not??
    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True)
    idents = re.split('[:;, ] *', arg)
    api_doc.arg_descrs.append( (idents, descr) )

register_field_handler(process_group_field, 'group')
register_field_handler(process_deffield_field, 'deffield', 'newfield')
register_field_handler(process_sort_field, 'sort')
register_field_handler(process_summary_field, 'summary')
register_field_handler(process_undocumented_field, 'undocumented')
register_field_handler(process_include_field, 'include')
register_field_handler(process_var_field, 'var', 'variable')
register_field_handler(process_type_field, 'type')
register_field_handler(process_cvar_field, 'cvar', 'cvariable')
register_field_handler(process_ivar_field, 'ivar', 'ivariable')
register_field_handler(process_return_field, 'return', 'returns')
register_field_handler(process_rtype_field, 'rtype', 'returntype')
register_field_handler(process_arg_field, 'arg', 'argument',
                                          'parameter', 'param')
register_field_handler(process_kwarg_field, 'kwarg', 'keyword', 'kwparam')
register_field_handler(process_raise_field, 'raise', 'raises',
                                            'except', 'exception')

# Tags related to function parameters
PARAMETER_TAGS = ('arg', 'argument', 'parameter', 'param',
                  'kwarg', 'keyword', 'kwparam')

# Tags related to variables in a class
VARIABLE_TAGS = ('cvar', 'cvariable', 'ivar', 'ivariable')

# Tags related to exceptions
EXCEPTION_TAGS = ('raise', 'raises', 'except', 'exception')

######################################################################
#{ Helper Functions
######################################################################

def check_type_fields(api_doc, field_warnings):
    """Check to make sure that all type fields correspond to some
    documented parameter; if not, append a warning to field_warnings."""
    if isinstance(api_doc, RoutineDoc):
        for arg in api_doc.arg_types:
            if arg not in api_doc.all_args():
                for args, descr in api_doc.arg_descrs:
                    if arg in args:
                        break
                else:
                    field_warnings.append(BAD_PARAM % ('type', '"%s"' % arg))

def set_var_descr(api_doc, ident, descr):
    if ident not in api_doc.variables:
        api_doc.variables[ident] = VariableDoc(
            container=api_doc, name=ident,
            canonical_name=api_doc.canonical_name+ident)
                                      
    var_doc = api_doc.variables[ident]
    if var_doc.descr not in (None, UNKNOWN):
        raise ValueError(REDEFINED % ('description for '+ident))
    var_doc.descr = descr
    if var_doc.summary in (None, UNKNOWN):
        var_doc.summary, var_doc.other_docs = var_doc.descr.summary()

def set_var_type(api_doc, ident, descr):
    if ident not in api_doc.variables:
        api_doc.variables[ident] = VariableDoc(
            container=api_doc, name=ident,
            canonical_name=api_doc.canonical_name+ident)
        
    var_doc = api_doc.variables[ident]
    if var_doc.type_descr not in (None, UNKNOWN):
        raise ValueError(REDEFINED % ('type for '+ident))
    var_doc.type_descr = descr
        
def _check(api_doc, tag, arg, context=None, expect_arg=None):
    if context is not None:
        if not isinstance(api_doc, context):
            raise ValueError(BAD_CONTEXT % tag)
    if expect_arg is not None:
        if expect_arg == True:
            if arg is None:
                raise ValueError(EXPECTED_ARG % tag)
        elif expect_arg == False:
            if arg is not None:
                raise ValueError(UNEXPECTED_ARG % tag)
        elif expect_arg == 'single':
            if (arg is None or ' ' in arg):
                raise ValueError(EXPECTED_SINGLE_ARG % tag)
        else:
            assert 0, 'bad value for expect_arg'

def get_docformat(api_doc, docindex):
    """
    Return the name of the markup language that should be used to
    parse the API documentation for the given object.
    """
    # Find the module that defines api_doc.
    module = api_doc.defining_module
    # Look up its docformat.
    if module is not UNKNOWN and module.docformat not in (None, UNKNOWN):
        docformat = module.docformat
    else:
        docformat = DEFAULT_DOCFORMAT
    # Convert to lower case & strip region codes.
    try: return docformat.lower().split()[0]
    except: return DEFAULT_DOCFORMAT

def unindent_docstring(docstring):
    # [xx] copied from inspect.getdoc(); we can't use inspect.getdoc()
    # itself, since it expects an object, not a string.
    
    if not docstring: return ''
    lines = docstring.expandtabs().split('\n')

    # Find minimum indentation of any non-blank lines after first line.
    margin = sys.maxint
    for line in lines[1:]:
        content = len(line.lstrip())
        if content:
            indent = len(line) - content
            margin = min(margin, indent)
    # Remove indentation.
    if lines:
        lines[0] = lines[0].lstrip()
    if margin < sys.maxint:
        for i in range(1, len(lines)): lines[i] = lines[i][margin:]
    # Remove any trailing (but not leading!) blank lines.
    while lines and not lines[-1]:
        lines.pop()
    #while lines and not lines[0]:
    #    lines.pop(0)
    return '\n'.join(lines)
                           
_IDENTIFIER_LIST_REGEXP = re.compile(r'^[\w.\*]+([\s,:;]\s*[\w.\*]+)*$')
def _descr_to_identifiers(descr):
    """
    Given a C{ParsedDocstring} that contains a list of identifiers,
    return a list of those identifiers.  This is used by fields such
    as C{@group} and C{@sort}, which expect lists of identifiers as
    their values.  To extract the identifiers, the docstring is first
    converted to plaintext, and then split.  The plaintext content of
    the docstring must be a a list of identifiers, separated by
    spaces, commas, colons, or semicolons.
    
    @rtype: C{list} of C{string}
    @return: A list of the identifier names contained in C{descr}.
    @type descr: L{markup.ParsedDocstring}
    @param descr: A C{ParsedDocstring} containing a list of
        identifiers.
    @raise ValueError: If C{descr} does not contain a valid list of
        identifiers.
    """
    idents = descr.to_plaintext(None).strip()
    idents = re.sub(r'\s+', ' ', idents)
    if not _IDENTIFIER_LIST_REGEXP.match(idents):
        raise ValueError, 'Bad Identifier list: %r' % idents
    rval = re.split('[:;, ] *', idents)
    return rval
    
def _descr_to_docstring_field(arg, descr):
    tags = [s.lower() for s in re.split('[:;, ] *', arg)]
    descr = descr.to_plaintext(None).strip()
    args = re.split('[:;,] *', descr)
    if len(args) == 0 or len(args) > 3:
        raise ValueError, 'Wrong number of arguments'
    singular = args[0]
    if len(args) >= 2: plural = args[1]
    else: plural = None
    short = 0
    if len(args) >= 3:
        if args[2] == 'short': short = 1
        else: raise ValueError('Bad arg 2 (expected "short")')
    return DocstringField(tags, singular, plural, short)

######################################################################
#{ Function Signature Extraction
######################################################################

# [XX] todo: add optional type modifiers?
_SIGNATURE_RE = re.compile(
    # Class name (for builtin methods)
    r'^\s*((?P<self>\w+)\.)?' +
    # The function name (must match exactly) [XX] not anymore!
    r'(?P<func>\w+)' +
    # The parameters
    r'\((?P<params>(\s*\[?\s*\*{0,2}[\w\-\.]+(\s*=.+?)?'+
    r'(\s*\[?\s*,\s*\]?\s*\*{0,2}[\w\-\.]+(\s*=.+?)?)*\]*)?)\s*\)' +
    # The return value (optional)
    r'(\s*(->)\s*(?P<return>\S.*?))?'+
    # The end marker
    r'\s*(\n|\s+(--|<=+>)\s+|$|\.\s+|\.\n)')
"""A regular expression that is used to extract signatures from
docstrings."""
    
def parse_function_signature(func_doc, doc_source, docformat, parse_errors):
    """
    Construct the signature for a builtin function or method from
    its docstring.  If the docstring uses the standard convention
    of including a signature in the first line of the docstring
    (and formats that signature according to standard
    conventions), then it will be used to extract a signature.
    Otherwise, the signature will be set to a single varargs
    variable named C{"..."}.

    @param func_doc: The target object where to store parsed signature. Also
        container of the docstring to parse if doc_source is C{None}
    @type func_doc: L{RoutineDoc}
    @param doc_source: Contains the docstring to parse. If C{None}, parse
        L{func_doc} docstring instead
    @type doc_source: L{APIDoc}
    @rtype: C{None}
    """
    if doc_source is None:
        doc_source = func_doc

    # If there's no docstring, then don't do anything.
    if not doc_source.docstring: return False

    m = _SIGNATURE_RE.match(doc_source.docstring)
    if m is None: return False

    # Do I want to be this strict?
    # Notice that __init__ must match the class name instead, if the signature
    # comes from the class docstring
#     if not (m.group('func') == func_doc.canonical_name[-1] or
#             '_'+m.group('func') == func_doc.canonical_name[-1]):
#         log.warning("Not extracting function signature from %s's "
#                     "docstring, since the name doesn't match." %
#                     func_doc.canonical_name)
#         return False
    
    params = m.group('params')
    rtype = m.group('return')
    selfparam = m.group('self')
    
    # Extract the parameters from the signature.
    func_doc.posargs = []
    func_doc.vararg = None
    func_doc.kwarg = None
    if func_doc.posarg_defaults is UNKNOWN:
        func_doc.posarg_defaults = []
    if params:
        # Figure out which parameters are optional.
        while '[' in params or ']' in params:
            m2 = re.match(r'(.*)\[([^\[\]]+)\](.*)', params)
            if not m2: return False
            (start, mid, end) = m2.groups()
            mid = re.sub(r'((,|^)\s*[\w\-\.]+)', r'\1=...', mid)
            params = start+mid+end

        params = re.sub(r'=...=' , r'=', params)
        for name in params.split(','):
            if '=' in name:
                (name, default_repr) = name.split('=',1)
                default = GenericValueDoc(parse_repr=default_repr)
            else:
                default = None
            name = name.strip()
            if name == '...':
                func_doc.vararg = '...'
            elif name.startswith('**'):
                func_doc.kwarg = name[2:]
            elif name.startswith('*'):
                func_doc.vararg = name[1:]
            else:
                func_doc.posargs.append(name)
                if len(func_doc.posarg_defaults) < len(func_doc.posargs):
                    func_doc.posarg_defaults.append(default)
                elif default is not None:
                    argnum = len(func_doc.posargs)-1
                    func_doc.posarg_defaults[argnum] = default

    # Extract the return type/value from the signature
    if rtype:
        func_doc.return_type = markup.parse(rtype, docformat, parse_errors,
                                            inline=True)

    # Add the self parameter, if it was specified.
    if selfparam:
        func_doc.posargs.insert(0, selfparam)
        func_doc.posarg_defaults.insert(0, None)

    # Remove the signature from the docstring.
    doc_source.docstring = doc_source.docstring[m.end():]
        
    # We found a signature.
    return True