This file is indexed.

/usr/lib/python3/dist-packages/check_manifest.py is in check-manifest 0.36-2.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
#!/usr/bin/env python
"""Check the MANIFEST.in file in a Python source package for completeness.

This script works by building a source distribution archive (by running
setup.py sdist), then checking the file list in the archive against the
file list in version control (Subversion, Git, Mercurial, Bazaar are
supported).

Since the first check can fail to catch missing MANIFEST.in entries when
you've got the right setuptools version control system support plugins
installed, the script copies all the versioned files into a temporary
directory and building the source distribution again.  This also avoids issues
with stale egg-info/SOURCES.txt files that may cause files not mentioned in
MANIFEST.in to be included nevertheless.
"""
from __future__ import print_function

import argparse
import codecs
import fnmatch
import locale
import os
import posixpath
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import tempfile
import unicodedata
import zipfile
from distutils.filelist import glob_to_re
from contextlib import contextmanager, closing
from distutils.text_file import TextFile
from xml.etree import cElementTree as ET

try:
    import ConfigParser
except ImportError:
    # Python 3.x
    import configparser as ConfigParser


__version__ = '0.36'
__author__ = 'Marius Gedminas <marius@gedmin.as>'
__licence__ = 'MIT'
__url__ = 'https://github.com/mgedmin/check-manifest'


class Failure(Exception):
    """An expected failure (as opposed to a bug in this script)."""


#
# User interface
#

VERBOSE = False

_to_be_continued = False
def _check_tbc():
    global _to_be_continued
    if _to_be_continued:
        print()
        _to_be_continued = False


def info(message):
    _check_tbc()
    print(message)


def info_begin(message):
    if not VERBOSE:
        return
    _check_tbc()
    sys.stdout.write(message)
    sys.stdout.flush()
    global _to_be_continued
    _to_be_continued = True


def info_continue(message):
    if not VERBOSE:
        return
    sys.stdout.write(message)
    sys.stdout.flush()
    global _to_be_continued
    _to_be_continued = True


def info_end(message):
    if not VERBOSE:
        return
    print(message)
    global _to_be_continued
    _to_be_continued = False


def error(message):
    _check_tbc()
    print(message, file=sys.stderr)


def warning(message):
    _check_tbc()
    print(message, file=sys.stderr)


def format_list(list_of_strings):
    return "\n".join("  " + s for s in list_of_strings)


def format_missing(missing_from_a, missing_from_b, name_a, name_b):
    res = []
    if missing_from_a:
        res.append("missing from %s:\n%s"
                   % (name_a, format_list(sorted(missing_from_a))))
    if missing_from_b:
        res.append("missing from %s:\n%s"
                   % (name_b, format_list(sorted(missing_from_b))))
    return '\n'.join(res)


#
# Filesystem/OS utilities
#

class CommandFailed(Failure):
    def __init__(self, command, status, output):
        Failure.__init__(self, "%s failed (status %s):\n%s" % (
                               command, status, output))


def run(command, encoding=None, decode=True, cwd=None):
    """Run a command [cmd, arg1, arg2, ...].

    Returns the output (stdout + stderr).

    Raises CommandFailed in cases of error.
    """
    if not encoding:
        encoding = locale.getpreferredencoding()
    try:
        with open(os.devnull, 'rb') as devnull:
            pipe = subprocess.Popen(command, stdin=devnull,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT, cwd=cwd)
    except OSError as e:
        raise Failure("could not run %s: %s" % (command, e))
    output = pipe.communicate()[0]
    if decode:
        output = output.decode(encoding)
    status = pipe.wait()
    if status != 0:
        raise CommandFailed(command, status, output)
    return output


@contextmanager
def cd(directory):
    """Change the current working directory, temporarily.

    Use as a context manager: with cd(d): ...
    """
    old_dir = os.getcwd()
    try:
        os.chdir(directory)
        yield
    finally:
        os.chdir(old_dir)


@contextmanager
def mkdtemp(hint=''):
    """Create a temporary directory, then clean it up.

    Use as a context manager: with mkdtemp('-purpose'): ...
    """
    dirname = tempfile.mkdtemp(prefix='check-manifest-', suffix=hint)
    try:
        yield dirname
    finally:
        rmtree(dirname)


def chmod_plus(path, add_bits=stat.S_IWUSR):
    """Change a file's mode by adding a few bits.

    Like chmod +<bits> <path> in a Unix shell.
    """
    try:
        os.chmod(path, stat.S_IMODE(os.stat(path).st_mode) | add_bits)
    except OSError:  # pragma: nocover
        pass  # well, we tried


def rmtree(path):
    """A version of rmtree that can deal with read-only files and directories.

    Needed because the stock shutil.rmtree() fails with an access error
    when there are read-only files in the directory on Windows, or when the
    directory itself is read-only on Unix.
    """
    def onerror(func, path, exc_info):
        # Did you know what on Python 3.3 on Windows os.remove() and
        # os.unlink() are distinct functions?
        if func is os.remove or func is os.unlink or func is os.rmdir:
            if sys.platform != 'win32':
                chmod_plus(os.path.dirname(path), stat.S_IWUSR | stat.S_IXUSR)
            chmod_plus(path)
            func(path)
        else:
            raise
    shutil.rmtree(path, onerror=onerror)


def copy_files(filelist, destdir):
    """Copy a list of files to destdir, preserving directory structure.

    File names should be relative to the current working directory.
    """
    for filename in filelist:
        destfile = os.path.join(destdir, filename)
        # filename should not be absolute, but let's double-check
        assert destfile.startswith(destdir + os.path.sep)
        destfiledir = os.path.dirname(destfile)
        if not os.path.isdir(destfiledir):
            os.makedirs(destfiledir)
        if os.path.isdir(filename):
            os.mkdir(destfile)
        else:
            shutil.copy2(filename, destfile)


def get_one_file_in(dirname):
    """Return the pathname of the one file in a directory.

    Raises if the directory has no files or more than one file.
    """
    files = os.listdir(dirname)
    if len(files) > 1:
        raise Failure('More than one file exists in %s:\n%s' %
                      (dirname, '\n'.join(sorted(files))))
    elif not files:
        raise Failure('No files found in %s' % dirname)
    return os.path.join(dirname, files[0])


def unicodify(filename):
    """Make sure filename is Unicode.

    Because the tarfile module on Python 2 doesn't return Unicode.
    """
    if isinstance(filename, bytes):
        return filename.decode(locale.getpreferredencoding())
    else:
        return filename


def get_archive_file_list(archive_filename):
    """Return the list of files in an archive.

    Supports .tar.gz and .zip.
    """
    if archive_filename.endswith('.zip'):
        with closing(zipfile.ZipFile(archive_filename)) as zf:
            return add_directories(zf.namelist())
    elif archive_filename.endswith(('.tar.gz', '.tar.bz2', '.tar')):
        with closing(tarfile.open(archive_filename)) as tf:
            return add_directories(list(map(unicodify, tf.getnames())))
    else:
        ext = os.path.splitext(archive_filename)[-1]
        raise Failure('Unrecognized archive type: %s' % ext)


def strip_toplevel_name(filelist):
    """Strip toplevel name from a file list.

        >>> strip_toplevel_name(['a', 'a/b', 'a/c', 'a/c/d'])
        ['b', 'c', 'c/d']

        >>> strip_toplevel_name(['a/b', 'a/c', 'a/c/d'])
        ['b', 'c', 'c/d']

    """
    if not filelist:
        return filelist
    prefix = filelist[0]
    if '/' in prefix:
        prefix = prefix.partition('/')[0] + '/'
        names = filelist
    else:
        prefix = prefix + '/'
        names = filelist[1:]
    for name in names:
        if not name.startswith(prefix):
            raise Failure("File doesn't have the common prefix (%s): %s"
                          % (name, prefix))
    return [name[len(prefix):] for name in names]


def add_prefix_to_each(prefix, filelist):
    """Add a prefix to each name in a file list.

        >>> filelist = add_prefix_to_each('foo/bar', ['a', 'b', 'c/d'])
        >>> [f.replace(os.path.sep, '/') for f in filelist]
        ['foo/bar/a', 'foo/bar/b', 'foo/bar/c/d']

    """
    return [posixpath.join(prefix, name) for name in filelist]


class VCS(object):

    @classmethod
    def detect(cls, location):
        return os.path.isdir(os.path.join(location, cls.metadata_name))


class Git(VCS):
    metadata_name = '.git'

    # Git for Windows uses UTF-8 instead of the locale encoding.
    # Git on POSIX systems uses the locale encoding.
    _encoding = 'UTF-8' if sys.platform == 'win32' else None

    @classmethod
    def detect(cls, location):
        # .git can be a file for submodules
        return os.path.exists(os.path.join(location, cls.metadata_name))

    @classmethod
    def get_versioned_files(cls):
        """List all files versioned by git in the current directory."""
        files = cls._git_ls_files()
        submodules = cls._list_submodules()
        for subdir in submodules:
            subdir = os.path.relpath(subdir).replace(os.path.sep, '/')
            files += add_prefix_to_each(subdir, cls._git_ls_files(subdir))
        return add_directories(files)

    @classmethod
    def _git_ls_files(cls, cwd=None):
        output = run(['git', 'ls-files', '-z'], encoding=cls._encoding, cwd=cwd)
        return output.split('\0')[:-1]

    @classmethod
    def _list_submodules(cls):
        # This is incredibly expensive on my Jenkins instance (50 seconds for
        # each invocation, even when there are no submodules whatsoever).
        # Curiously, I cannot reproduce that in Appveyor, or even on the same
        # Jenkins machine but when I run the tests manually.  Still, 2-hour
        # Jenkins runs are bad, so let's avoid running 'git submodule' when
        # there's no .gitmodules file.
        if not os.path.exists('.gitmodules'):
            return []
        return run(['git', 'submodule', '--quiet', 'foreach', '--recursive',
                    'printf "%s/%s\\n" $toplevel $path'], encoding=cls._encoding).splitlines()


class Mercurial(VCS):
    metadata_name = '.hg'

    @staticmethod
    def get_versioned_files():
        """List all files under Mercurial control in the current directory."""
        output = run(['hg', 'status', '-ncamd', '.'])
        return add_directories(output.splitlines())


class Bazaar(VCS):
    metadata_name = '.bzr'

    @classmethod
    def _get_terminal_encoding(self):
        # Python 3.6 lets us name the OEM codepage directly, which is lucky
        # because it also breaks our old method of OEM codepage detection
        # (PEP-528 changed sys.stdout.encoding to UTF-8).
        try:
            codecs.lookup('oem')
        except LookupError:
            pass
        else:  # pragma: nocover
            return 'oem'
        # Based on bzrlib.osutils.get_terminal_encoding()
        encoding = getattr(sys.stdout, 'encoding', None)
        if not encoding:
            encoding = getattr(sys.stdin, 'encoding', None)
        if encoding == 'cp0':  # "no codepage"
            encoding = None
        # NB: bzrlib falls back on bzrlib.osutils.get_user_encoding(),
        # which is like locale.getpreferredencoding() on steroids, and
        # also includes a fallback from 'ascii' to 'utf-8' when
        # sys.platform is 'darwin'.  This is probably something we might
        # want to do in run(), but I'll wait for somebody to complain
        # first, since I don't have a Mac OS X machine and cannot test.
        return encoding

    @classmethod
    def get_versioned_files(cls):
        """List all files versioned in Bazaar in the current directory."""
        encoding = cls._get_terminal_encoding()
        output = run(['bzr', 'ls', '-VR'], encoding=encoding)
        return output.splitlines()


class Subversion(VCS):
    metadata_name = '.svn'

    @classmethod
    def get_versioned_files(cls):
        """List all files under SVN control in the current directory."""
        output = run(['svn', 'st', '-vq', '--xml'], decode=False)
        tree = ET.XML(output)
        return sorted(entry.get('path') for entry in tree.findall('.//entry')
                      if cls.is_interesting(entry))

    @staticmethod
    def is_interesting(entry):
        """Is this entry interesting?

        ``entry`` is an XML node representing one entry of the svn status
        XML output.  It looks like this::

            <entry path="unchanged.txt">
              <wc-status item="normal" revision="1" props="none">
                <commit revision="1">
                  <author>mg</author>
                  <date>2015-02-06T07:52:38.163516Z</date>
                </commit>
              </wc-status>
            </entry>

            <entry path="added-but-not-committed.txt">
              <wc-status item="added" revision="-1" props="none"></wc-status>
            </entry>

            <entry path="ext">
              <wc-status item="external" props="none"></wc-status>
            </entry>

            <entry path="unknown.txt">
              <wc-status props="none" item="unversioned"></wc-status>
            </entry>

        """
        if entry.get('path') == '.':
            return False
        status = entry.find('wc-status')
        if status is None:
            warning('svn status --xml parse error: <entry path="%s"> without'
                    ' <wc-status>' % entry.get('path'))
            return False
        # For SVN externals we get two entries: one mentioning the
        # existence of the external, and one about the status of the external.
        if status.get('item') in ('unversioned', 'external'):
            return False
        return True


def detect_vcs():
    """Detect the version control system used for the current directory."""
    location = os.path.abspath('.')
    while True:
        for vcs in Git, Mercurial, Bazaar, Subversion:
            if vcs.detect(location):
                return vcs
        parent = os.path.dirname(location)
        if parent == location:
            raise Failure("Couldn't find version control data"
                          " (git/hg/bzr/svn supported)")
        location = parent


def get_vcs_files():
    """List all files under version control in the current directory."""
    vcs = detect_vcs()
    return normalize_names(vcs.get_versioned_files())


def normalize_names(names):
    """Normalize file names."""
    return list(map(normalize_name, names))


def normalize_name(name):
    """Some VCS print directory names with trailing slashes.  Strip them.

    Easiest is to normalize the path.

    And encodings may trip us up too, especially when comparing lists
    of files.  Plus maybe lowercase versus uppercase.
    """
    name = os.path.normpath(name)
    name = unicodify(name)
    if sys.platform == 'darwin':
        # Mac OSX may have problems comparing non-ascii filenames, so
        # we convert them.
        name = unicodedata.normalize('NFC', name)
    return name


def add_directories(names):
    """Git/Mercurial/zip files omit directories, let's add them back."""
    res = list(names)
    seen = set(names)
    for name in names:
        while True:
            name = os.path.dirname(name)
            if not name or name in seen:
                break
            res.append(name)
            seen.add(name)
    return sorted(res)


#
# Packaging logic
#

# it's fine if any of these are missing in the VCS or in the sdist
IGNORE = [
    'PKG-INFO',     # always generated
    '*.egg-info',   # always generated
    '*.egg-info/*', # always generated
    'setup.cfg',    # always generated, sometimes also kept in source control
    # it's not a problem if the sdist is lacking these files:
    '.hgtags', '.hgsigs', '.hgignore', '.gitignore', '.bzrignore',
    '.gitattributes',
    '.travis.yml',
    'Jenkinsfile',
    # it's convenient to ship compiled .mo files in sdists, but they shouldn't
    # be checked in
    '*.mo',
]

IGNORE_REGEXPS = [
    # Regular expressions for filename to ignore.  This is useful for
    # filename patterns where the '*' part must not search in
    # directories.
]

WARN_ABOUT_FILES_IN_VCS = [
    # generated files should not be committed into the VCS
    'PKG-INFO',
    '*.egg-info',
    '*.mo',
    '*.py[co]',
    '*.so',
    '*.pyd',
    '*~',
    '.*.sw[po]',
    '.#*',
]

IGNORE_BAD_IDEAS = []

_sep = r'\\' if os.path.sep == '\\' else os.path.sep

SUGGESTIONS = [(re.compile(pattern.replace('/', _sep)), suggestion) for pattern, suggestion in [
    # regexp -> suggestion
    ('^([^/]+[.](cfg|ini))$',       r'include \1'),
    ('^([.]travis[.]yml)$',         r'include \1'),
    ('^([.]coveragerc)$',           r'include \1'),
    ('^([A-Z]+)$',                  r'include \1'),
    ('^(Makefile)$',                r'include \1'),
    ('^[^/]+[.](txt|rst|py)$',      r'include *.\1'),
    ('^([a-zA-Z_][a-zA-Z_0-9]*)/'
     '.*[.](py|zcml|pt|mako|xml|html|txt|rst|css|png|jpg|dot|po|pot|mo|ui|desktop|bat)$',
                                    r'recursive-include \1 *.\2'),
    ('^([a-zA-Z_][a-zA-Z_0-9]*)(?:/.*)?/(Makefile)$',
                                    r'recursive-include \1 \2'),
    # catch-all rules that actually cover some of the above; somewhat
    # experimental: I fear false positives
    ('^([a-zA-Z_0-9]+)$',           r'include \1'),
    ('^[^/]+[.]([a-zA-Z_0-9]+)$',   r'include *.\1'),
    ('^([a-zA-Z_][a-zA-Z_0-9]*)/.*[.]([a-zA-Z_0-9]+)$',
                                    r'recursive-include \1 *.\2'),
]]

CFG_SECTION_CHECK_MANIFEST = 'check-manifest'
CFG_IGNORE_DEFAULT_RULES = (CFG_SECTION_CHECK_MANIFEST, 'ignore-default-rules')
CFG_IGNORE = (CFG_SECTION_CHECK_MANIFEST, 'ignore')
CFG_IGNORE_BAD_IDEAS = (CFG_SECTION_CHECK_MANIFEST, 'ignore-bad-ideas')


def read_config():
    """Read configuration from setup.cfg."""
    # XXX modifies global state, which is kind of evil
    config = ConfigParser.ConfigParser()
    config.read(['setup.cfg'])
    if not config.has_section(CFG_SECTION_CHECK_MANIFEST):
        return
    if (config.has_option(*CFG_IGNORE_DEFAULT_RULES)
            and config.getboolean(*CFG_IGNORE_DEFAULT_RULES)):
        del IGNORE[:]
    if config.has_option(*CFG_IGNORE):
        patterns = [p.strip() for p in config.get(*CFG_IGNORE).splitlines()]
        IGNORE.extend(p for p in patterns if p)
    if config.has_option(*CFG_IGNORE_BAD_IDEAS):
        lines = config.get(*CFG_IGNORE_BAD_IDEAS).splitlines()
        patterns = [p.strip() for p in lines]
        IGNORE_BAD_IDEAS.extend(p for p in patterns if p)


def read_manifest():
    """Read existing configuration from MANIFEST.in.

    We use that to ignore anything the MANIFEST.in ignores.
    """
    # XXX modifies global state, which is kind of evil
    if not os.path.isfile('MANIFEST.in'):
        return
    ignore, ignore_regexps = _get_ignore_from_manifest('MANIFEST.in')
    IGNORE.extend(ignore)
    IGNORE_REGEXPS.extend(ignore_regexps)


def _glob_to_regexp(pat):
    """Compile a glob pattern into a regexp.

    We need to do this because fnmatch allows * to match /, which we
    don't want.  E.g. a MANIFEST.in exclude of 'dirname/*css' should
    match 'dirname/foo.css' but not 'dirname/subdir/bar.css'.
    """
    return glob_to_re(pat)


def _get_ignore_from_manifest(filename):
    """Gather the various ignore patterns from a MANIFEST.in.

    Returns a list of standard ignore patterns and a list of regular
    expressions to ignore.
    """

    class MyTextFile(TextFile):
        def error(self, msg, line=None):  # pragma: nocover
            # (this is never called by TextFile in current versions of CPython)
            raise Failure(self.gen_error(msg, line))

        def warn(self, msg, line=None):
            warning(self.gen_error(msg, line))

    template = MyTextFile(filename,
                          strip_comments=True,
                          skip_blanks=True,
                          join_lines=True,
                          lstrip_ws=True,
                          rstrip_ws=True,
                          collapse_join=True)
    try:
        lines = template.readlines()
    finally:
        template.close()
    return _get_ignore_from_manifest_lines(lines)


def _get_ignore_from_manifest_lines(lines):
    """Gather the various ignore patterns from a MANIFEST.in.

    'lines' should be a list of strings with comments removed
    and continuation lines joined.

    Returns a list of standard ignore patterns and a list of regular
    expressions to ignore.
    """
    ignore = []
    ignore_regexps = []
    for line in lines:
        try:
            cmd, rest = line.split(None, 1)
        except ValueError:
            # no whitespace, so not interesting
            continue
        for part in rest.split():
            # distutils enforces these warnings on Windows only
            if part.startswith('/'):
                warning("ERROR: Leading slashes are not allowed in MANIFEST.in on Windows: %s" % part)
            if part.endswith('/'):
                warning("ERROR: Trailing slashes are not allowed in MANIFEST.in on Windows: %s" % part)
        if cmd == 'exclude':
            # An exclude of 'dirname/*css' can match 'dirname/foo.css'
            # but not 'dirname/subdir/bar.css'.  We need a regular
            # expression for that, since fnmatch doesn't pay attention to
            # directory separators.
            for pat in rest.split():
                if '*' in pat or '?' in pat or '[!' in pat:
                    ignore_regexps.append(_glob_to_regexp(pat))
                else:
                    # No need for special handling.
                    ignore.append(pat)
        elif cmd == 'global-exclude':
            ignore.extend(rest.split())
        elif cmd == 'recursive-exclude':
            try:
                dirname, patterns = rest.split(None, 1)
            except ValueError:
                # Wrong MANIFEST.in line.
                warning("You have a wrong line in MANIFEST.in: %r\n"
                        "'recursive-exclude' expects <dir> <pattern1> "
                        "<pattern2> ..." % line)
                continue
            # Strip path separator for clarity.
            dirname = dirname.rstrip(os.path.sep)
            for pattern in patterns.split():
                if pattern.startswith('*'):
                    ignore.append(dirname + os.path.sep + pattern)
                else:
                    # 'recursive-exclude plone metadata.xml' should
                    # exclude plone/metadata.xml and
                    # plone/*/metadata.xml, where * can be any number
                    # of sub directories.  We could use a regexp, but
                    # two ignores seems easier.
                    ignore.append(dirname + os.path.sep + pattern)
                    ignore.append(dirname + os.path.sep + '*' + os.path.sep +
                                  pattern)
        elif cmd == 'prune':
            # rest is considered to be a directory name.  It should
            # not contain a path separator, as it actually has no
            # effect in that case, but that could differ per python
            # version.  We strip it here to avoid double separators.
            # XXX: mg: I'm not 100% sure the above is correct, AFAICS
            # all pythons from 2.6 complain if the path has a leading or
            # trailing slash -- on Windows, that is.
            rest = rest.rstrip('/\\')
            ignore.append(rest)
            ignore.append(rest + os.path.sep + '*')
    return ignore, ignore_regexps


def file_matches(filename, patterns):
    """Does this filename match any of the patterns?"""
    return any(fnmatch.fnmatch(filename, pat) or
               fnmatch.fnmatch(os.path.basename(filename), pat)
               for pat in patterns)


def file_matches_regexps(filename, patterns):
    """Does this filename match any of the regular expressions?"""
    return any(re.match(pat, filename) for pat in patterns)


def strip_sdist_extras(filelist):
    """Strip generated files that are only present in source distributions.

    We also strip files that are ignored for other reasons, like
    command line arguments, setup.cfg rules or MANIFEST.in rules.
    """
    return [name for name in filelist
            if not file_matches(name, IGNORE)
            and not file_matches_regexps(name, IGNORE_REGEXPS)]


def find_bad_ideas(filelist):
    """Find files matching WARN_ABOUT_FILES_IN_VCS patterns."""
    return [name for name in filelist
            if file_matches(name, WARN_ABOUT_FILES_IN_VCS)]


def find_suggestions(filelist):
    """Suggest MANIFEST.in patterns for missing files."""
    suggestions = set()
    unknowns = []
    for filename in filelist:
        if os.path.isdir(filename):
            # it's impossible to add empty directories via MANIFEST.in anyway,
            # and non-empty directories will be added automatically when we
            # specify patterns for files inside them
            continue
        for pattern, suggestion in SUGGESTIONS:
            m = pattern.match(filename)
            if m is not None:
                suggestions.add(pattern.sub(suggestion, filename))
                break
        else:
            unknowns.append(filename)
    return sorted(suggestions), unknowns


def is_package(source_tree='.'):
    """Is the directory the root of a Python package?

    Note: the term "package" here refers to a collection of files
    with a setup.py, not to a directory with an __init__.py.
    """
    return os.path.exists(os.path.join(source_tree, 'setup.py'))


def check_manifest(source_tree='.', create=False, update=False,
                   python=sys.executable):
    """Compare a generated source distribution with list of files in a VCS.

    Returns True if the manifest is fine.
    """
    all_ok = True
    if os.path.sep in python:
        python = os.path.abspath(python)
    with cd(source_tree):
        if not is_package():
            raise Failure('This is not a Python project (no setup.py).')
        read_config()
        read_manifest()
        info_begin("listing source files under version control")
        all_source_files = sorted(get_vcs_files())
        source_files = strip_sdist_extras(all_source_files)
        info_continue(": %d files and directories" % len(source_files))
        if not all_source_files:
            raise Failure('There are no files added to version control!')
        info_begin("building an sdist")
        with mkdtemp('-sdist') as tempdir:
            run([python, 'setup.py', 'sdist', '-d', tempdir])
            sdist_filename = get_one_file_in(tempdir)
            info_continue(": %s" % os.path.basename(sdist_filename))
            sdist_files = sorted(normalize_names(strip_sdist_extras(
                strip_toplevel_name(get_archive_file_list(sdist_filename)))))
            info_continue(": %d files and directories" % len(sdist_files))
        existing_source_files = list(filter(os.path.exists, all_source_files))
        missing_source_files = sorted(set(all_source_files) - set(existing_source_files))
        if missing_source_files:
            warning("some files listed as being under source control are missing:\n%s"
                    % format_list(missing_source_files))
        info_begin("copying source files to a temporary directory")
        with mkdtemp('-sources') as tempsourcedir:
            copy_files(existing_source_files, tempsourcedir)
            if os.path.exists('MANIFEST.in') and 'MANIFEST.in' not in source_files:
                # See https://github.com/mgedmin/check-manifest/issues/7
                # if do this, we will emit a warning about MANIFEST.in not
                # being in source control, if we don't do this, the user
                # gets confused about their new manifest rules being
                # ignored.
                copy_files(['MANIFEST.in'], tempsourcedir)
            if 'setup.py' not in source_files:
                # See https://github.com/mgedmin/check-manifest/issues/46
                # if do this, we will emit a warning about setup.py not
                # being in source control, if we don't do this, the user
                # gets a scary error
                copy_files(['setup.py'], tempsourcedir)
            info_begin("building a clean sdist")
            with cd(tempsourcedir):
                with mkdtemp('-sdist') as tempdir:
                    run([python, 'setup.py', 'sdist', '-d', tempdir])
                    sdist_filename = get_one_file_in(tempdir)
                    info_continue(": %s" % os.path.basename(sdist_filename))
                    clean_sdist_files = sorted(normalize_names(strip_sdist_extras(
                        strip_toplevel_name(get_archive_file_list(sdist_filename)))))
                    info_continue(": %d files and directories" % len(clean_sdist_files))
        missing_from_manifest = set(source_files) - set(clean_sdist_files)
        missing_from_VCS = set(sdist_files + clean_sdist_files) - set(source_files)
        if not missing_from_manifest and not missing_from_VCS:
            info("lists of files in version control and sdist match")
        else:
            error("lists of files in version control and sdist do not match!\n%s"
                  % format_missing(missing_from_VCS, missing_from_manifest,
                                   "VCS", "sdist"))
            suggestions, unknowns = find_suggestions(missing_from_manifest)
            user_asked_for_help = update or (create and not
                                                os.path.exists('MANIFEST.in'))
            if 'MANIFEST.in' not in existing_source_files:
                if suggestions and not user_asked_for_help:
                    info("no MANIFEST.in found; you can run 'check-manifest -c' to create one")
                else:
                    info("no MANIFEST.in found")
            if suggestions:
                info("suggested MANIFEST.in rules:\n%s"
                     % format_list(suggestions))
                if user_asked_for_help:
                    existed = os.path.exists('MANIFEST.in')
                    with open('MANIFEST.in', 'a') as f:
                        if not existed:
                            info("creating MANIFEST.in")
                        else:
                            info("updating MANIFEST.in")
                            f.write('\n# added by check_manifest.py\n')
                        f.write('\n'.join(suggestions) + '\n')
                    if unknowns:
                        info("don't know how to come up with rules matching\n%s"
                             % format_list(unknowns))
            elif user_asked_for_help:
                info("don't know how to come up with rules"
                     " matching any of the files, sorry!")
            all_ok = False
        bad_ideas = find_bad_ideas(all_source_files)
        filtered_bad_ideas = [bad_idea for bad_idea in bad_ideas
                              if not file_matches(bad_idea, IGNORE_BAD_IDEAS)]
        if filtered_bad_ideas:
            warning("you have %s in source control!\nthat's a bad idea:"
                    " auto-generated files should not be versioned"
                    % filtered_bad_ideas[0])
            if len(filtered_bad_ideas) > 1:
                warning("this also applies to the following:\n%s"
                        % format_list(filtered_bad_ideas[1:]))
            all_ok = False
    return all_ok


#
# Main script
#

def main():
    parser = argparse.ArgumentParser(
        description="Check a Python MANIFEST.in file for completeness",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('source_tree', default='.', nargs='?',
        help='location for the source tree')
    parser.add_argument('--version', action='version',
                        version='%(prog)s version ' + __version__)
    parser.add_argument('-v', '--verbose', action='store_true',
        help='more verbose output')
    parser.add_argument('-c', '--create', action='store_true',
        help='create a MANIFEST.in if missing')
    parser.add_argument('-u', '--update', action='store_true',
        help='append suggestions to MANIFEST.in (implies --create)')
    parser.add_argument('-p', '--python', default=sys.executable,
        help='use this Python interpreter for running setup.py sdist')
    parser.add_argument('--ignore', metavar='patterns', default=None,
                        help='ignore files/directories matching these'
                             ' comma-separated patterns')
    parser.add_argument('--ignore-bad-ideas', metavar='patterns',
                        default=[], help='ignore bad idea files/directories '
                        'matching these comma-separated patterns')
    args = parser.parse_args()

    if args.ignore:
        IGNORE.extend(args.ignore.split(','))

    if args.ignore_bad_ideas:
        IGNORE_BAD_IDEAS.extend(args.ignore_bad_ideas.split(','))

    if args.verbose:
        global VERBOSE
        VERBOSE = True

    try:
        if not check_manifest(args.source_tree, create=args.create,
                              update=args.update, python=args.python):
            sys.exit(1)
    except Failure as e:
        error(str(e))
        sys.exit(2)


#
# zest.releaser integration
#

def zest_releaser_check(data):
    """Check the completeness of MANIFEST.in before the release.

    This is an entry point for zest.releaser.  See the documentation at
    https://zestreleaser.readthedocs.io/en/latest/entrypoints.html
    """
    from zest.releaser.utils import ask
    source_tree = data['workingdir']
    if not is_package(source_tree):
        # You can use zest.releaser on things that are not Python packages.
        # It's pointless to run check-manifest in those circumstances.
        # See https://github.com/mgedmin/check-manifest/issues/9 for details.
        return
    if not ask("Do you want to run check-manifest?"):
        return
    try:
        if not check_manifest(source_tree):
            if not ask("MANIFEST.in has problems. "
                       " Do you want to continue despite that?", default=False):
                sys.exit(1)
    except Failure as e:
        error(str(e))
        if not ask("Something bad happened. "
                   " Do you want to continue despite that?", default=False):
            sys.exit(2)


if __name__ == '__main__':
    main()