python-kitchen 1.1.1-1 / usr / share / pyshared / kitchen / text / display.py

# -*- coding: utf-8 -*-
#
# Copyright (c) 2010 Red Hat, Inc.
# Copyright (c) 2010 Ville Skyttä
# Copyright (c) 2009 Tim Lauridsen
# Copyright (c) 2007 Markus Kuhn
#
# kitchen is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# kitchen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
# more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with kitchen; if not, see <http://www.gnu.org/licenses/>
#
# Authors:
#   James Antill <james@fedoraproject.org>
#   Markus Kuhn
#   Toshio Kuratomi <toshio@fedoraproject.org>
#   Tim Lauridsen
#   Ville Skyttä
#
# Portions of this are from yum/i18n.py
'''
-----------------------
Format Text for Display
-----------------------

Functions related to displaying unicode text.  Unicode characters don't all
have the same width so we need helper functions for displaying them.

.. versionadded:: 0.2 kitchen.display API 1.0.0
'''
import itertools
import unicodedata

from kitchen import b_
from kitchen.text.converters import to_unicode, to_bytes
from kitchen.text.exceptions import ControlCharError

# This is ported from ustr_utf8_* which I got from:
#     http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
#  I've tried to leave it close to the original C (same names etc.) so that
# it is easy to read/compare both versions... James Antilles

#
# Reimplemented quite a bit of this for speed.  Use the bzr log or annotate
# commands to see what I've changed since importing this file.-Toshio Kuratomi

# ----------------------------- BEG utf8 ------------------to-----------
# This is an implementation of wcwidth() and wcswidth() (defined in
# IEEE Std 1002.1-2001) for Unicode.
#
# http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
# http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
#
# In fixed-width output devices, Latin characters all occupy a single
# "cell" position of equal width, whereas ideographic CJK characters
# occupy two such cells. Interoperability between terminal-line
# applications and (teletype-style) character terminals using the
# UTF-8 encoding requires agreement on which character should advance
# the cursor by how many cell positions. No established formal
# standards exist at present on which Unicode character shall occupy
# how many cell positions on character terminals. These routines are
# a first attempt of defining such behavior based on simple rules
# applied to data provided by the Unicode Consortium.
#
# [...]
#
# Markus Kuhn -- 2007-05-26 (Unicode 5.0)
#
# Permission to use, copy, modify, and distribute this software
# for any purpose and without fee is hereby granted. The author
# disclaims all warranties with regard to this software.
#
# Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c

# Renamed but still pretty much JA's port of MK's code
def _interval_bisearch(value, table):
    '''Binary search in an interval table.

    :arg value: numeric value to search for
    :arg table: Ordered list of intervals.  This is a list of two-tuples.  The
        elements of the two-tuple define an interval's start and end points.
    :returns: If :attr:`value` is found within an interval in the :attr:`table`
        return :data:`True`.  Otherwise, :data:`False`

    This function checks whether a numeric value is present within a table
    of intervals.  It checks using a binary search algorithm, dividing the
    list of values in half and checking against the values until it determines
    whether the value is in the table.
    '''
    minimum = 0
    maximum = len(table) - 1
    if value < table[minimum][0] or value > table[maximum][1]:
        return False

    while maximum >= minimum:
        mid = (minimum + maximum) / 2
        if value > table[mid][1]:
            minimum = mid + 1
        elif value < table[mid][0]:
            maximum = mid - 1
        else:
            return True

    return False

_COMBINING = (
        (0x300, 0x36f), (0x483, 0x489), (0x591, 0x5bd),
        (0x5bf, 0x5bf), (0x5c1, 0x5c2), (0x5c4, 0x5c5),
        (0x5c7, 0x5c7), (0x600, 0x603), (0x610, 0x61a),
        (0x64b, 0x65e), (0x670, 0x670), (0x6d6, 0x6e4),
        (0x6e7, 0x6e8), (0x6ea, 0x6ed), (0x70f, 0x70f),
        (0x711, 0x711), (0x730, 0x74a), (0x7a6, 0x7b0),
        (0x7eb, 0x7f3), (0x816, 0x819), (0x81b, 0x823),
        (0x825, 0x827), (0x829, 0x82d), (0x901, 0x902),
        (0x93c, 0x93c), (0x941, 0x948), (0x94d, 0x94d),
        (0x951, 0x954), (0x962, 0x963), (0x981, 0x981),
        (0x9bc, 0x9bc), (0x9c1, 0x9c4), (0x9cd, 0x9cd),
        (0x9e2, 0x9e3), (0xa01, 0xa02), (0xa3c, 0xa3c),
        (0xa41, 0xa42), (0xa47, 0xa48), (0xa4b, 0xa4d),
        (0xa70, 0xa71), (0xa81, 0xa82), (0xabc, 0xabc),
        (0xac1, 0xac5), (0xac7, 0xac8), (0xacd, 0xacd),
        (0xae2, 0xae3), (0xb01, 0xb01), (0xb3c, 0xb3c),
        (0xb3f, 0xb3f), (0xb41, 0xb43), (0xb4d, 0xb4d),
        (0xb56, 0xb56), (0xb82, 0xb82), (0xbc0, 0xbc0),
        (0xbcd, 0xbcd), (0xc3e, 0xc40), (0xc46, 0xc48),
        (0xc4a, 0xc4d), (0xc55, 0xc56), (0xcbc, 0xcbc),
        (0xcbf, 0xcbf), (0xcc6, 0xcc6), (0xccc, 0xccd),
        (0xce2, 0xce3), (0xd41, 0xd43), (0xd4d, 0xd4d),
        (0xdca, 0xdca), (0xdd2, 0xdd4), (0xdd6, 0xdd6),
        (0xe31, 0xe31), (0xe34, 0xe3a), (0xe47, 0xe4e),
        (0xeb1, 0xeb1), (0xeb4, 0xeb9), (0xebb, 0xebc),
        (0xec8, 0xecd), (0xf18, 0xf19), (0xf35, 0xf35),
        (0xf37, 0xf37), (0xf39, 0xf39), (0xf71, 0xf7e),
        (0xf80, 0xf84), (0xf86, 0xf87), (0xf90, 0xf97),
        (0xf99, 0xfbc), (0xfc6, 0xfc6), (0x102d, 0x1030),
        (0x1032, 0x1032), (0x1036, 0x1037), (0x1039, 0x103a),
        (0x1058, 0x1059), (0x108d, 0x108d), (0x1160, 0x11ff),
        (0x135f, 0x135f), (0x1712, 0x1714), (0x1732, 0x1734),
        (0x1752, 0x1753), (0x1772, 0x1773), (0x17b4, 0x17b5),
        (0x17b7, 0x17bd), (0x17c6, 0x17c6), (0x17c9, 0x17d3),
        (0x17dd, 0x17dd), (0x180b, 0x180d), (0x18a9, 0x18a9),
        (0x1920, 0x1922), (0x1927, 0x1928), (0x1932, 0x1932),
        (0x1939, 0x193b), (0x1a17, 0x1a18), (0x1a60, 0x1a60),
        (0x1a75, 0x1a7c), (0x1a7f, 0x1a7f), (0x1b00, 0x1b03),
        (0x1b34, 0x1b34), (0x1b36, 0x1b3a), (0x1b3c, 0x1b3c),
        (0x1b42, 0x1b42), (0x1b44, 0x1b44), (0x1b6b, 0x1b73),
        (0x1baa, 0x1baa), (0x1c37, 0x1c37), (0x1cd0, 0x1cd2),
        (0x1cd4, 0x1ce0), (0x1ce2, 0x1ce8), (0x1ced, 0x1ced),
        (0x1dc0, 0x1de6), (0x1dfd, 0x1dff), (0x200b, 0x200f),
        (0x202a, 0x202e), (0x2060, 0x2063), (0x206a, 0x206f),
        (0x20d0, 0x20f0), (0x2cef, 0x2cf1), (0x2de0, 0x2dff),
        (0x302a, 0x302f), (0x3099, 0x309a), (0xa66f, 0xa66f),
        (0xa67c, 0xa67d), (0xa6f0, 0xa6f1), (0xa806, 0xa806),
        (0xa80b, 0xa80b), (0xa825, 0xa826), (0xa8c4, 0xa8c4),
        (0xa8e0, 0xa8f1), (0xa92b, 0xa92d), (0xa953, 0xa953),
        (0xa9b3, 0xa9b3), (0xa9c0, 0xa9c0), (0xaab0, 0xaab0),
        (0xaab2, 0xaab4), (0xaab7, 0xaab8), (0xaabe, 0xaabf),
        (0xaac1, 0xaac1), (0xabed, 0xabed), (0xfb1e, 0xfb1e),
        (0xfe00, 0xfe0f), (0xfe20, 0xfe26), (0xfeff, 0xfeff),
        (0xfff9, 0xfffb), (0x101fd, 0x101fd), (0x10a01, 0x10a03),
        (0x10a05, 0x10a06), (0x10a0c, 0x10a0f), (0x10a38, 0x10a3a),
        (0x10a3f, 0x10a3f), (0x110b9, 0x110ba), (0x1d165, 0x1d169),
        (0x1d16d, 0x1d182), (0x1d185, 0x1d18b), (0x1d1aa, 0x1d1ad),
        (0x1d242, 0x1d244), (0xe0001, 0xe0001), (0xe0020, 0xe007f),
        (0xe0100, 0xe01ef), )
'''
Internal table, provided by this module to list :term:`code points` which
combine with other characters and therefore should have no :term:`textual
width`.  This is a sorted :class:`tuple` of non-overlapping intervals.  Each
interval is a :class:`tuple` listing a starting :term:`code point` and ending
:term:`code point`.  Every :term:`code point` between the two end points is
a combining character.

.. seealso::

    :func:`~kitchen.text.display._generate_combining_table`
        for how this table is generated

This table was last regenerated on python-2.7.0 with
:data:`unicodedata.unidata_version` 5.1.0
'''

# New function from Toshio Kuratomi (LGPLv2+)
def _generate_combining_table():
    '''Combine Markus Kuhn's data with :mod:`unicodedata` to make combining
    char list

    :rtype: :class:`tuple` of tuples
    :returns: :class:`tuple` of intervals of :term:`code points` that are
        combining character.  Each interval is a 2-:class:`tuple` of the
        starting :term:`code point` and the ending :term:`code point` for the
        combining characters.

    In normal use, this function serves to tell how we're generating the
    combining char list.  For speed reasons, we use this to generate a static
    list and just use that later.

    Markus Kuhn's list of combining characters is more complete than what's in
    the python :mod:`unicodedata` library but the python :mod:`unicodedata` is
    synced against later versions of the unicode database

    This is used to generate the :data:`~kitchen.text.display._COMBINING`
    table.
    '''
    # Markus Kuhn's sorted list of non-overlapping intervals of non-spacing
    # characters generated ifrom Unicode 5.0 data by:
    # "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
    markus_kuhn_combining_5_0 = (
        ( 0x0300, 0x036F ), ( 0x0483, 0x0486 ), ( 0x0488, 0x0489 ),
        ( 0x0591, 0x05BD ), ( 0x05BF, 0x05BF ), ( 0x05C1, 0x05C2 ),
        ( 0x05C4, 0x05C5 ), ( 0x05C7, 0x05C7 ), ( 0x0600, 0x0603 ),
        ( 0x0610, 0x0615 ), ( 0x064B, 0x065E ), ( 0x0670, 0x0670 ),
        ( 0x06D6, 0x06E4 ), ( 0x06E7, 0x06E8 ), ( 0x06EA, 0x06ED ),
        ( 0x070F, 0x070F ), ( 0x0711, 0x0711 ), ( 0x0730, 0x074A ),
        ( 0x07A6, 0x07B0 ), ( 0x07EB, 0x07F3 ), ( 0x0901, 0x0902 ),
        ( 0x093C, 0x093C ), ( 0x0941, 0x0948 ), ( 0x094D, 0x094D ),
        ( 0x0951, 0x0954 ), ( 0x0962, 0x0963 ), ( 0x0981, 0x0981 ),
        ( 0x09BC, 0x09BC ), ( 0x09C1, 0x09C4 ), ( 0x09CD, 0x09CD ),
        ( 0x09E2, 0x09E3 ), ( 0x0A01, 0x0A02 ), ( 0x0A3C, 0x0A3C ),
        ( 0x0A41, 0x0A42 ), ( 0x0A47, 0x0A48 ), ( 0x0A4B, 0x0A4D ),
        ( 0x0A70, 0x0A71 ), ( 0x0A81, 0x0A82 ), ( 0x0ABC, 0x0ABC ),
        ( 0x0AC1, 0x0AC5 ), ( 0x0AC7, 0x0AC8 ), ( 0x0ACD, 0x0ACD ),
        ( 0x0AE2, 0x0AE3 ), ( 0x0B01, 0x0B01 ), ( 0x0B3C, 0x0B3C ),
        ( 0x0B3F, 0x0B3F ), ( 0x0B41, 0x0B43 ), ( 0x0B4D, 0x0B4D ),
        ( 0x0B56, 0x0B56 ), ( 0x0B82, 0x0B82 ), ( 0x0BC0, 0x0BC0 ),
        ( 0x0BCD, 0x0BCD ), ( 0x0C3E, 0x0C40 ), ( 0x0C46, 0x0C48 ),
        ( 0x0C4A, 0x0C4D ), ( 0x0C55, 0x0C56 ), ( 0x0CBC, 0x0CBC ),
        ( 0x0CBF, 0x0CBF ), ( 0x0CC6, 0x0CC6 ), ( 0x0CCC, 0x0CCD ),
        ( 0x0CE2, 0x0CE3 ), ( 0x0D41, 0x0D43 ), ( 0x0D4D, 0x0D4D ),
        ( 0x0DCA, 0x0DCA ), ( 0x0DD2, 0x0DD4 ), ( 0x0DD6, 0x0DD6 ),
        ( 0x0E31, 0x0E31 ), ( 0x0E34, 0x0E3A ), ( 0x0E47, 0x0E4E ),
        ( 0x0EB1, 0x0EB1 ), ( 0x0EB4, 0x0EB9 ), ( 0x0EBB, 0x0EBC ),
        ( 0x0EC8, 0x0ECD ), ( 0x0F18, 0x0F19 ), ( 0x0F35, 0x0F35 ),
        ( 0x0F37, 0x0F37 ), ( 0x0F39, 0x0F39 ), ( 0x0F71, 0x0F7E ),
        ( 0x0F80, 0x0F84 ), ( 0x0F86, 0x0F87 ), ( 0x0F90, 0x0F97 ),
        ( 0x0F99, 0x0FBC ), ( 0x0FC6, 0x0FC6 ), ( 0x102D, 0x1030 ),
        ( 0x1032, 0x1032 ), ( 0x1036, 0x1037 ), ( 0x1039, 0x1039 ),
        ( 0x1058, 0x1059 ), ( 0x1160, 0x11FF ), ( 0x135F, 0x135F ),
        ( 0x1712, 0x1714 ), ( 0x1732, 0x1734 ), ( 0x1752, 0x1753 ),
        ( 0x1772, 0x1773 ), ( 0x17B4, 0x17B5 ), ( 0x17B7, 0x17BD ),
        ( 0x17C6, 0x17C6 ), ( 0x17C9, 0x17D3 ), ( 0x17DD, 0x17DD ),
        ( 0x180B, 0x180D ), ( 0x18A9, 0x18A9 ), ( 0x1920, 0x1922 ),
        ( 0x1927, 0x1928 ), ( 0x1932, 0x1932 ), ( 0x1939, 0x193B ),
        ( 0x1A17, 0x1A18 ), ( 0x1B00, 0x1B03 ), ( 0x1B34, 0x1B34 ),
        ( 0x1B36, 0x1B3A ), ( 0x1B3C, 0x1B3C ), ( 0x1B42, 0x1B42 ),
        ( 0x1B6B, 0x1B73 ), ( 0x1DC0, 0x1DCA ), ( 0x1DFE, 0x1DFF ),
        ( 0x200B, 0x200F ), ( 0x202A, 0x202E ), ( 0x2060, 0x2063 ),
        ( 0x206A, 0x206F ), ( 0x20D0, 0x20EF ), ( 0x302A, 0x302F ),
        ( 0x3099, 0x309A ), ( 0xA806, 0xA806 ), ( 0xA80B, 0xA80B ),
        ( 0xA825, 0xA826 ), ( 0xFB1E, 0xFB1E ), ( 0xFE00, 0xFE0F ),
        ( 0xFE20, 0xFE23 ), ( 0xFEFF, 0xFEFF ), ( 0xFFF9, 0xFFFB ),
        ( 0x10A01, 0x10A03 ), ( 0x10A05, 0x10A06 ), ( 0x10A0C, 0x10A0F ),
        ( 0x10A38, 0x10A3A ), ( 0x10A3F, 0x10A3F ), ( 0x1D167, 0x1D169 ),
        ( 0x1D173, 0x1D182 ), ( 0x1D185, 0x1D18B ), ( 0x1D1AA, 0x1D1AD ),
        ( 0x1D242, 0x1D244 ), ( 0xE0001, 0xE0001 ), ( 0xE0020, 0xE007F ),
        ( 0xE0100, 0xE01EF ))
    combining = []
    in_interval = False
    interval = []
    for codepoint in xrange (0, 0xFFFFF + 1):
        if _interval_bisearch(codepoint, markus_kuhn_combining_5_0) or \
                unicodedata.combining(unichr(codepoint)):
            if not in_interval:
                # Found first part of an interval
                interval = [codepoint]
                in_interval = True
        else:
            if in_interval:
                in_interval = False
                interval.append(codepoint - 1)
                combining.append(interval)

    if in_interval:
        # If we're at the end and the interval is open, close it.
        # :W0631: We looped through a static range so we know codepoint is
        #   defined here
        #pylint:disable-msg=W0631
        interval.append(codepoint)
        combining.append(interval)

    return tuple(itertools.imap(tuple, combining))

# New function from Toshio Kuratomi (LGPLv2+)
def _print_combining_table():
    '''Print out a new :data:`_COMBINING` table

    This will print a new :data:`_COMBINING` table in the format used in
    :file:`kitchen/text/display.py`.  It's useful for updating the
    :data:`_COMBINING` table with updated data from a new python as the format
    won't change from what's already in the file.
    '''
    table = _generate_combining_table()
    entries = 0
    print '_COMBINING = ('
    for pair in table:
        if entries >= 3:
            entries = 0
            print
        if entries == 0:
            print '       ',
        entries += 1
        entry = '(0x%x, 0x%x),' % pair
        print entry,
    print ')'

# Handling of control chars rewritten.  Rest is JA's port of MK's C code.
# -Toshio Kuratomi
def _ucp_width(ucs, control_chars='guess'):
    '''Get the :term:`textual width` of a ucs character

    :arg ucs: integer representing a single unicode :term:`code point`
    :kwarg control_chars: specify how to deal with :term:`control characters`.
        Possible values are:

        :guess: (default) will take a guess for :term:`control character`
            widths.  Most codes will return zero width.  ``backspace``,
            ``delete``, and ``clear delete`` return -1.  ``escape`` currently
            returns -1 as well but this is not guaranteed as it's not always
            correct
        :strict: will raise :exc:`~kitchen.text.exceptions.ControlCharError`
            if a :term:`control character` is encountered

    :raises ControlCharError: if the :term:`code point` is a unicode
        :term:`control character` and :attr:`control_chars` is set to 'strict'
    :returns: :term:`textual width` of the character.

    .. note::

        It's important to remember this is :term:`textual width` and not the
        number of characters or bytes.
    '''
    # test for 8-bit control characters
    if ucs < 32 or (ucs < 0xa0 and ucs >= 0x7f):
        # Control character detected
        if control_chars == 'strict':
            raise ControlCharError(b_('_ucp_width does not understand how to'
                ' assign a width value to control characters.'))
        if ucs in (0x08, 0x07F, 0x94):
            # Backspace, delete, and clear delete remove a single character
            return -1
        if ucs == 0x1b:
            # Excape is tricky.  It removes some number of characters that
            # come after it but the amount is dependent on what is
            # interpreting the code.
            # So this is going to often be wrong but other values will be
            # wrong as well.
            return -1
        # All other control characters get 0 width
        return 0

    if _interval_bisearch(ucs, _COMBINING):
        # Combining characters return 0 width as they will be combined with
        # the width from other characters
        return 0

    # if we arrive here, ucs is not a combining or C0/C1 control character

    return (1 +
      (ucs >= 0x1100 and
       (ucs <= 0x115f or                     # Hangul Jamo init. consonants
        ucs == 0x2329 or ucs == 0x232a or
        (ucs >= 0x2e80 and ucs <= 0xa4cf and
         ucs != 0x303f) or                   # CJK ... Yi
        (ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables
        (ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs
        (ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms
        (ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms
        (ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms
        (ucs >= 0xffe0 and ucs <= 0xffe6) or
        (ucs >= 0x20000 and ucs <= 0x2fffd) or
        (ucs >= 0x30000 and ucs <= 0x3fffd))))

# Wholly rewritten by me (LGPLv2+) -Toshio Kuratomi
def textual_width(msg, control_chars='guess', encoding='utf-8',
        errors='replace'):
    '''Get the :term:`textual width` of a string

    :arg msg: :class:`unicode` string or byte :class:`str` to get the width of
    :kwarg control_chars: specify how to deal with :term:`control characters`.
        Possible values are:

        :guess: (default) will take a guess for :term:`control character`
            widths.  Most codes will return zero width.  ``backspace``,
            ``delete``, and ``clear delete`` return -1.  ``escape`` currently
            returns -1 as well but this is not guaranteed as it's not always
            correct
        :strict: will raise :exc:`kitchen.text.exceptions.ControlCharError`
            if a :term:`control character` is encountered

    :kwarg encoding: If we are given a byte :class:`str` this is used to
        decode it into :class:`unicode` string.  Any characters that are not
        decodable in this encoding will get a value dependent on the
        :attr:`errors` parameter.
    :kwarg errors: How to treat errors encoding the byte :class:`str` to
        :class:`unicode` string.  Legal values are the same as for
        :func:`kitchen.text.converters.to_unicode`.  The default value of
        ``replace`` will cause undecodable byte sequences to have a width of
        one. ``ignore`` will have a width of zero.
    :raises ControlCharError: if :attr:`msg` contains a :term:`control
        character` and :attr:`control_chars` is ``strict``.
    :returns: :term:`Textual width` of the :attr:`msg`.  This is the amount of
        space that the string will consume on a monospace display.  It's
        measured in the number of cell positions or columns it will take up on
        a monospace display.  This is **not** the number of glyphs that are in
        the string.

    .. note::

        This function can be wrong sometimes because Unicode does not specify
        a strict width value for all of the :term:`code points`.  In
        particular, we've found that some Tamil characters take up to four
        character cells but we return a lesser amount.
    '''
    # On python 2.6.4, x86_64, I've benchmarked a few alternate
    # implementations::
    #
    #   timeit.repeat('display.textual_width(data)',
    #       'from __main__ import display, data', number=100)
    # I varied data by size and content (1MB of ascii, a few words, 43K utf8,
    # unicode type
    #
    # :this implementation: fastest across the board
    #
    # :list comprehension: 6-16% slower
    #   return sum([_ucp_width(ord(c), control_chars=control_chars)
    #       for c in msg])
    #
    # :generator expression: 9-18% slower
    #   return sum((_ucp_width(ord(c), control_chars=control_chars) for c in
    #           msg))
    #
    # :lambda: 10-19% slower
    #   return sum(itertools.imap(lambda x: _ucp_width(ord(x), control_chars),
    #           msg))
    #
    # :partial application: 13-22% slower
    #   func = functools.partial(_ucp_width, control_chars=control_chars)
    #   return sum(itertools.imap(func, itertools.imap(ord, msg)))
    #
    # :the original code: 4-38% slower
    #   The 4% was for the short, ascii only string.  All the other pieces of
    #   data yielded over 30% slower times.

    # Non decodable data is just assigned a single cell width
    msg = to_unicode(msg, encoding=encoding, errors=errors)
    # Add the width of each char
    return sum(
            # calculate width of each char
            itertools.starmap(_ucp_width,
                # Setup the arguments to _ucp_width
                itertools.izip(
                    # int value of each char
                    itertools.imap(ord, msg),
                    # control_chars arg in a form that izip will deal with
                    itertools.repeat(control_chars))))

# Wholly rewritten by me -Toshio Kuratomi
def textual_width_chop(msg, chop, encoding='utf-8', errors='replace'):
    '''Given a string, return it chopped to a given :term:`textual width`

    :arg msg: :class:`unicode` string or byte :class:`str` to chop
    :arg chop: Chop :attr:`msg` if it exceeds this :term:`textual width`
    :kwarg encoding: If we are given a byte :class:`str`, this is used to
        decode it into a :class:`unicode` string.  Any characters that are not
        decodable in this encoding will be assigned a width of one.
    :kwarg errors: How to treat errors encoding the byte :class:`str` to
        :class:`unicode`.  Legal values are the same as for
        :func:`kitchen.text.converters.to_unicode`
    :rtype: :class:`unicode` string
    :returns: :class:`unicode` string of the :attr:`msg` chopped at the given
        :term:`textual width`

    This is what you want to use instead of ``%.*s``, as it does the "right"
    thing with regard to :term:`UTF-8` sequences, :term:`control characters`,
    and characters that take more than one cell position. Eg::

        >>> # Wrong: only displays 8 characters because it is operating on bytes
        >>> print "%.*s" % (10, 'café ñunru!')
        café ñun
        >>> # Properly operates on graphemes
        >>> '%s' % (textual_width_chop('café ñunru!', 10))
        café ñunru
        >>> # takes too many columns because the kanji need two cell positions
        >>> print '1234567890\\n%.*s' % (10, u'一二三四五六七八九十')
        1234567890
        一二三四五六七八九十
        >>> # Properly chops at 10 columns
        >>> print '1234567890\\n%s' % (textual_width_chop(u'一二三四五六七八九十', 10))
        1234567890
        一二三四五

    '''

    msg = to_unicode(msg, encoding=encoding, errors=errors)

    width = textual_width(msg)
    if width <= chop:
        return msg
    maximum = len(msg)
    if maximum > chop * 2:
        # A character can take at most 2 cell positions so this is the actual
        # maximum
        maximum = chop * 2
    minimum = 0
    eos = maximum
    if eos > chop:
        eos = chop
    width = textual_width(msg[:eos])

    while True:
        # if current width is high,
        if width > chop:
            # calculate new midpoint
            mid = minimum + (eos - minimum) / 2
            if mid == eos:
                break
            if (eos - chop) < (eos - mid):
                while width > chop:
                    width = width - _ucp_width(ord(msg[eos-1]))
                    eos -= 1
                return msg[:eos]
            # subtract distance between eos and mid from width
            width = width - textual_width(msg[mid:eos])
            maximum = eos
            eos = mid
        # if current width is low,
        elif width < chop:
            # Note: at present, the if (eos - chop) < (eos - mid):
            # short-circuit above means that we never use this branch.

            # calculate new midpoint
            mid = eos + (maximum - eos) / 2
            if mid == eos:
                break
            if (chop - eos) < (mid - eos):
                while width < chop:
                    new_width = _ucp_width(ord(msg[eos]))
                    width = width + new_width
                    eos += 1
                return msg[:eos]

            # add distance between eos and new mid to width
            width = width + textual_width(msg[eos:mid])
            minimum = eos
            eos = mid
            if eos > maximum:
                eos = maximum
                break
        # if current is just right
        else:
            return msg[:eos]
    return msg[:eos]

# I made some adjustments for using unicode but largely unchanged from JA's
# port of MK's code -Toshio
def textual_width_fill(msg, fill, chop=None, left=True, prefix='', suffix=''):
    '''Expand a :class:`unicode` string to a specified :term:`textual width`
    or chop to same

    :arg msg: :class:`unicode` string to format
    :arg fill: pad string until the :term:`textual width` of the string is
        this length
    :kwarg chop: before doing anything else, chop the string to this length.
        Default: Don't chop the string at all
    :kwarg left: If :data:`True` (default) left justify the string and put the
        padding on the right.  If :data:`False`, pad on the left side.
    :kwarg prefix: Attach this string before the field we're filling
    :kwarg suffix: Append this string to the end of the field we're filling
    :rtype: :class:`unicode` string
    :returns: :attr:`msg` formatted to fill the specified width.  If no
        :attr:`chop` is specified, the string could exceed the fill length
        when completed.  If :attr:`prefix` or :attr:`suffix` are printable
        characters, the string could be longer than the fill width.

    .. note::

        :attr:`prefix` and :attr:`suffix` should be used for "invisible"
        characters like highlighting, color changing escape codes, etc.  The
        fill characters are appended outside of any :attr:`prefix` or
        :attr:`suffix` elements.  This allows you to only highlight
        :attr:`msg` inside of the field you're filling.

    .. warning::

        :attr:`msg`, :attr:`prefix`, and :attr:`suffix` should all be
        representable as unicode characters.  In particular, any escape
        sequences in :attr:`prefix` and :attr:`suffix` need to be convertible
        to :class:`unicode`.  If you need to use byte sequences here rather
        than unicode characters, use
        :func:`~kitchen.text.display.byte_string_textual_width_fill` instead.

    This function expands a string to fill a field of a particular
    :term:`textual width`.  Use it instead of ``%*.*s``, as it does the
    "right" thing with regard to :term:`UTF-8` sequences, :term:`control
    characters`, and characters that take more than one cell position in
    a display.  Example usage::

        >>> msg = u'一二三四五六七八九十'
        >>> # Wrong: This uses 10 characters instead of 10 cells:
        >>> u":%-*.*s:" % (10, 10, msg[:9])
        :一二三四五六七八九 :
        >>> # This uses 10 cells like we really want:
        >>> u":%s:" % (textual_width_fill(msg[:9], 10, 10))
        :一二三四五:

        >>> # Wrong: Right aligned in the field, but too many cells
        >>> u"%20.10s" % (msg)
                  一二三四五六七八九十
        >>> # Correct: Right aligned with proper number of cells
        >>> u"%s" % (textual_width_fill(msg, 20, 10, left=False))
                  一二三四五

        >>> # Wrong: Adding some escape characters to highlight the line but too many cells
        >>> u"%s%20.10s%s" % (prefix, msg, suffix)
        u'\x1b[7m          一二三四五六七八九十\x1b[0m'
        >>> # Correct highlight of the line
        >>> u"%s%s%s" % (prefix, display.textual_width_fill(msg, 20, 10, left=False), suffix)
        u'\x1b[7m          一二三四五\x1b[0m'

        >>> # Correct way to not highlight the fill
        >>> u"%s" % (display.textual_width_fill(msg, 20, 10, left=False, prefix=prefix, suffix=suffix))
        u'          \x1b[7m一二三四五\x1b[0m'
    '''
    msg = to_unicode(msg)
    if chop is not None:
        msg = textual_width_chop(msg, chop)
    width = textual_width(msg)
    if width >= fill:
        if prefix or suffix:
            msg = u''.join([prefix, msg, suffix])
    else:
        extra = u' ' * (fill - width)
        if left:
            msg = u''.join([prefix, msg, suffix, extra])
        else:
            msg = u''.join([extra, prefix, msg, suffix])
    return msg

def _textual_width_le(width, *args):
    '''Optimize the common case when deciding which :term:`textual width` is
    larger

    :arg width: :term:`textual width` to compare against.
    :arg \*args: :class:`unicode` strings to check the total :term:`textual
        width` of
    :returns: :data:`True` if the total length of :attr:`args` are less than
        or equal to :attr:`width`.  Otherwise :data:`False`.

    We often want to know "does X fit in Y".  It takes a while to use
    :func:`textual_width` to calculate this.  However, we know that the number
    of canonically composed :class:`unicode` characters is always going to
    have 1 or 2 for the :term:`textual width` per character.  With this we can
    take the following shortcuts:

    1) If the number of canonically composed characters is more than width,
       the true :term:`textual width` cannot be less than width.
    2) If the number of canonically composed characters * 2 is less than the
       width then the :term:`textual width` must be ok.

    :term:`textual width` of a canonically composed :class:`unicode` string
    will always be greater than or equal to the the number of :class:`unicode`
    characters.  So we can first check if the number of composed
    :class:`unicode` characters is less than the asked for width.  If it is we
    can return :data:`True` immediately.  If not, then we must do a full
    :term:`textual width` lookup.
    '''
    string = ''.join(args)
    string = unicodedata.normalize('NFC', string)
    if len(string) > width:
        return False
    elif len(string) * 2 <= width:
        return True
    elif len(to_bytes(string)) <= width:
        # Check against bytes.
        # utf8 has the property of having the same amount or more bytes per
        # character than textual width.
        return True
    else:
        true_width = textual_width(string)
    return true_width <= width

def wrap(text, width=70, initial_indent=u'', subsequent_indent=u'',
        encoding='utf-8', errors='replace'):
    '''Works like we want :func:`textwrap.wrap` to work,

    :arg text: :class:`unicode` string or byte :class:`str` to wrap
    :kwarg width: :term:`textual width` at which to wrap.  Default: 70
    :kwarg initial_indent: string to use to indent the first line.  Default:
        do not indent.
    :kwarg subsequent_indent: string to use to wrap subsequent lines.
        Default: do not indent
    :kwarg encoding: Encoding to use if :attr:`text` is a byte :class:`str`
    :kwarg errors: error handler to use if :attr:`text` is a byte :class:`str`
        and contains some undecodable characters.
    :rtype: :class:`list` of :class:`unicode` strings
    :returns: list of lines that have been text wrapped and indented.

    :func:`textwrap.wrap` from the |stdlib|_ has two drawbacks that this
    attempts to fix:

    1. It does not handle :term:`textual width`.  It only operates on bytes or
       characters which are both inadequate (due to multi-byte and double
       width characters).
    2. It malforms lists and blocks.
    '''
    # Tested with:
    # yum info robodoc gpicview php-pear-Net-Socket wmctrl ustr moreutils
    #          mediawiki-HNP ocspd insight yum mousepad
    # ...at 120, 80 and 40 chars.
    # Also, notable among lots of others, searching for "\n  ":
    #   exim-clamav, jpackage-utils, tcldom, synaptics, "quake3",
    #   perl-Class-Container, ez-ipupdate, perl-Net-XMPP, "kipi-plugins",
    #   perl-Apache-DBI, netcdf, python-configobj, "translate-toolkit", alpine,
    #   "udunits", "conntrack-tools"
    #
    # Note that, we "fail" on:
    #   alsa-plugins-jack, setools*, dblatex, uisp, "perl-Getopt-GUI-Long",
    #   suitesparse, "synce-serial", writer2latex, xenwatch, ltsp-utils

    def _indent_at_beg(line):
        '''Return the indent to use for this and (possibly) subsequent lines

        :arg line: :class:`unicode` line of text to process
        :rtype: tuple
        :returns: tuple of count of whitespace before getting to the start of
            this line followed by a count to the following indent if this
            block of text is an entry in a list.
        '''
        # Find the first non-whitespace character
        try:
            char = line.strip()[0]
        except IndexError:
            # All whitespace
            return 0, 0
        else:
            count = line.find(char)

        # if we have a bullet character, check for list
        if char not in u'-*.o\u2022\u2023\u2218':
            # No bullet; not a list
            return count, 0

        # List: Keep searching until we hit the innermost list
        nxt = _indent_at_beg(line[count+1:])
        nxt = nxt[1] or nxt[0]
        if nxt:
            return count, count + 1 + nxt
        return count, 0

    initial_indent = to_unicode(initial_indent, encoding=encoding,
            errors=errors)
    subsequent_indent = to_unicode(subsequent_indent, encoding=encoding,
            errors=errors)
    subsequent_indent_width = textual_width(subsequent_indent)

    text = to_unicode(text, encoding=encoding, errors=errors).rstrip(u'\n')
    lines = text.expandtabs().split(u'\n')

    ret = []
    indent = initial_indent
    wrap_last = False
    cur_sab = 0
    cur_spc_indent = 0
    for line in lines:
        line = line.rstrip(u' ')
        (last_sab, last_spc_indent) = (cur_sab, cur_spc_indent)
        (cur_sab, cur_spc_indent) = _indent_at_beg(line)
        force_nl = False # We want to stop wrapping under "certain" conditions:
        if wrap_last and cur_spc_indent:      # if line starts a list or
            force_nl = True
        if wrap_last and cur_sab == len(line):# is empty line
            force_nl = True
        if wrap_last and not last_spc_indent: # if we don't continue a list
            if cur_sab >= 4 and cur_sab != last_sab: # and is "block indented"
                force_nl = True
        if force_nl:
            ret.append(indent.rstrip(u' '))
            indent = subsequent_indent
            wrap_last = False
        if cur_sab == len(line): # empty line, remove spaces to make it easier.
            line = u''
        if wrap_last:
            line = line.lstrip(u' ')
            cur_spc_indent = last_spc_indent

        if _textual_width_le(width, indent, line):
            wrap_last = False
            ret.append(indent + line)
            indent = subsequent_indent
            continue

        wrap_last = True
        words = line.split(u' ')
        line = indent
        spcs = cur_spc_indent
        if not spcs and cur_sab >= 4:
            spcs = cur_sab
        for word in words:
            if (not _textual_width_le(width, line, word) and
                textual_width(line) > subsequent_indent_width):
                ret.append(line.rstrip(u' '))
                line = subsequent_indent + u' ' * spcs
            line += word
            line += u' '
        indent = line.rstrip(u' ') + u' '
    if wrap_last:
        ret.append(indent.rstrip(u' '))

    return ret

def fill(text, *args, **kwargs):
    '''Works like we want :func:`textwrap.fill` to work

    :arg text: :class:`unicode` string or byte :class:`str` to process
    :returns: :class:`unicode` string with each line separated by a newline

    .. seealso::

        :func:`kitchen.text.display.wrap`
            for other parameters that you can give this command.

    This function is a light wrapper around :func:`kitchen.text.display.wrap`.
    Where that function returns a :class:`list` of lines, this function
    returns one string with each line separated by a newline.
    '''
    return u'\n'.join(wrap(text, *args, **kwargs))

#
# Byte strings
#

def byte_string_textual_width_fill(msg, fill, chop=None, left=True, prefix='',
        suffix='', encoding='utf-8', errors='replace'):
    '''Expand a byte :class:`str` to a specified :term:`textual width` or chop
    to same

    :arg msg: byte :class:`str` encoded in :term:`UTF-8` that we want formatted
    :arg fill: pad :attr:`msg` until the :term:`textual width` is this long
    :kwarg chop: before doing anything else, chop the string to this length.
        Default: Don't chop the string at all
    :kwarg left: If :data:`True` (default) left justify the string and put the
        padding on the right.  If :data:`False`, pad on the left side.
    :kwarg prefix: Attach this byte :class:`str` before the field we're
        filling
    :kwarg suffix: Append this byte :class:`str` to the end of the field we're
        filling
    :rtype: byte :class:`str`
    :returns: :attr:`msg` formatted to fill the specified :term:`textual
        width`.  If no :attr:`chop` is specified, the string could exceed the
        fill length when completed.  If :attr:`prefix` or :attr:`suffix` are
        printable characters, the string could be longer than fill width.

    .. note::

        :attr:`prefix` and :attr:`suffix` should be used for "invisible"
        characters like highlighting, color changing escape codes, etc.  The
        fill characters are appended outside of any :attr:`prefix` or
        :attr:`suffix` elements.  This allows you to only highlight
        :attr:`msg` inside of the field you're filling.

    .. seealso::

        :func:`~kitchen.text.display.textual_width_fill`
            For example usage.  This function has only two differences.

            1. it takes byte :class:`str` for :attr:`prefix` and
               :attr:`suffix` so you can pass in arbitrary sequences of
               bytes, not just unicode characters.
            2. it returns a byte :class:`str` instead of a :class:`unicode`
               string.
    '''
    prefix = to_bytes(prefix, encoding=encoding, errors=errors)
    suffix = to_bytes(suffix, encoding=encoding, errors=errors)

    if chop is not None:
        msg = textual_width_chop(msg, chop, encoding=encoding, errors=errors)
    width = textual_width(msg)
    msg = to_bytes(msg)

    if width >= fill:
        if prefix or suffix:
            msg = ''.join([prefix, msg, suffix])
    else:
        extra = ' ' * (fill - width)
        if left:
            msg = ''.join([prefix, msg, suffix, extra])
        else:
            msg = ''.join([extra, prefix, msg, suffix])

    return msg

__all__ = ('byte_string_textual_width_fill', 'fill', 'textual_width',
        'textual_width_chop', 'textual_width_fill', 'wrap')