Package pyxb :: Package utils :: Module unicode |
[hide private]
[frames] | no frames]

# Source Code for Module pyxb.utils.unicode

      1  # -*- coding: utf-8 -*-
      2  # Copyright 2009-2013, Peter A. Bigot
      3  #
      4  # Licensed under the Apache License, Version 2.0 (the "License"); you may
      5  # not use this file except in compliance with the License. You may obtain a
      6  # copy of the License at:
      7  #
      8  #            http://www.apache.org/licenses/LICENSE-2.0
      9  #
     10  # Unless required by applicable law or agreed to in writing, software
     11  # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
     12  # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
     13  # License for the specific language governing permissions and limitations
     14  # under the License.
     15
     16  """This module contains support for Unicode characters as required to
     17  support the regular expression syntax defined in U{annex F
     18  <http://www/Documentation/W3C/www.w3.org/TR/xmlschema-2/index.html#regexs>}
     19  of the XML Schema definition.
     20
     21  In particular, we need to be able to identify character properties and
     22  block escapes, as defined in F.1.1, by name.
     23
     24   - Block data: U{http://www.unicode.org/Public/3.1-Update/Blocks-4.txt}
     25   - Property list data: U{http://www.unicode.org/Public/3.1-Update/PropList-3.1.0.txt}
     26   - Full dataset: U{http://www.unicode.org/Public/3.1-Update/UnicodeData-3.1.0.txt}
     27
     28  The Unicode database active at the time XML Schema 1.0 was defined is
     29  archived at
     30  U{http://www.unicode.org/Public/3.1-Update/UnicodeCharacterDatabase-3.1.0.html},
     31  and refers to U{Unicode Standard Annex #27: Unicode 3.1
     32  <http://www.unicode.org/unicode/reports/tr27/>}.
     33  """
     34
     35  import re
     36  import logging
     37  import pyxb.utils.utility
     38
     39  _log = logging.getLogger(__name__)
     40
     41  SupportsWideUnicode = False
     42  try:
     43      re.compile(u'[\U0001d7ce-\U0001d7ff]')
     44      SupportsWideUnicode = True
     45  except:
     46      pass
     47
     48  import bisect
     49
     50 -class CodePointSetError (LookupError):
     51      """Raised when some abuse of a L{CodePointSet} is detected."""
     52      pass
     53
     54  @pyxb.utils.utility.BackfillComparisons
     55 -class CodePointSet (object):
     56      """Represent a set of Unicode code points.
     57
     58      Each code point is an integral value between 0 and 0x10FFFF.  This
     59      class is used to represent a set of code points in a manner
     60      suitable for use as regular expression character sets."""
     61
     62      MaxCodePoint = 0x10FFFF
     63      """The maximum value for a code point in the Unicode code point
     64      space.  This is normally 0xFFFF, because wide unicode characters
     65      are generally not enabled in Python builds.  If, however, they are
     66      enabled, this will be the full value of 0x10FFFF."""
     67
     68      MaxShortCodePoint = 0xFFFF
     69      if not SupportsWideUnicode:
     70          MaxCodePoint = MaxShortCodePoint
     71
     72      # The internal representation of the codepoints is as a sorted
     73      # list where values at an even index denote the first codepoint in
     74      # a range that is in the set, and the immediately following value
     75      # indicates the next following codepoint that is not in the set.
     76      # A missing value at the end is interpreted as MaxCodePoint.  For
     77      # example, the sequence [ 12, 15, 200 ] denotes the set containing
     78      # codepoints 12, 13, 14, and everything above 199.
     79      __codepoints = None
     80
     81 -    def _codepoints (self):
     82          """For testing purrposes only, access to the codepoints
     83          internal representation."""
     84          return self.__codepoints
     85
     86 -    def __hash__ (self):
     87          return hash(self.__codepoints)
     88
     89 -    def __eq__ (self, other):
     90          """Equality is delegated to the codepoints list."""
     91          return self.__codepoints == other.__codepoints
     92
     93 -    def __lt__ (self, other):
     94          return self.__codepoints < other.__codepoints
     95
     96 -    def __init__ (self, *args):
     97          self.__codepoints = []
     98          if 1 == len(args):
     99              if isinstance(args[0], CodePointSet):
    100                  self.__codepoints.extend(args[0].__codepoints)
    101                  return
    102              if isinstance(args[0], list):
    103                  args = args[0]
    104          for a in args:
    105              self.add(a)
    106
    107 -    def __mutate (self, value, do_add):
    108          # Identify the start (inclusive) and end (exclusive) code
    109          # points of the value's range.
    110          if isinstance(value, tuple):
    111              (s, e) = value
    112              e += 1
    113          elif isinstance(value, basestring):
    114              if 1 < len(value):
    115                  raise TypeError()
    116              s = ord(value)
    117              e = s+1
    118          else:
    119              s = int(value)
    120              e = s+1
    121          if s >= e:
    122              raise ValueError('codepoint range value order')
    123
    124          # Validate the range for the code points supported by this
    125          # Python interpreter.  Recall that e is exclusive.
    126          if s > self.MaxCodePoint:
    127              return self
    128          if e > self.MaxCodePoint:
    129              e = self.MaxCodePoint+1
    130
    131          # Index of first code point equal to or greater than s
    132          li = bisect.bisect_left(self.__codepoints, s)
    133          # Index of last code point less than or equal to e
    134          ri = bisect.bisect_right(self.__codepoints, e)
    135          # There are four cases; if we're subtracting, they reflect.
    136          case = ((li & 1) << 1) | (ri & 1)
    137          if not do_add:
    138              case = 3 - case
    139          if 0x03 == case:
    140              # Add: Incoming value begins and ends within existing ranges
    141              del self.__codepoints[li:ri]
    142          elif 0x02 == case:
    143              # Add: Incoming value extends into an excluded range
    144              del self.__codepoints[li+1:ri]
    145              self.__codepoints[li] = e
    146          elif 0x01 == case:
    147              # Add: Incoming value begins in an excluded range
    148              del self.__codepoints[li+1:ri]
    149              self.__codepoints[li] = s
    150          else:
    151              # Add: Incoming value begins and ends within excluded ranges
    152              self.__codepoints[li:ri] = [s, e]
    153          return self
    154
    155 -    def add (self, value):
    156          """Add the given value to the code point set.
    157
    158          @param value: An integral value denoting a code point, or a
    159          tuple C{(s,e)} denoting the start and end (inclusive) code
    160          points in a range.
    161          @return: C{self}"""
    162          return self.__mutate(value, True)
    163
    164 -    def extend (self, values):
    165          """Add multiple values to a code point set.
    166
    167          @param values: Either a L{CodePointSet} instance, or an iterable
    168          whose members are valid parameters to L{add}.
    169
    170          @return: C{self}"""
    171          if isinstance(values, CodePointSet):
    172              self.extend(values.asTuples())
    173          else:
    174              for v in values:
    175                  self.__mutate(v, True)
    176          return self
    177
    178 -    def subtract (self, value):
    179          """Remove the given value from the code point set.
    180
    181          @param value: An integral value denoting a code point, or a tuple
    182          C{(s,e)} denoting the start and end (inclusive) code points in a
    183          range, or a L{CodePointSet}.
    184
    185          @return: C{self}"""
    186          if isinstance(value, CodePointSet):
    187              for v in value.asTuples():
    188                  self.subtract(v)
    189              return self
    190          return self.__mutate(value, False)
    191
    192      # Escape sequences for characters that must not appear unescaped in
    193      # Python regular expression patterns.  Maps each bad character to a safe
    194      # escape sequence.
    195      __XMLtoPythonREMap = {
    196          u'\x00': u'\\x00',  # From docs for Python's "re" module: Regular
    197                              # expression pattern strings may not contain null
    198                              # bytes
    199          u'^': u'\\^',  # Indicates negation if it happens to occur at the
    200                         # start of a character group
    201          u'\\': u'\\\\',  # Escape character
    202          u'[': u'\\[',  # Actually doesn't need to be escaped inside a Python
    203                         # character group, but escaping it is less confusing.
    204          u']': u'\\]',  # End of character group
    205          u'-': u'\\-',  # Indicates a range of characters
    206      }
    207
    208      # Return the given code point as a unicode character suitable for
    209      # use in a regular expression
    210 -    def __unichr (self, code_point):
    211          rv = unichr(code_point)
    212          rv = self.__XMLtoPythonREMap.get(rv, rv)
    213          return rv
    214
    215 -    def asPattern (self, with_brackets=True):
    216          """Return the code point set as Unicode regular expression
    217          character group consisting of a sequence of characters or
    218          character ranges.
    219
    220          This returns a regular expression fragment using Python's
    221          regular expression syntax.  Note that different regular expression
    222          syntaxes are not compatible, often in subtle ways.
    223
    224          @param with_brackets: If C{True} (default), square brackets
    225          are added to enclose the returned character group."""
    226          rva = []
    227          if with_brackets:
    228              rva.append(u'[')
    229          for (s, e) in self.asTuples():
    230              if s == e:
    231                  rva.append(self.__unichr(s))
    232              else:
    233                  rva.extend([self.__unichr(s), '-', self.__unichr(e)])
    234          if with_brackets:
    235              rva.append(u']')
    236          return u''.join(rva)
    237
    238 -    def asTuples (self):
    239          """Return the codepoints as tuples denoting the ranges that are in
    240          the set.
    241
    242          Each tuple C{(s, e)} indicates that the code points from C{s}
    243          (inclusive) to C{e}) (inclusive) are in the set."""
    244
    245          rv = []
    246          start = None
    247          for ri in xrange(len(self.__codepoints)):
    248              if start is not None:
    249                  rv.append( (start, self.__codepoints[ri]-1) )
    250                  start = None
    251              else:
    252                  start = self.__codepoints[ri]
    253          if (start is not None) and (start <= self.MaxCodePoint):
    254              rv.append( (start, self.MaxCodePoint) )
    255          return rv
    256
    257 -    def negate (self):
    258          """Return an instance that represents the inverse of this set."""
    259          rv = type(self)()
    260          if (0 < len(self.__codepoints)) and (0 == self.__codepoints[0]):
    261              rv.__codepoints.extend(self.__codepoints[1:])
    262          else:
    263              rv.__codepoints.append(0)
    264              rv.__codepoints.extend(self.__codepoints)
    265          return rv
    266
    267 -    def asSingleCharacter (self):
    268          """If this set represents a single character, return it as its
    269          unicode string value.  Otherwise return C{None}."""
    270          if (2 != len(self.__codepoints)) or (1 < (self.__codepoints[1] - self.__codepoints[0])):
    271              return None
    272          return unichr(self.__codepoints[0])
    273
    274  from pyxb.utils.unicode_data import PropertyMap
    275  from pyxb.utils.unicode_data import BlockMap
    276
    277 -class XML1p0e2 (object):
    278      """Regular expression support for XML Schema Data Types.
    279
    280      This class holds character classes and regular expressions used to
    281      constrain the lexical space of XML Schema datatypes derived from
    282      U{string<http://www.w3.org/TR/xmlschema-2/#string>}.  They are
    283      from U{XML 1.0 (Second
    284      Edition)<http://www.w3.org/TR/2000/WD-xml-2e-20000814>} and
    285      U{Namespaces in XML
    286      <http://www.w3.org/TR/1999/REC-xml-names-19990114/>}.
    287
    288      Unlike the regular expressions used for pattern constraints in XML
    289      Schema, which are derived from the Unicode 3.1 specification,
    290      these are derived from the Unicode 2.0 specification.
    291
    292      The XML Schema definition refers explicitly to the second edition
    293      of XML, so we have to use these code point sets and patterns.  Be
    294      aware that U{subsequent updates to the XML specification
    295      <http://www.w3.org/XML/xml-V10-4e-errata#E09>} have changed the
    296      corresponding patterns for other uses of XML.  One significant
    297      change is that the original specification, used here, does not
    298      allow wide unicode characters."""
    299
    300      Char = CodePointSet(
    301          0x0009,
    302          0x000A,
    303          0x000D,
    304          ( 0x0020, 0xD7FF ),
    305          ( 0xE000, 0xFFFD )
    306          )
    307      if SupportsWideUnicode:
    308          Char.add( ( 1+CodePointSet.MaxShortCodePoint, CodePointSet.MaxCodePoint ) )
    309
    310      BaseChar = CodePointSet(
    311          ( 0x0041, 0x005A ),
    312          ( 0x0061, 0x007A ),
    313          ( 0x00C0, 0x00D6 ),
    314          ( 0x00D8, 0x00F6 ),
    315          ( 0x00F8, 0x00FF ),
    316          ( 0x0100, 0x0131 ),
    317          ( 0x0134, 0x013E ),
    318          ( 0x0141, 0x0148 ),
    319          ( 0x014A, 0x017E ),
    320          ( 0x0180, 0x01C3 ),
    321          ( 0x01CD, 0x01F0 ),
    322          ( 0x01F4, 0x01F5 ),
    323          ( 0x01FA, 0x0217 ),
    324          ( 0x0250, 0x02A8 ),
    325          ( 0x02BB, 0x02C1 ),
    326          0x0386,
    327          ( 0x0388, 0x038A ),
    328          0x038C,
    329          ( 0x038E, 0x03A1 ),
    330          ( 0x03A3, 0x03CE ),
    331          ( 0x03D0, 0x03D6 ),
    332          0x03DA,
    333          0x03DC,
    334          0x03DE,
    335          0x03E0,
    336          ( 0x03E2, 0x03F3 ),
    337          ( 0x0401, 0x040C ),
    338          ( 0x040E, 0x044F ),
    339          ( 0x0451, 0x045C ),
    340          ( 0x045E, 0x0481 ),
    341          ( 0x0490, 0x04C4 ),
    342          ( 0x04C7, 0x04C8 ),
    343          ( 0x04CB, 0x04CC ),
    344          ( 0x04D0, 0x04EB ),
    345          ( 0x04EE, 0x04F5 ),
    346          ( 0x04F8, 0x04F9 ),
    347          ( 0x0531, 0x0556 ),
    348          0x0559,
    349          ( 0x0561, 0x0586 ),
    350          ( 0x05D0, 0x05EA ),
    351          ( 0x05F0, 0x05F2 ),
    352          ( 0x0621, 0x063A ),
    353          ( 0x0641, 0x064A ),
    354          ( 0x0671, 0x06B7 ),
    355          ( 0x06BA, 0x06BE ),
    356          ( 0x06C0, 0x06CE ),
    357          ( 0x06D0, 0x06D3 ),
    358          0x06D5,
    359          ( 0x06E5, 0x06E6 ),
    360          ( 0x0905, 0x0939 ),
    361          0x093D,
    362          ( 0x0958, 0x0961 ),
    363          ( 0x0985, 0x098C ),
    364          ( 0x098F, 0x0990 ),
    365          ( 0x0993, 0x09A8 ),
    366          ( 0x09AA, 0x09B0 ),
    367          0x09B2,
    368          ( 0x09B6, 0x09B9 ),
    369          ( 0x09DC, 0x09DD ),
    370          ( 0x09DF, 0x09E1 ),
    371          ( 0x09F0, 0x09F1 ),
    372          ( 0x0A05, 0x0A0A ),
    373          ( 0x0A0F, 0x0A10 ),
    374          ( 0x0A13, 0x0A28 ),
    375          ( 0x0A2A, 0x0A30 ),
    376          ( 0x0A32, 0x0A33 ),
    377          ( 0x0A35, 0x0A36 ),
    378          ( 0x0A38, 0x0A39 ),
    379          ( 0x0A59, 0x0A5C ),
    380          0x0A5E,
    381          ( 0x0A72, 0x0A74 ),
    382          ( 0x0A85, 0x0A8B ),
    383          0x0A8D,
    384          ( 0x0A8F, 0x0A91 ),
    385          ( 0x0A93, 0x0AA8 ),
    386          ( 0x0AAA, 0x0AB0 ),
    387          ( 0x0AB2, 0x0AB3 ),
    388          ( 0x0AB5, 0x0AB9 ),
    389          0x0ABD,
    390          0x0AE0,
    391          ( 0x0B05, 0x0B0C ),
    392          ( 0x0B0F, 0x0B10 ),
    393          ( 0x0B13, 0x0B28 ),
    394          ( 0x0B2A, 0x0B30 ),
    395          ( 0x0B32, 0x0B33 ),
    396          ( 0x0B36, 0x0B39 ),
    397          0x0B3D,
    398          ( 0x0B5C, 0x0B5D ),
    399          ( 0x0B5F, 0x0B61 ),
    400          ( 0x0B85, 0x0B8A ),
    401          ( 0x0B8E, 0x0B90 ),
    402          ( 0x0B92, 0x0B95 ),
    403          ( 0x0B99, 0x0B9A ),
    404          0x0B9C,
    405          ( 0x0B9E, 0x0B9F ),
    406          ( 0x0BA3, 0x0BA4 ),
    407          ( 0x0BA8, 0x0BAA ),
    408          ( 0x0BAE, 0x0BB5 ),
    409          ( 0x0BB7, 0x0BB9 ),
    410          ( 0x0C05, 0x0C0C ),
    411          ( 0x0C0E, 0x0C10 ),
    412          ( 0x0C12, 0x0C28 ),
    413          ( 0x0C2A, 0x0C33 ),
    414          ( 0x0C35, 0x0C39 ),
    415          ( 0x0C60, 0x0C61 ),
    416          ( 0x0C85, 0x0C8C ),
    417          ( 0x0C8E, 0x0C90 ),
    418          ( 0x0C92, 0x0CA8 ),
    419          ( 0x0CAA, 0x0CB3 ),
    420          ( 0x0CB5, 0x0CB9 ),
    421          0x0CDE,
    422          ( 0x0CE0, 0x0CE1 ),
    423          ( 0x0D05, 0x0D0C ),
    424          ( 0x0D0E, 0x0D10 ),
    425          ( 0x0D12, 0x0D28 ),
    426          ( 0x0D2A, 0x0D39 ),
    427          ( 0x0D60, 0x0D61 ),
    428          ( 0x0E01, 0x0E2E ),
    429          0x0E30,
    430          ( 0x0E32, 0x0E33 ),
    431          ( 0x0E40, 0x0E45 ),
    432          ( 0x0E81, 0x0E82 ),
    433          0x0E84,
    434          ( 0x0E87, 0x0E88 ),
    435          0x0E8A,
    436          0x0E8D,
    437          ( 0x0E94, 0x0E97 ),
    438          ( 0x0E99, 0x0E9F ),
    439          ( 0x0EA1, 0x0EA3 ),
    440          0x0EA5,
    441          0x0EA7,
    442          ( 0x0EAA, 0x0EAB ),
    443          ( 0x0EAD, 0x0EAE ),
    444          0x0EB0,
    445          ( 0x0EB2, 0x0EB3 ),
    446          0x0EBD,
    447          ( 0x0EC0, 0x0EC4 ),
    448          ( 0x0F40, 0x0F47 ),
    449          ( 0x0F49, 0x0F69 ),
    450          ( 0x10A0, 0x10C5 ),
    451          ( 0x10D0, 0x10F6 ),
    452          0x1100,
    453          ( 0x1102, 0x1103 ),
    454          ( 0x1105, 0x1107 ),
    455          0x1109,
    456          ( 0x110B, 0x110C ),
    457          ( 0x110E, 0x1112 ),
    458          0x113C,
    459          0x113E,
    460          0x1140,
    461          0x114C,
    462          0x114E,
    463          0x1150,
    464          ( 0x1154, 0x1155 ),
    465          0x1159,
    466          ( 0x115F, 0x1161 ),
    467          0x1163,
    468          0x1165,
    469          0x1167,
    470          0x1169,
    471          ( 0x116D, 0x116E ),
    472          ( 0x1172, 0x1173 ),
    473          0x1175,
    474          0x119E,
    475          0x11A8,
    476          0x11AB,
    477          ( 0x11AE, 0x11AF ),
    478          ( 0x11B7, 0x11B8 ),
    479          0x11BA,
    480          ( 0x11BC, 0x11C2 ),
    481          0x11EB,
    482          0x11F0,
    483          0x11F9,
    484          ( 0x1E00, 0x1E9B ),
    485          ( 0x1EA0, 0x1EF9 ),
    486          ( 0x1F00, 0x1F15 ),
    487          ( 0x1F18, 0x1F1D ),
    488          ( 0x1F20, 0x1F45 ),
    489          ( 0x1F48, 0x1F4D ),
    490          ( 0x1F50, 0x1F57 ),
    491          0x1F59,
    492          0x1F5B,
    493          0x1F5D,
    494          ( 0x1F5F, 0x1F7D ),
    495          ( 0x1F80, 0x1FB4 ),
    496          ( 0x1FB6, 0x1FBC ),
    497          0x1FBE,
    498          ( 0x1FC2, 0x1FC4 ),
    499          ( 0x1FC6, 0x1FCC ),
    500          ( 0x1FD0, 0x1FD3 ),
    501          ( 0x1FD6, 0x1FDB ),
    502          ( 0x1FE0, 0x1FEC ),
    503          ( 0x1FF2, 0x1FF4 ),
    504          ( 0x1FF6, 0x1FFC ),
    505          0x2126,
    506          ( 0x212A, 0x212B ),
    507          0x212E,
    508          ( 0x2180, 0x2182 ),
    509          ( 0x3041, 0x3094 ),
    510          ( 0x30A1, 0x30FA ),
    511          ( 0x3105, 0x312C ),
    512          ( 0xAC00, 0xD7A3 )
    513          )
    514
    515      Ideographic = CodePointSet(
    516          ( 0x4E00, 0x9FA5 ),
    517          0x3007,
    518          ( 0x3021, 0x3029 )
    519          )
    520
    521      Letter = CodePointSet(BaseChar).extend(Ideographic)
    522
    523      CombiningChar = CodePointSet(
    524          ( 0x0300, 0x0345 ),
    525          ( 0x0360, 0x0361 ),
    526          ( 0x0483, 0x0486 ),
    527          ( 0x0591, 0x05A1 ),
    528          ( 0x05A3, 0x05B9 ),
    529          ( 0x05BB, 0x05BD ),
    530          0x05BF,
    531          ( 0x05C1, 0x05C2 ),
    532          0x05C4,
    533          ( 0x064B, 0x0652 ),
    534          0x0670,
    535          ( 0x06D6, 0x06DC ),
    536          ( 0x06DD, 0x06DF ),
    537          ( 0x06E0, 0x06E4 ),
    538          ( 0x06E7, 0x06E8 ),
    539          ( 0x06EA, 0x06ED ),
    540          ( 0x0901, 0x0903 ),
    541          0x093C,
    542          ( 0x093E, 0x094C ),
    543          0x094D,
    544          ( 0x0951, 0x0954 ),
    545          ( 0x0962, 0x0963 ),
    546          ( 0x0981, 0x0983 ),
    547          0x09BC,
    548          0x09BE,
    549          0x09BF,
    550          ( 0x09C0, 0x09C4 ),
    551          ( 0x09C7, 0x09C8 ),
    552          ( 0x09CB, 0x09CD ),
    553          0x09D7,
    554          ( 0x09E2, 0x09E3 ),
    555          0x0A02,
    556          0x0A3C,
    557          0x0A3E,
    558          0x0A3F,
    559          ( 0x0A40, 0x0A42 ),
    560          ( 0x0A47, 0x0A48 ),
    561          ( 0x0A4B, 0x0A4D ),
    562          ( 0x0A70, 0x0A71 ),
    563          ( 0x0A81, 0x0A83 ),
    564          0x0ABC,
    565          ( 0x0ABE, 0x0AC5 ),
    566          ( 0x0AC7, 0x0AC9 ),
    567          ( 0x0ACB, 0x0ACD ),
    568          ( 0x0B01, 0x0B03 ),
    569          0x0B3C,
    570          ( 0x0B3E, 0x0B43 ),
    571          ( 0x0B47, 0x0B48 ),
    572          ( 0x0B4B, 0x0B4D ),
    573          ( 0x0B56, 0x0B57 ),
    574          ( 0x0B82, 0x0B83 ),
    575          ( 0x0BBE, 0x0BC2 ),
    576          ( 0x0BC6, 0x0BC8 ),
    577          ( 0x0BCA, 0x0BCD ),
    578          0x0BD7,
    579          ( 0x0C01, 0x0C03 ),
    580          ( 0x0C3E, 0x0C44 ),
    581          ( 0x0C46, 0x0C48 ),
    582          ( 0x0C4A, 0x0C4D ),
    583          ( 0x0C55, 0x0C56 ),
    584          ( 0x0C82, 0x0C83 ),
    585          ( 0x0CBE, 0x0CC4 ),
    586          ( 0x0CC6, 0x0CC8 ),
    587          ( 0x0CCA, 0x0CCD ),
    588          ( 0x0CD5, 0x0CD6 ),
    589          ( 0x0D02, 0x0D03 ),
    590          ( 0x0D3E, 0x0D43 ),
    591          ( 0x0D46, 0x0D48 ),
    592          ( 0x0D4A, 0x0D4D ),
    593          0x0D57,
    594          0x0E31,
    595          ( 0x0E34, 0x0E3A ),
    596          ( 0x0E47, 0x0E4E ),
    597          0x0EB1,
    598          ( 0x0EB4, 0x0EB9 ),
    599          ( 0x0EBB, 0x0EBC ),
    600          ( 0x0EC8, 0x0ECD ),
    601          ( 0x0F18, 0x0F19 ),
    602          0x0F35,
    603          0x0F37,
    604          0x0F39,
    605          0x0F3E,
    606          0x0F3F,
    607          ( 0x0F71, 0x0F84 ),
    608          ( 0x0F86, 0x0F8B ),
    609          ( 0x0F90, 0x0F95 ),
    610          0x0F97,
    611          ( 0x0F99, 0x0FAD ),
    612          ( 0x0FB1, 0x0FB7 ),
    613          0x0FB9,
    614          ( 0x20D0, 0x20DC ),
    615          0x20E1,
    616          ( 0x302A, 0x302F ),
    617          0x3099,
    618          0x309A
    619          )
    620
    621      Digit = CodePointSet(
    622          ( 0x0030, 0x0039 ),
    623          ( 0x0660, 0x0669 ),
    624          ( 0x06F0, 0x06F9 ),
    625          ( 0x0966, 0x096F ),
    626          ( 0x09E6, 0x09EF ),
    627          ( 0x0A66, 0x0A6F ),
    628          ( 0x0AE6, 0x0AEF ),
    629          ( 0x0B66, 0x0B6F ),
    630          ( 0x0BE7, 0x0BEF ),
    631          ( 0x0C66, 0x0C6F ),
    632          ( 0x0CE6, 0x0CEF ),
    633          ( 0x0D66, 0x0D6F ),
    634          ( 0x0E50, 0x0E59 ),
    635          ( 0x0ED0, 0x0ED9 ),
    636          ( 0x0F20, 0x0F29 )
    637          )
    638
    639      Extender = CodePointSet(
    640          0x00B7,
    641          0x02D0,
    642          0x02D1,
    643          0x0387,
    644          0x0640,
    645          0x0E46,
    646          0x0EC6,
    647          0x3005,
    648          ( 0x3031, 0x3035 ),
    649          ( 0x309D, 0x309E ),
    650          ( 0x30FC, 0x30FE )
    651          )
    652
    653      # Not an explicit production, but used in Name production
    654      NameStartChar = CodePointSet(Letter)
    655      NameStartChar.add(ord('_'))
    656      NameStartChar.add(ord(':'))
    657
    658      NCNameStartChar = CodePointSet(Letter)
    659      NCNameStartChar.add(ord('_'))
    660
    661      NameChar = CodePointSet(Letter)
    662      NameChar.extend(Digit)
    663      NameChar.add(ord('.'))
    664      NameChar.add(ord('-'))
    665      NameChar.add(ord('_'))
    666      NameChar.add(ord(':'))
    667      NameChar.extend(CombiningChar)
    668      NameChar.extend(Extender)
    669
    670      NCNameChar = CodePointSet(Letter)
    671      NCNameChar.extend(Digit)
    672      NCNameChar.add(ord('.'))
    673      NCNameChar.add(ord('-'))
    674      NCNameChar.add(ord('_'))
    675      NCNameChar.extend(CombiningChar)
    676      NCNameChar.extend(Extender)
    677
    678      Name_pat = '%s%s*' % (NameStartChar.asPattern(), NameChar.asPattern())
    679      Name_re = re.compile('^%s$' % (Name_pat,))
    680      NmToken_pat = '%s+' % (NameChar.asPattern(),)
    681      NmToken_re = re.compile('^%s$' % (NmToken_pat,))
    682      NCName_pat = '%s%s*' % (NCNameStartChar.asPattern(), NCNameChar.asPattern())
    683      NCName_re = re.compile('^%s$' % (NCName_pat,))
    684      QName_pat = '(%s:)?%s' % (NCName_pat, NCName_pat)
    685      QName_re = re.compile('^%s$' % (QName_pat,))
    686
    687  # Production 24 : Single Character Escapes
    688  SingleCharEsc = { 'n' : CodePointSet(0x0A),
    689                    'r' : CodePointSet(0x0D),
    690                    't' : CodePointSet(0x09) }
    691  for c in r'\|.-^?*+{}()[]':
    692      SingleCharEsc[c] = CodePointSet(ord(c))
    693
    694  # Production 25 : Category Escapes
    695  # Production 26: Complemented Category Escapes
    696  catEsc = { }
    697  complEsc = { }
    698  for k, v in PropertyMap.iteritems():
    699      catEsc[u'p{%s}' % (k,)] = v
    700      catEsc[u'P{%s}' % (k,)] = v.negate()
    701
    702  # Production 36 : IsBlock escapes
    703  IsBlockEsc = { }
    704  for k, v in BlockMap.iteritems():
    705      IsBlockEsc[u'p{Is%s}' % (k,)] = v
    706      IsBlockEsc[u'P{Is%s}' % (k,)] = v.negate()
    707
    708  # Production 37 : Multi-Character Escapes
    709  WildcardEsc = CodePointSet(ord('\n'), ord('\r')).negate()
    710  MultiCharEsc = { }
    711  MultiCharEsc['s'] = CodePointSet(0x20, ord('\t'), ord('\n'), ord('\r'))
    712  MultiCharEsc['S'] = MultiCharEsc['s'].negate()
    713  MultiCharEsc['i'] = CodePointSet(XML1p0e2.Letter).add(ord('_')).add(ord(':'))
    714  MultiCharEsc['I'] = MultiCharEsc['i'].negate()
    715  MultiCharEsc['c'] = CodePointSet(XML1p0e2.NameChar)
    716  MultiCharEsc['C'] = MultiCharEsc['c'].negate()
    717  MultiCharEsc['d'] = PropertyMap['Nd']
    718  MultiCharEsc['D'] = MultiCharEsc['d'].negate()
    719  MultiCharEsc['W'] = CodePointSet(PropertyMap['P']).extend(PropertyMap['Z']).extend(PropertyMap['C'])
    720  MultiCharEsc['w'] = MultiCharEsc['W'].negate()
    721