Package epydoc :: Package markup :: Module doctest

[hide private]
[frames] | no frames]

# Source Code for Module epydoc.markup.doctest

      1  #
      2  # doctest.py: Syntax Highlighting for doctest blocks
      3  # Edward Loper
      4  #
      5  # Created [06/28/03 02:52 AM]
      6  # $Id: restructuredtext.py 1210 2006-04-10 13:25:50Z edloper $
      7  #
      8
      9  """
     10  Syntax highlighting for doctest blocks.  This module defines two
     11  functions, L{doctest_to_html()} and L{doctest_to_latex()}, which can
     12  be used to perform syntax highlighting on doctest blocks.  It also
     13  defines the more general C{colorize_doctest()}, which could be used to
     14  do syntac highlighting on doctest blocks with other output formats.
     15  (Both C{doctest_to_html()} and C{doctest_to_latex()} are defined using
     16  C{colorize_doctest()}.)
     17  """
     18  __docformat__ = 'epytext en'
     19
     20  import re
     21  from epydoc.util import plaintext_to_html, plaintext_to_latex
     22
     23  __all__ = ['doctest_to_html', 'doctest_to_latex',
     24             'DoctestColorizer', 'XMLDoctestColorizer',
     25             'HTMLDoctestColorizer', 'LaTeXDoctestColorizer']
     26
     27 -def doctest_to_html(s):
     28      """
     29      Perform syntax highlighting on the given doctest string, and
     30      return the resulting HTML code.  This code consists of a C{<pre>}
     31      block with class=py-doctest.  Syntax highlighting is performed
     32      using the following css classes:
     33
     34        - C{py-prompt} -- the Python PS1 prompt (>>>)
     35        - C{py-more} -- the Python PS2 prompt (...)
     36        - C{py-keyword} -- a Python keyword (for, if, etc.)
     37        - C{py-builtin} -- a Python builtin name (abs, dir, etc.)
     38        - C{py-string} -- a string literal
     39        - C{py-comment} -- a comment
     40        - C{py-except} -- an exception traceback (up to the next >>>)
     41        - C{py-output} -- the output from a doctest block.
     42        - C{py-defname} -- the name of a function or class defined by
     43          a C{def} or C{class} statement.
     44      """
     45      return HTMLDoctestColorizer().colorize_doctest(s)
     46
     47 -def doctest_to_latex(s):
     48      """
     49      Perform syntax highlighting on the given doctest string, and
     50      return the resulting LaTeX code.  This code consists of an
     51      C{alltt} environment.  Syntax highlighting is performed using
     52      the following new latex commands, which must be defined externally:
     53        - C{\pysrcprompt} -- the Python PS1 prompt (>>>)
     54        - C{\pysrcmore} -- the Python PS2 prompt (...)
     55        - C{\pysrckeyword} -- a Python keyword (for, if, etc.)
     56        - C{\pysrcbuiltin} -- a Python builtin name (abs, dir, etc.)
     57        - C{\pysrcstring} -- a string literal
     58        - C{\pysrccomment} -- a comment
     59        - C{\pysrcexcept} -- an exception traceback (up to the next >>>)
     60        - C{\pysrcoutput} -- the output from a doctest block.
     61        - C{\pysrcdefname} -- the name of a function or class defined by
     62          a C{def} or C{class} statement.
     63      """
     64      return LaTeXDoctestColorizer().colorize_doctest(s)
     65
     66 -class DoctestColorizer:
     67      """
     68      An abstract base class for performing syntax highlighting on
     69      doctest blocks and other bits of Python code.  Subclasses should
     70      provide definitions for:
     71
     72        - The L{markup()} method, which takes a substring and a tag, and
     73          returns a colorized version of the substring.
     74        - The L{PREFIX} and L{SUFFIX} variables, which will be added
     75          to the beginning and end of the strings returned by
     76          L{colorize_codeblock} and L{colorize_doctest}.
     77      """
     78
     79      #: A string that is added to the beginning of the strings
     80      #: returned by L{colorize_codeblock} and L{colorize_doctest}.
     81      #: Typically, this string begins a preformatted area.
     82      PREFIX = None
     83
     84      #: A string that is added to the end of the strings
     85      #: returned by L{colorize_codeblock} and L{colorize_doctest}.
     86      #: Typically, this string ends a preformatted area.
     87      SUFFIX = None
     88
     89      #: A list of the names of all Python keywords.  ('as' is included
     90      #: even though it is technically not a keyword.)
     91      _KEYWORDS = ("and       del       for       is        raise"
     92                   "assert    elif      from      lambda    return"
     93                   "break     else      global    not       try"
     94                   "class     except    if        or        while"
     95                   "continue  exec      import    pass      yield"
     96                   "def       finally   in        print     as").split()
     97
     98      #: A list of all Python builtins.
     99      _BUILTINS = [_BI for _BI in dir(__builtins__)
    100                   if not _BI.startswith('__')]
    101
    102      #: A regexp group that matches keywords.
    103      _KEYWORD_GRP = '|'.join([r'\b%s\b' % _KW for _KW in _KEYWORDS])
    104
    105      #: A regexp group that matches Python builtins.
    106      _BUILTIN_GRP = (r'(?<!\.)(?:%s)' % '|'.join([r'\b%s\b' % _BI
    107                                                   for _BI in _BUILTINS]))
    108
    109      #: A regexp group that matches Python strings.
    110      _STRING_GRP = '|'.join(
    111          [r'("""("""|.*?((?!").)"""))', r'("("|.*?((?!").)"))',
    112           r"('''('''|.*?[^\\']'''))", r"('('|.*?[^\\']'))"])
    113
    114      #: A regexp group that matches Python comments.
    115      _COMMENT_GRP = '(#.*?$)'
    116
    117      #: A regexp group that matches Python ">>>" prompts.
    118      _PROMPT1_GRP = r'^[ \t]*>>>(?:[ \t]|$)'
    119
    120      #: A regexp group that matches Python "..." prompts.
    121      _PROMPT2_GRP = r'^[ \t]*\.\.\.(?:[ \t]|$)'
    122
    123      #: A regexp group that matches function and class definitions.
    124      _DEFINE_GRP = r'\b(?:def|class)[ \t]+\w+'
    125
    126      #: A regexp that matches Python prompts
    127      PROMPT_RE = re.compile('(%s|%s)' % (_PROMPT1_GRP, _PROMPT2_GRP),
    128                             re.MULTILINE | re.DOTALL)
    129
    130      #: A regexp that matches Python "..." prompts.
    131      PROMPT2_RE = re.compile('(%s)' % _PROMPT2_GRP,
    132                              re.MULTILINE | re.DOTALL)
    133
    134      #: A regexp that matches doctest exception blocks.
    135      EXCEPT_RE = re.compile(r'^[ \t]*Traceback \(most recent call last\):.*',
    136                             re.DOTALL | re.MULTILINE)
    137
    138      #: A regexp that matches doctest directives.
    139      DOCTEST_DIRECTIVE_RE = re.compile(r'#[ \t]*doctest:.*')
    140
    141      #: A regexp that matches all of the regions of a doctest block
    142      #: that should be colored.
    143      DOCTEST_RE = re.compile(
    144          r'(.*?)((?P<STRING>%s)|(?P<COMMENT>%s)|(?P<DEFINE>%s)|'
    145                r'(?P<KEYWORD>%s)|(?P<BUILTIN>%s)|'
    146                r'(?P<PROMPT1>%s)|(?P<PROMPT2>%s)|(?P<EOS>\Z))' % (
    147          _STRING_GRP, _COMMENT_GRP, _DEFINE_GRP, _KEYWORD_GRP, _BUILTIN_GRP,
    148          _PROMPT1_GRP, _PROMPT2_GRP), re.MULTILINE | re.DOTALL)
    149
    150      #: This regular expression is used to find doctest examples in a
    151      #: string.  This is copied from the standard Python doctest.py
    152      #: module (after the refactoring in Python 2.4+).
    153      DOCTEST_EXAMPLE_RE = re.compile(r'''
    154          # Source consists of a PS1 line followed by zero or more PS2 lines.
    155          (?P<source>
    156              (?:^(?P<indent> [ ]*) >>>    .*)    # PS1 line
    157              (?:\n           [ ]*  \.\.\. .*)*   # PS2 lines
    158            \n?)
    159          # Want consists of any non-blank lines that do not start with PS1.
    160          (?P<want> (?:(?![ ]*$)    # Not a blank line
    161                       (?![ ]*>>>)  # Not a line starting with PS1
    162                       .*$\n?       # But any other line
    163                    )*)
    164          ''', re.MULTILINE | re.VERBOSE)
    165
    166 -    def colorize_inline(self, s):
    167          """
    168          Colorize a string containing Python code.  Do not add the
    169          L{PREFIX} and L{SUFFIX} strings to the returned value.  This
    170          method is intended for generating syntax-highlighted strings
    171          that are appropriate for inclusion as inline expressions.
    172          """
    173          return self.DOCTEST_RE.sub(self.subfunc, s)
    174
    175 -    def colorize_codeblock(self, s):
    176          """
    177          Colorize a string containing only Python code.  This method
    178          differs from L{colorize_doctest} in that it will not search
    179          for doctest prompts when deciding how to colorize the string.
    180          """
    181          body = self.DOCTEST_RE.sub(self.subfunc, s)
    182          return self.PREFIX + body + self.SUFFIX
    183
    184 -    def colorize_doctest(self, s, strip_directives=False):
    185          """
    186          Colorize a string containing one or more doctest examples.
    187          """
    188          output = []
    189          charno = 0
    190          for m in self.DOCTEST_EXAMPLE_RE.finditer(s):
    191              # Parse the doctest example:
    192              pysrc, want = m.group('source', 'want')
    193              # Pre-example text:
    194              output.append(s[charno:m.start()])
    195              # Example source code:
    196              output.append(self.DOCTEST_RE.sub(self.subfunc, pysrc))
    197              # Example output:
    198              if want:
    199                  if self.EXCEPT_RE.match(want):
    200                      output += '\n'.join([self.markup(line, 'except')
    201                                           for line in want.split('\n')])
    202                  else:
    203                      output += '\n'.join([self.markup(line, 'output')
    204                                           for line in want.split('\n')])
    205              # Update charno
    206              charno = m.end()
    207          # Add any remaining post-example text.
    208          output.append(s[charno:])
    209
    210          return self.PREFIX + ''.join(output) + self.SUFFIX
    211
    212 -    def subfunc(self, match):
    213          other, text = match.group(1, 2)
    214          #print 'M %20r %20r' % (other, text) # <- for debugging
    215          if other:
    216              other = '\n'.join([self.markup(line, 'other')
    217                                 for line in other.split('\n')])
    218
    219          if match.group('PROMPT1'):
    220              return other + self.markup(text, 'prompt')
    221          elif match.group('PROMPT2'):
    222              return other + self.markup(text, 'more')
    223          elif match.group('KEYWORD'):
    224              return other + self.markup(text, 'keyword')
    225          elif match.group('BUILTIN'):
    226              return other + self.markup(text, 'builtin')
    227          elif match.group('COMMENT'):
    228              return other + self.markup(text, 'comment')
    229          elif match.group('STRING') and '\n' not in text:
    230              return other + self.markup(text, 'string')
    231          elif match.group('STRING'):
    232              # It's a multiline string; colorize the string & prompt
    233              # portion of each line.
    234              pieces = []
    235              for line in text.split('\n'):
    236                  if self.PROMPT2_RE.match(line):
    237                      if len(line) > 4:
    238                          pieces.append(self.markup(line[:4], 'more') +
    239                                        self.markup(line[4:], 'string'))
    240                      else:
    241                          pieces.append(self.markup(line[:4], 'more'))
    242                  elif line:
    243                      pieces.append(self.markup(line, 'string'))
    244                  else:
    245                      pieces.append('')
    246              return other + '\n'.join(pieces)
    247          elif match.group('DEFINE'):
    248              m = re.match('(?P<def>\w+)(?P<space>\s+)(?P<name>\w+)', text)
    249              return other + (self.markup(m.group('def'), 'keyword') +
    250                          self.markup(m.group('space'), 'other') +
    251                          self.markup(m.group('name'), 'defname'))
    252          elif match.group('EOS') is not None:
    253              return other
    254          else:
    255              assert 0, 'Unexpected match!'
    256
    257 -    def markup(self, s, tag):
    258          """
    259          Apply syntax highlighting to a single substring from a doctest
    260          block.  C{s} is the substring, and C{tag} is the tag that
    261          should be applied to the substring.  C{tag} will be one of the
    262          following strings:
    263
    264            - C{prompt} -- the Python PS1 prompt (>>>)
    265            - C{more} -- the Python PS2 prompt (...)
    266            - C{keyword} -- a Python keyword (for, if, etc.)
    267            - C{builtin} -- a Python builtin name (abs, dir, etc.)
    268            - C{string} -- a string literal
    269            - C{comment} -- a comment
    270            - C{except} -- an exception traceback (up to the next >>>)
    271            - C{output} -- the output from a doctest block.
    272            - C{defname} -- the name of a function or class defined by
    273              a C{def} or C{class} statement.
    274            - C{other} -- anything else (does *not* include output.)
    275          """
    276          raise AssertionError("Abstract method")
    277
    278 -class XMLDoctestColorizer(DoctestColorizer):
    279      """
    280      A subclass of DoctestColorizer that generates XML-like output.
    281      This class is mainly intended to be used for testing purposes.
    282      """
    283      PREFIX = '<colorized>\n'
    284      SUFFIX = '</colorized>\n'
    285 -    def markup(self, s, tag):
    286          s = s.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
    287          if tag == 'other': return s
    288          else: return '<%s>%s</%s>' % (tag, s, tag)
    289
    290 -class HTMLDoctestColorizer(DoctestColorizer):
    291      """A subclass of DoctestColorizer that generates HTML output."""
    292      PREFIX = '<pre class="py-doctest">\n'
    293      SUFFIX = '</pre>\n'
    294 -    def markup(self, s, tag):
    295          if tag == 'other':
    296              return plaintext_to_html(s)
    297          else:
    298              return ('<span class="py-%s">%s</span>' %
    299                      (tag, plaintext_to_html(s)))
    300
    301 -class LaTeXDoctestColorizer(DoctestColorizer):
    302      """A subclass of DoctestColorizer that generates LaTeX output."""
    303      PREFIX = '\\begin{alltt}\n'
    304      SUFFIX = '\\end{alltt}\n'
    305 -    def markup(self, s, tag):
    306          if tag == 'other':
    307              return plaintext_to_latex(s)
    308          else:
    309              return '\\pysrc%s{%s}' % (tag, plaintext_to_latex(s))
    310