Package epydoc :: Module docparser

[hide private]
[frames] | no frames]

# Source Code for Module epydoc.docparser

       1  # epydoc -- Source code parsing
       2  #
       3  # Copyright (C) 2005 Edward Loper
       4  # Author: Edward Loper <edloper@loper.org>
       5  # URL: <http://epydoc.sf.net>
       6  #
       7  # $Id: docparser.py 1673 2008-01-29 05:42:58Z edloper $
       8
       9  """
      10  Extract API documentation about python objects by parsing their source
      11  code.
      12
      13  The function L{parse_docs()}, which provides the main interface
      14  of this module, reads and parses the Python source code for a
      15  module, and uses it to create an L{APIDoc} object containing
      16  the API documentation for the variables and values defined in
      17  that modules.
      18
      19  Currently, C{parse_docs()} extracts documentation from the following
      20  source code constructions:
      21
      22    - module docstring
      23    - import statements
      24    - class definition blocks
      25    - function definition blocks
      26    - assignment statements
      27      - simple assignment statements
      28      - assignment statements with multiple C{'='}s
      29      - assignment statements with unpacked left-hand sides
      30      - assignment statements that wrap a function in classmethod
      31        or staticmethod.
      32      - assignment to special variables __path__, __all__, and
      33        __docformat__.
      34    - delete statements
      35
      36  C{parse_docs()} does not yet support the following source code
      37  constructions:
      38
      39    - assignment statements that create properties
      40
      41  By default, C{parse_docs()} will expore the contents of top-level
      42  C{try} and C{if} blocks.  If desired, C{parse_docs()} can also
      43  be configured to explore the contents of C{while} and C{for} blocks.
      44  (See the configuration constants, below.)
      45
      46  @todo: Make it possible to extend the functionality of C{parse_docs()},
      47         by replacing process_line with a dispatch table that can be
      48         customized (similarly to C{docintrospector.register_introspector()}).
      49  """
      50  __docformat__ = 'epytext en'
      51
      52  ######################################################################
      53  ## Imports
      54  ######################################################################
      55
      56  # Python source code parsing:
      57  import token, tokenize
      58  # Finding modules:
      59  import imp
      60  # File services:
      61  import os, os.path, sys
      62  # Unicode:
      63  import codecs
      64  # API documentation encoding:
      65  from epydoc.apidoc import *
      66  # For looking up the docs of builtins:
      67  import __builtin__, exceptions
      68  import epydoc.docintrospecter
      69  # Misc utility functions:
      70  from epydoc.util import *
      71  # Backwards compatibility
      72  from epydoc.compat import *
      73
      74  ######################################################################
      75  ## Doc Parser
      76  ######################################################################
      77
      78 -class ParseError(Exception):
      79      """
      80      An exception that is used to signify that C{docparser} encountered
      81      syntactically invalid Python code while processing a Python source
      82      file.
      83      """
      84
      85  _moduledoc_cache = {}
      86  """A cache of C{ModuleDoc}s that we've already created.
      87  C{_moduledoc_cache} is a dictionary mapping from filenames to
      88  C{ValueDoc} objects.
      89  @type: C{dict}"""
      90
      91  #////////////////////////////////////////////////////////////
      92  # Configuration Constants
      93  #////////////////////////////////////////////////////////////
      94
      95  #{ Configuration Constants: Control Flow
      96  PARSE_TRY_BLOCKS = True
      97  """Should the contents of C{try} blocks be examined?"""
      98  PARSE_EXCEPT_BLOCKS = True
      99  """Should the contents of C{except} blocks be examined?"""
     100  PARSE_FINALLY_BLOCKS = True
     101  """Should the contents of C{finally} blocks be examined?"""
     102  PARSE_IF_BLOCKS = True
     103  """Should the contents of C{if} blocks be examined?"""
     104  PARSE_ELSE_BLOCKS = True
     105  """Should the contents of C{else} and C{elif} blocks be examined?"""
     106  PARSE_WHILE_BLOCKS = False
     107  """Should the contents of C{while} blocks be examined?"""
     108  PARSE_FOR_BLOCKS = False
     109  """Should the contents of C{for} blocks be examined?"""
     110
     111  #{ Configuration Constants: Imports
     112  IMPORT_HANDLING = 'link'
     113  """What should C{docparser} do when it encounters an import
     114  statement?
     115    - C{'link'}: Create variabledoc objects with imported_from pointers
     116      to the source object.
     117    - C{'parse'}: Parse the imported file, to find the actual
     118      documentation for the imported object.  (This will fall back
     119      to the 'link' behavior if the imported file can't be parsed,
     120      e.g., if it's a builtin.)
     121  """
     122
     123  IMPORT_STAR_HANDLING = 'parse'
     124  """When C{docparser} encounters a C{'from M{m} import *'}
     125  statement, and is unable to parse C{M{m}} (either because
     126  L{IMPORT_HANDLING}=C{'link'}, or because parsing failed), how
     127  should it determine the list of identifiers expored by C{M{m}}?
     128    - C{'ignore'}: ignore the import statement, and don't create
     129      any new variables.
     130    - C{'parse'}: parse it to find a list of the identifiers that it
     131      exports.  (This will fall back to the 'ignore' behavior if the
     132      imported file can't be parsed, e.g., if it's a builtin.)
     133    - C{'introspect'}: import the module and introspect it (using C{dir})
     134      to find a list of the identifiers that it exports.  (This will
     135      fall back to the 'ignore' behavior if the imported file can't
     136      be parsed, e.g., if it's a builtin.)
     137  """
     138
     139  DEFAULT_DECORATOR_BEHAVIOR = 'transparent'
     140  """When C{DocParse} encounters an unknown decorator, what should
     141  it do to the documentation of the decorated function?
     142    - C{'transparent'}: leave the function's documentation as-is.
     143    - C{'opaque'}: replace the function's documentation with an
     144      empty C{ValueDoc} object, reflecting the fact that we have no
     145      knowledge about what value the decorator returns.
     146  """
     147
     148  BASE_HANDLING = 'parse'#'link'
     149  """What should C{docparser} do when it encounters a base class that
     150  was imported from another module?
     151    - C{'link'}: Create a valuedoc with a C{proxy_for} pointer to the
     152      base class.
     153    - C{'parse'}: Parse the file containing the base class, to find
     154      the actual documentation for it.  (This will fall back to the
     155      'link' behavior if the imported file can't be parsed, e.g., if
     156      it's a builtin.)
     157  """
     158
     159  #{ Configuration Constants: Comment docstrings
     160  COMMENT_DOCSTRING_MARKER = '#:'
     161  """The prefix used to mark comments that contain attribute
     162  docstrings for variables."""
     163
     164  #{ Configuration Constants: Grouping
     165  START_GROUP_MARKER = '#{'
     166  """The prefix used to mark a comment that starts a group.  This marker
     167  should be followed (on the same line) by the name of the group.
     168  Following a start-group comment, all variables defined at the same
     169  indentation level will be assigned to this group name, until the
     170  parser reaches the end of the file, a matching end-group comment, or
     171  another start-group comment at the same indentation level.
     172  """
     173
     174  END_GROUP_MARKER = '#}'
     175  """The prefix used to mark a comment that ends a group.  See
     176  L{START_GROUP_MARKER}."""
     177
     178  #/////////////////////////////////////////////////////////////////
     179  #{ Module parser
     180  #/////////////////////////////////////////////////////////////////
     181
     182 -def parse_docs(filename=None, name=None, context=None, is_script=False):
     183      """
     184      Generate the API documentation for a specified object by
     185      parsing Python source files, and return it as a L{ValueDoc}.
     186      The object to generate documentation for may be specified
     187      using the C{filename} parameter I{or} the C{name} parameter.
     188      (It is an error to specify both a filename and a name; or to
     189      specify neither a filename nor a name).
     190
     191      @param filename: The name of the file that contains the python
     192          source code for a package, module, or script.  If
     193          C{filename} is specified, then C{parse} will return a
     194          C{ModuleDoc} describing its contents.
     195      @param name: The fully-qualified python dotted name of any
     196          value (including packages, modules, classes, and
     197          functions).  C{parse_docs()} will automatically figure out
     198          which module(s) it needs to parse in order to find the
     199          documentation for the specified object.
     200      @param context: The API documentation for the package that
     201          contains C{filename}.  If no context is given, then
     202          C{filename} is assumed to contain a top-level module or
     203          package.  It is an error to specify a C{context} if the
     204          C{name} argument is used.
     205      @rtype: L{ValueDoc}
     206      """
     207      # Always introspect __builtins__ & exceptions (e.g., in case
     208      # they're used as base classes.)
     209      epydoc.docintrospecter.introspect_docs(__builtin__)
     210      epydoc.docintrospecter.introspect_docs(exceptions)
     211
     212      # If our input is a python object name, then delegate to
     213      # _find().
     214      if filename is None and name is not None:
     215          if context:
     216              raise ValueError("context should only be specified together "
     217                               "with filename, not with name.")
     218          name = DottedName(name)
     219          val_doc = _find(name)
     220          if val_doc.canonical_name is UNKNOWN:
     221              val_doc.canonical_name = name
     222          return val_doc
     223
     224      # If our input is a filename, then create a ModuleDoc for it,
     225      # and use process_file() to populate its attributes.
     226      elif filename is not None and name is None:
     227          # Use a python source version, if possible.
     228          if not is_script:
     229              try: filename = py_src_filename(filename)
     230              except ValueError, e: raise ImportError('%s' % e)
     231
     232          # Check the cache, first.
     233          if filename in _moduledoc_cache:
     234              return _moduledoc_cache[filename]
     235
     236          log.info("Parsing %s" % filename)
     237
     238          # If the context wasn't provided, then check if the file is in
     239          # a package directory.  If so, then update basedir & name to
     240          # contain the topmost package's directory and the fully
     241          # qualified name for this file.  (This update assume the
     242          # default value of __path__ for the parent packages; if the
     243          # parent packages override their __path__s, then this can
     244          # cause us not to find the value.)
     245          if context is None and not is_script:
     246              basedir = os.path.split(filename)[0]
     247              name = os.path.splitext(os.path.split(filename)[1])[0]
     248              if name == '__init__':
     249                  basedir, name = os.path.split(basedir)
     250              context = _parse_package(basedir)
     251
     252          # Figure out the canonical name of the module we're parsing.
     253          if not is_script:
     254              module_name, is_pkg = _get_module_name(filename, context)
     255          else:
     256              module_name = DottedName(munge_script_name(filename))
     257              is_pkg = False
     258
     259          # Create a new ModuleDoc for the module, & add it to the cache.
     260          module_doc = ModuleDoc(canonical_name=module_name, variables={},
     261                                 sort_spec=[], imports=[],
     262                                 filename=filename, package=context,
     263                                 is_package=is_pkg, submodules=[],
     264                                 docs_extracted_by='parser')
     265          module_doc.defining_module = module_doc
     266          _moduledoc_cache[filename] = module_doc
     267
     268          # Set the module's __path__ to its default value.
     269          if is_pkg:
     270              module_doc.path = [os.path.split(module_doc.filename)[0]]
     271
     272          # Add this module to the parent package's list of submodules.
     273          if context is not None:
     274              context.submodules.append(module_doc)
     275
     276          # Tokenize & process the contents of the module's source file.
     277          try:
     278              process_file(module_doc)
     279          except tokenize.TokenError, e:
     280              msg, (srow, scol) = e.args
     281              raise ParseError('Error during parsing: %s '
     282                               '(%s, line %d, char %d)' %
     283                               (msg, module_doc.filename, srow, scol))
     284          except IndentationError, e:
     285              raise ParseError('Error during parsing: %s (%s)' %
     286                               (e, module_doc.filename))
     287
     288          # Handle any special variables (__path__, __docformat__, etc.)
     289          handle_special_module_vars(module_doc)
     290
     291          # Return the completed ModuleDoc
     292          return module_doc
     293      else:
     294          raise ValueError("Expected exactly one of the following "
     295                           "arguments: name, filename")
     296
     297 -def _parse_package(package_dir):
     298      """
     299      If the given directory is a package directory, then parse its
     300      __init__.py file (and the __init__.py files of all ancestor
     301      packages); and return its C{ModuleDoc}.
     302      """
     303      if not is_package_dir(package_dir):
     304          return None
     305      parent_dir = os.path.split(package_dir)[0]
     306      parent_doc = _parse_package(parent_dir)
     307      package_file = os.path.join(package_dir, '__init__')
     308      return parse_docs(filename=package_file, context=parent_doc)
     309
     310  # Special vars:
     311  # C{__docformat__}, C{__all__}, and C{__path__}.
     312 -def handle_special_module_vars(module_doc):
     313      # If __docformat__ is defined, parse its value.
     314      toktree = _module_var_toktree(module_doc, '__docformat__')
     315      if toktree is not None:
     316          try: module_doc.docformat = parse_string(toktree)
     317          except: pass
     318          del module_doc.variables['__docformat__']
     319
     320      # If __all__ is defined, parse its value.
     321      toktree = _module_var_toktree(module_doc, '__all__')
     322      if toktree is not None:
     323          try:
     324              public_names = set(parse_string_list(toktree))
     325              for name, var_doc in module_doc.variables.items():
     326                  if name in public_names:
     327                      var_doc.is_public = True
     328                      if not isinstance(var_doc, ModuleDoc):
     329                          var_doc.is_imported = False
     330                  else:
     331                      var_doc.is_public = False
     332          except ParseError:
     333              # If we couldn't parse the list, give precedence to introspection.
     334              for name, var_doc in module_doc.variables.items():
     335                  if not isinstance(var_doc, ModuleDoc):
     336                      var_doc.is_imported = UNKNOWN
     337          del module_doc.variables['__all__']
     338
     339      # If __path__ is defined, then extract its value (pkgs only)
     340      if module_doc.is_package:
     341          toktree = _module_var_toktree(module_doc, '__path__')
     342          if toktree is not None:
     343              try:
     344                  module_doc.path = parse_string_list(toktree)
     345              except ParseError:
     346                  pass # [xx]
     347              del module_doc.variables['__path__']
     348
     349 -def _module_var_toktree(module_doc, name):
     350      var_doc = module_doc.variables.get(name)
     351      if (var_doc is None or var_doc.value in (None, UNKNOWN) or
     352          var_doc.value.toktree is UNKNOWN):
     353          return None
     354      else:
     355          return var_doc.value.toktree
     356
     357  #////////////////////////////////////////////////////////////
     358  #{ Module Lookup
     359  #////////////////////////////////////////////////////////////
     360
     361 -def _find(name, package_doc=None):
     362      """
     363      Return the API documentaiton for the object whose name is
     364      C{name}.  C{package_doc}, if specified, is the API
     365      documentation for the package containing the named object.
     366      """
     367      # If we're inside a package, then find the package's path.
     368      if package_doc is None:
     369          path = None
     370      elif package_doc.path is not UNKNOWN:
     371          path = package_doc.path
     372      else:
     373          path = [os.path.split(package_doc.filename)[0]]
     374
     375      # The leftmost identifier in `name` should be a module or
     376      # package on the given path; find it and parse it.
     377      filename = _get_filename(name[0], path)
     378      module_doc = parse_docs(filename, context=package_doc)
     379
     380      # If the name just has one identifier, then the module we just
     381      # parsed is the object we're looking for; return it.
     382      if len(name) == 1: return module_doc
     383
     384      # Otherwise, we're looking for something inside the module.
     385      # First, check to see if it's in a variable (but ignore
     386      # variables that just contain imported submodules).
     387      if not _is_submodule_import_var(module_doc, name[1]):
     388          try: return _find_in_namespace(name[1:], module_doc)
     389          except ImportError: pass
     390
     391      # If not, then check to see if it's in a subpackage.
     392      if module_doc.is_package:
     393          return _find(name[1:], module_doc)
     394
     395      # If it's not in a variable or a subpackage, then we can't
     396      # find it.
     397      raise ImportError('Could not find value')
     398
     399 -def _is_submodule_import_var(module_doc, var_name):
     400      """
     401      Return true if C{var_name} is the name of a variable in
     402      C{module_doc} that just contains an C{imported_from} link to a
     403      submodule of the same name.  (I.e., is a variable created when
     404      a package imports one of its own submodules.)
     405      """
     406      var_doc = module_doc.variables.get(var_name)
     407      full_var_name = DottedName(module_doc.canonical_name, var_name)
     408      return (var_doc is not None and
     409              var_doc.imported_from == full_var_name)
     410
     411 -def _find_in_namespace(name, namespace_doc):
     412      if name[0] not in namespace_doc.variables:
     413          raise ImportError('Could not find value')
     414
     415      # Look up the variable in the namespace.
     416      var_doc = namespace_doc.variables[name[0]]
     417      if var_doc.value is UNKNOWN:
     418          raise ImportError('Could not find value')
     419      val_doc = var_doc.value
     420
     421      # If the variable's value was imported, then follow its
     422      # alias link.
     423      if var_doc.imported_from not in (None, UNKNOWN):
     424          return _find(var_doc.imported_from+name[1:])
     425
     426      # Otherwise, if the name has one identifier, then this is the
     427      # value we're looking for; return it.
     428      elif len(name) == 1:
     429          return val_doc
     430
     431      # Otherwise, if this value is a namespace, look inside it.
     432      elif isinstance(val_doc, NamespaceDoc):
     433          return _find_in_namespace(name[1:], val_doc)
     434
     435      # Otherwise, we ran into a dead end.
     436      else:
     437          raise ImportError('Could not find value')
     438
     439 -def _get_filename(identifier, path=None):
     440      if path is UNKNOWN: path = None
     441      try:
     442          fp, filename, (s,m,typ) = imp.find_module(identifier, path)
     443          if fp is not None: fp.close()
     444      except ImportError:
     445          raise ImportError, 'No Python source file found.'
     446
     447      if typ == imp.PY_SOURCE:
     448          return filename
     449      elif typ == imp.PY_COMPILED:
     450          # See if we can find a corresponding non-compiled version.
     451          filename = re.sub('.py\w$', '.py', filename)
     452          if not os.path.exists(filename):
     453              raise ImportError, 'No Python source file found.'
     454          return filename
     455      elif typ == imp.PKG_DIRECTORY:
     456          filename = os.path.join(filename, '__init__.py')
     457          if not os.path.exists(filename):
     458              filename = os.path.join(filename, '__init__.pyw')
     459              if not os.path.exists(filename):
     460                  raise ImportError, 'No package file found.'
     461          return filename
     462      elif typ == imp.C_BUILTIN:
     463          raise ImportError, 'No Python source file for builtin modules.'
     464      elif typ == imp.C_EXTENSION:
     465          raise ImportError, 'No Python source file for c extensions.'
     466      else:
     467          raise ImportError, 'No Python source file found.'
     468
     469  #/////////////////////////////////////////////////////////////////
     470  #{ File tokenization loop
     471  #/////////////////////////////////////////////////////////////////
     472
     473 -def process_file(module_doc):
     474      """
     475      Read the given C{ModuleDoc}'s file, and add variables
     476      corresponding to any objects defined in that file.  In
     477      particular, read and tokenize C{module_doc.filename}, and
     478      process each logical line using L{process_line()}.
     479      """
     480      # Keep track of the current line number:
     481      lineno = None
     482
     483      # Use this list to collect the tokens on a single logical line:
     484      line_toks = []
     485
     486      # This list contains one APIDoc for each indentation level.
     487      # The first element is the APIDoc for the module, and each
     488      # subsequent element is the APIDoc for the object at that
     489      # indentation level.  The final element of the list is the
     490      # C{APIDoc} for the entity that we're currently processing.
     491      parent_docs = [module_doc]
     492
     493      # The APIDoc for the object that was defined by the previous
     494      # line, if any; or None otherwise.  This is used to update
     495      # parent_docs when we encounter an indent; and to decide what
     496      # object (if any) is described by a docstring.
     497      prev_line_doc = module_doc
     498
     499      # A list of comments that occur before or on the current
     500      # logical line, used to build the comment docstring.  Each
     501      # element is a tuple (comment_text, comment_lineno).
     502      comments = []
     503
     504      # A list of decorator lines that occur before the current
     505      # logical line.  This is used so we can process a function
     506      # declaration line and its decorators all at once.
     507      decorators = []
     508
     509      # A list of group names, one for each indentation level.  This is
     510      # used to keep track groups that are defined by comment markers
     511      # START_GROUP_MARKER and END_GROUP_MARKER.
     512      groups = [None]
     513
     514      # When we encounter a comment start group marker, set this to the
     515      # name of the group; but wait until we're ready to process the
     516      # next line before we actually set groups[-1] to this value.  This
     517      # is necessary because at the top of a block, the tokenizer gives
     518      # us comments before the INDENT token; but if we encounter a group
     519      # start marker at the top of a block, then we want it to apply
     520      # inside that block, not outside it.
     521      start_group = None
     522
     523      # Check if the source file declares an encoding.
     524      encoding = get_module_encoding(module_doc.filename)
     525
     526      # The token-eating loop:
     527      try:
     528          module_file = codecs.open(module_doc.filename, 'rU', encoding)
     529      except LookupError:
     530          log.warning("Unknown encoding %r for %s; using the default"
     531                      "encoding instead (iso-8859-1)" %
     532                      (encoding, module_doc.filename))
     533          encoding = 'iso-8859-1'
     534          module_file = codecs.open(module_doc.filename, 'rU', encoding)
     535      tok_iter = tokenize.generate_tokens(module_file.readline)
     536      for toktype, toktext, (srow,scol), (erow,ecol), line_str in tok_iter:
     537          # BOM encoding marker: ignore.
     538          if (toktype == token.ERRORTOKEN and
     539              (toktext == u'\ufeff' or
     540               toktext.encode(encoding) == '\xef\xbb\xbf')):
     541              pass
     542
     543          # Error token: abort
     544          elif toktype == token.ERRORTOKEN:
     545              raise ParseError('Error during parsing: invalid syntax '
     546                               '(%s, line %d, char %d: %r)' %
     547                               (module_doc.filename, srow, scol, toktext))
     548
     549          # Indent token: update the parent_doc stack.
     550          elif toktype == token.INDENT:
     551              if prev_line_doc is None:
     552                  parent_docs.append(parent_docs[-1])
     553              else:
     554                  parent_docs.append(prev_line_doc)
     555              groups.append(None)
     556
     557          # Dedent token: update the parent_doc stack.
     558          elif toktype == token.DEDENT:
     559              if line_toks == []:
     560                  parent_docs.pop()
     561                  groups.pop()
     562              else:
     563                  # This *should* only happen if the file ends on an
     564                  # indented line, with no final newline.
     565                  # (otherwise, this is the wrong thing to do.)
     566                  pass
     567
     568          # Line-internal newline token: if we're still at the start of
     569          # the logical line, and we've seen one or more comment lines,
     570          # then discard them: blank lines are not allowed between a
     571          # comment block and the thing it describes.
     572          elif toktype == tokenize.NL:
     573              if comments and not line_toks:
     574                  log.warning('Ignoring docstring comment block followed by '
     575                              'a blank line in %r on line %r' %
     576                              (module_doc.filename, srow-1))
     577                  comments = []
     578
     579          # Comment token: add to comments if appropriate.
     580          elif toktype == tokenize.COMMENT:
     581              if toktext.startswith(COMMENT_DOCSTRING_MARKER):
     582                  comment_line = toktext[len(COMMENT_DOCSTRING_MARKER):].rstrip()
     583                  if comment_line.startswith(" "):
     584                      comment_line = comment_line[1:]
     585                  comments.append( [comment_line, srow])
     586              elif toktext.startswith(START_GROUP_MARKER):
     587                  start_group = toktext[len(START_GROUP_MARKER):].strip()
     588              elif toktext.startswith(END_GROUP_MARKER):
     589                  for i in range(len(groups)-1, -1, -1):
     590                      if groups[i]:
     591                          groups[i] = None
     592                          break
     593                  else:
     594                      log.warning("Got group end marker without a corresponding "
     595                                  "start marker in %r on line %r" %
     596                                  (module_doc.filename, srow))
     597
     598          # Normal token: Add it to line_toks.  (If it's a non-unicode
     599          # string literal, then we need to re-encode using the file's
     600          # encoding, to get back to the original 8-bit data; and then
     601          # convert that string with 8-bit data to a 7-bit ascii
     602          # representation.)
     603          elif toktype != token.NEWLINE and toktype != token.ENDMARKER:
     604              if lineno is None: lineno = srow
     605              if toktype == token.STRING:
     606                  str_prefixes = re.match('[^\'"]*', toktext).group()
     607                  if 'u' not in str_prefixes:
     608                      s = toktext.encode(encoding)
     609                      toktext = decode_with_backslashreplace(s)
     610              line_toks.append( (toktype, toktext) )
     611
     612          # Decorator line: add it to the decorators list.
     613          elif line_toks and line_toks[0] == (token.OP, '@'):
     614              decorators.append(shallow_parse(line_toks))
     615              line_toks = []
     616
     617          # End of line token, but nothing to do.
     618          elif line_toks == []:
     619              pass
     620
     621          # End of line token: parse the logical line & process it.
     622          else:
     623              if start_group:
     624                  groups[-1] = start_group
     625                  start_group = None
     626
     627              if parent_docs[-1] != 'skip_block':
     628                  try:
     629                      prev_line_doc = process_line(
     630                          shallow_parse(line_toks), parent_docs, prev_line_doc,
     631                          lineno, comments, decorators, encoding)
     632                  except ParseError, e:
     633                      raise ParseError('Error during parsing: invalid '
     634                                       'syntax (%s, line %d) -- %s' %
     635                                       (module_doc.filename, lineno, e))
     636                  except KeyboardInterrupt, e: raise
     637                  except Exception, e:
     638                      log.error('Internal error during parsing (%s, line '
     639                                '%s):\n%s' % (module_doc.filename, lineno, e))
     640                      raise
     641
     642                  # grouping...
     643                  if groups[-1] and prev_line_doc not in (None, 'skip_block'):
     644                      if isinstance(prev_line_doc, VariableDoc):
     645                          # prev_line_doc's container will only be
     646                          # UNKNOWN if it's an instance variable that
     647                          # didn't have a doc-comment, but might still
     648                          # be followed by a docstring.  Since we
     649                          # tokenize in order, we can't do lookahead to
     650                          # see if the variable will have a comment; but
     651                          # it should only be added to the container if
     652                          # it does.  So we defer the grouping of that
     653                          # to be handled by process_docstring instead.
     654                          if prev_line_doc.container is not UNKNOWN:
     655                              add_to_group(prev_line_doc.container,
     656                                           prev_line_doc, groups[-1])
     657                      elif isinstance(parent_docs[-1], NamespaceDoc):
     658                          add_to_group(parent_docs[-1], prev_line_doc,
     659                                       groups[-1])
     660              else:
     661                  prev_line_doc = None
     662
     663              # Reset line contents.
     664              line_toks = []
     665              lineno = None
     666              comments = []
     667              decorators = []
     668
     669 -def add_to_group(container, api_doc, group_name):
     670      if container.group_specs is UNKNOWN:
     671          container.group_specs = []
     672
     673      if isinstance(api_doc, VariableDoc):
     674          var_name = api_doc.name
     675      else:
     676          if api_doc.canonical_name is UNKNOWN: log.debug('ouch', `api_doc`)
     677          var_name = api_doc.canonical_name[-1]
     678
     679      for (name, group_vars) in container.group_specs:
     680          if name == group_name:
     681              group_vars.append(var_name)
     682              return
     683      else:
     684          container.group_specs.append( (group_name, [var_name]) )
     685
     686 -def script_guard(line):
     687      """Detect the idiomatic trick C{if __name__ == "__main__":}"""
     688      return (len(line) == 5
     689          and line[1][1] == '__name__' # this is the most selective
     690          and line[0][1] == 'if'
     691          and line[2][1] == '=='
     692          and line[4][1] == ':'
     693          and line[3][1][1:-1] == '__main__')
     694
     695  #/////////////////////////////////////////////////////////////////
     696  #{ Shallow parser
     697  #/////////////////////////////////////////////////////////////////
     698
     699 -def shallow_parse(line_toks):
     700      """
     701      Given a flat list of tokens, return a nested tree structure
     702      (called a X{token tree}), whose leaves are identical to the
     703      original list, but whose structure reflects the structure
     704      implied by the grouping tokens (i.e., parenthases, braces, and
     705      brackets).  If the parenthases, braces, and brackets do not
     706      match, or are not balanced, then raise a ParseError.
     707
     708      Assign some structure to a sequence of structure (group parens).
     709      """
     710      stack = [[]]
     711      parens = []
     712      for tok in line_toks:
     713          toktype, toktext = tok
     714          if toktext in ('(','[','{'):
     715              parens.append(tok)
     716              stack.append([tok])
     717          elif toktext in ('}',']',')'):
     718              if not parens:
     719                  raise ParseError('Unbalanced parens')
     720              left_paren = parens.pop()[1]
     721              if left_paren+toktext not in ('()', '[]', '{}'):
     722                  raise ParseError('Mismatched parens')
     723              lst = stack.pop()
     724              lst.append(tok)
     725              stack[-1].append(lst)
     726          else:
     727              stack[-1].append(tok)
     728      if len(stack) != 1 or len(parens) != 0:
     729          raise ParseError('Unbalanced parens')
     730      return stack[0]
     731
     732  #/////////////////////////////////////////////////////////////////
     733  #{ Line processing
     734  #/////////////////////////////////////////////////////////////////
     735  # The methods process_*() are used to handle lines.
     736
     737 -def process_line(line, parent_docs, prev_line_doc, lineno,
     738                   comments, decorators, encoding):
     739      """
     740      @return: C{new-doc}, C{decorator}..?
     741      """
     742      args = (line, parent_docs, prev_line_doc, lineno,
     743              comments, decorators, encoding)
     744
     745      if not line: # blank line.
     746          return None
     747      elif (token.OP, ':') in line[:-1]:
     748          return process_one_line_block(*args)
     749      elif (token.OP, ';') in line:
     750          return process_multi_stmt(*args)
     751      elif line[0] == (token.NAME, 'def'):
     752          return process_funcdef(*args)
     753      elif line[0] == (token.OP, '@'):
     754          return process_funcdef(*args)
     755      elif line[0] == (token.NAME, 'class'):
     756          return process_classdef(*args)
     757      elif line[0] == (token.NAME, 'import'):
     758          return process_import(*args)
     759      elif line[0] == (token.NAME, 'from'):
     760          return process_from_import(*args)
     761      elif line[0] == (token.NAME, 'del'):
     762          return process_del(*args)
     763      elif len(line)==1 and line[0][0] == token.STRING:
     764          return process_docstring(*args)
     765      elif (token.OP, '=') in line:
     766          return process_assignment(*args)
     767      elif (line[0][0] == token.NAME and
     768            line[0][1] in CONTROL_FLOW_KEYWORDS):
     769          return process_control_flow_line(*args)
     770      else:
     771          return None
     772          # [xx] do something with control structures like for/if?
     773
     774  #/////////////////////////////////////////////////////////////////
     775  # Line handler: control flow
     776  #/////////////////////////////////////////////////////////////////
     777
     778  CONTROL_FLOW_KEYWORDS = [
     779      #: A list of the control flow keywords.  If a line begins with
     780      #: one of these keywords, then it should be handled by
     781      #: C{process_control_flow_line}.
     782      'if', 'elif', 'else', 'while', 'for', 'try', 'except', 'finally']
     783
     784 -def process_control_flow_line(line, parent_docs, prev_line_doc,
     785                                lineno, comments, decorators, encoding):
     786      keyword = line[0][1]
     787
     788      # If it's a 'for' block: create the loop variable.
     789      if keyword == 'for' and PARSE_FOR_BLOCKS:
     790          loopvar_name = parse_dotted_name(
     791              split_on(line[1:], (token.NAME, 'in'))[0])
     792          parent = get_lhs_parent(loopvar_name, parent_docs)
     793          if parent is not None:
     794              var_doc = VariableDoc(name=loopvar_name[-1], is_alias=False,
     795                                    is_imported=False, is_instvar=False,
     796                                    docs_extracted_by='parser')
     797              set_variable(parent, var_doc)
     798
     799      if ((keyword == 'if' and PARSE_IF_BLOCKS and not script_guard(line)) or
     800          (keyword == 'elif' and PARSE_ELSE_BLOCKS) or
     801          (keyword == 'else' and PARSE_ELSE_BLOCKS) or
     802          (keyword == 'while' and PARSE_WHILE_BLOCKS) or
     803          (keyword == 'for' and PARSE_FOR_BLOCKS) or
     804          (keyword == 'try' and PARSE_TRY_BLOCKS) or
     805          (keyword == 'except' and PARSE_EXCEPT_BLOCKS) or
     806          (keyword == 'finally' and PARSE_FINALLY_BLOCKS)):
     807          # Return "None" to indicate that we should process the
     808          # block using the same context that we were already in.
     809          return None
     810      else:
     811          # Return 'skip_block' to indicate that we should ignore
     812          # the contents of this block.
     813          return 'skip_block'
     814
     815  #/////////////////////////////////////////////////////////////////
     816  # Line handler: imports
     817  #/////////////////////////////////////////////////////////////////
     818  # [xx] I could optionally add ValueDoc's for the imported
     819  # variables with proxy_for set to the imported source; but
     820  # I don't think I gain much of anything by doing so.
     821
     822 -def process_import(line, parent_docs, prev_line_doc, lineno,
     823                     comments, decorators, encoding):
     824      if not isinstance(parent_docs[-1], NamespaceDoc): return
     825
     826      names = split_on(line[1:], (token.OP, ','))
     827
     828      for name in names:
     829          name_pieces = split_on(name, (token.NAME, 'as'))
     830          if len(name_pieces) == 1:
     831              src_name = parse_dotted_name(name_pieces[0])
     832              _import_var(src_name, parent_docs)
     833          elif len(name_pieces) == 2:
     834              if len(name_pieces[1]) != 1:
     835                  raise ParseError('Expected identifier after "as"')
     836              src_name = parse_dotted_name(name_pieces[0])
     837              var_name = parse_name(name_pieces[1][0])
     838              _import_var_as(src_name, var_name, parent_docs)
     839          else:
     840              raise ParseError('Multiple "as" tokens in import')
     841
     842 -def process_from_import(line, parent_docs, prev_line_doc, lineno,
     843                          comments, decorators, encoding):
     844      if not isinstance(parent_docs[-1], NamespaceDoc): return
     845
     846      pieces = split_on(line[1:], (token.NAME, 'import'))
     847      if len(pieces) != 2 or not pieces[0] or not pieces[1]:
     848          raise ParseError("Bad from-import")
     849      lhs, rhs = pieces
     850
     851      # The RHS might be parenthasized, as specified by PEP 328:
     852      # http://www.python.org/peps/pep-0328.html
     853      if (len(rhs) == 1 and isinstance(rhs[0], list) and
     854          rhs[0][0] == (token.OP, '(') and rhs[0][-1] == (token.OP, ')')):
     855          rhs = rhs[0][1:-1]
     856
     857      # >>> from __future__ import nested_scopes
     858      if lhs == [(token.NAME, '__future__')]:
     859          return
     860
     861      # >>> from sys import *
     862      elif rhs == [(token.OP, '*')]:
     863          src_name = parse_dotted_name(lhs)
     864          _process_fromstar_import(src_name, parent_docs)
     865
     866      # >>> from os.path import join, split
     867      else:
     868          # Allow relative imports in this case, as per PEP 328
     869          src_name = parse_dotted_name(lhs,
     870              parent_name=parent_docs[-1].canonical_name)
     871          parts = split_on(rhs, (token.OP, ','))
     872          for part in parts:
     873              # from m import x
     874              if len(part) == 1:
     875                  var_name = parse_name(part[0])
     876                  _import_var_as(DottedName(src_name, var_name),
     877                                      var_name, parent_docs)
     878
     879              # from m import x as y
     880              elif len(part) == 3 and part[1] == (token.NAME, 'as'):
     881                  orig_name = parse_name(part[0])
     882                  var_name = parse_name(part[2])
     883                  _import_var_as(DottedName(src_name, orig_name),
     884                                      var_name, parent_docs)
     885
     886              else:
     887                  ParseError("Bad from-import")
     888
     889 -def _process_fromstar_import(src, parent_docs):
     890      """
     891      Handle a statement of the form:
     892          >>> from <src> import *
     893
     894      If L{IMPORT_HANDLING} is C{'parse'}, then first try to parse
     895      the module C{M{<src>}}, and copy all of its exported variables
     896      to C{parent_docs[-1]}.
     897
     898      Otherwise, try to determine the names of the variables exported by
     899      C{M{<src>}}, and create a new variable for each export.  If
     900      L{IMPORT_STAR_HANDLING} is C{'parse'}, then the list of exports if
     901      found by parsing C{M{<src>}}; if it is C{'introspect'}, then the
     902      list of exports is found by importing and introspecting
     903      C{M{<src>}}.
     904      """
     905      # This is redundant: already checked by caller.
     906      if not isinstance(parent_docs[-1], NamespaceDoc): return
     907
     908      # If src is package-local, then convert it to a global name.
     909      src = _global_name(src, parent_docs)
     910
     911      # Record the import
     912      parent_docs[0].imports.append(src) # mark that it's .*??
     913
     914      # [xx] add check for if we already have the source docs in our
     915      # cache??
     916
     917      if (IMPORT_HANDLING == 'parse' or
     918          IMPORT_STAR_HANDLING == 'parse'): # [xx] is this ok?
     919          try: module_doc = _find(src)
     920          except ImportError: module_doc = None
     921          if isinstance(module_doc, ModuleDoc):
     922              for name, imp_var in module_doc.variables.items():
     923                  # [xx] this is not exactly correct, but close.  It
     924                  # does the wrong thing if a __var__ is explicitly
     925                  # listed in __all__.
     926                  if (imp_var.is_public and
     927                      not (name.startswith('__') and name.endswith('__'))):
     928                      var_doc = _add_import_var(DottedName(src, name), name,
     929                                                parent_docs[-1])
     930                      if IMPORT_HANDLING == 'parse':
     931                          var_doc.value = imp_var.value
     932
     933      # If we got here, then either IMPORT_HANDLING='link' or we
     934      # failed to parse the `src` module.
     935      if IMPORT_STAR_HANDLING == 'introspect':
     936          try: module = __import__(str(src), {}, {}, [0])
     937          except: return # We couldn't import it.
     938          if module is None: return # We couldn't import it.
     939          if hasattr(module, '__all__'):
     940              names = list(module.__all__)
     941          else:
     942              names = [n for n in dir(module) if not n.startswith('_')]
     943          for name in names:
     944              _add_import_var(DottedName(src, name), name, parent_docs[-1])
     945
     946 -def _import_var(name, parent_docs):
     947      """
     948      Handle a statement of the form:
     949          >>> import <name>
     950
     951      If L{IMPORT_HANDLING} is C{'parse'}, then first try to find
     952      the value by parsing; and create an appropriate variable in
     953      parentdoc.
     954
     955      Otherwise, add a variable for the imported variable.  (More than
     956      one variable may be created for cases like C{'import a.b'}, where
     957      we need to create a variable C{'a'} in parentdoc containing a
     958      proxy module; and a variable C{'b'} in the proxy module.
     959      """
     960      # This is redundant: already checked by caller.
     961      if not isinstance(parent_docs[-1], NamespaceDoc): return
     962
     963      # If name is package-local, then convert it to a global name.
     964      src = _global_name(name, parent_docs)
     965      src_prefix = src[:len(src)-len(name)]
     966
     967      # Record the import
     968      parent_docs[0].imports.append(name)
     969
     970      # [xx] add check for if we already have the source docs in our
     971      # cache??
     972
     973      if IMPORT_HANDLING == 'parse':
     974          # Check to make sure that we can actually find the value.
     975          try: val_doc = _find(src)
     976          except ImportError: val_doc = None
     977          if val_doc is not None:
     978              # We found it; but it's not the value itself we want to
     979              # import, but the module containing it; so import that
     980              # module (=top_mod) and create a variable for it.
     981              top_mod = src_prefix+name[0]
     982              var_doc = _add_import_var(top_mod, name[0], parent_docs[-1])
     983              var_doc.value = _find(DottedName(name[0]))
     984              return
     985
     986      # If we got here, then either IMPORT_HANDLING='link', or we
     987      # did not successfully find the value's docs by parsing; use
     988      # a variable with an UNKNOWN value.
     989
     990      # Create any necessary intermediate proxy module values.
     991      container = parent_docs[-1]
     992      for i, identifier in enumerate(name[:-1]):
     993          if (identifier not in container.variables or
     994              not isinstance(container.variables[identifier], ModuleDoc)):
     995              var_doc = _add_import_var(name[:i+1], identifier, container)
     996              var_doc.value = ModuleDoc(variables={}, sort_spec=[],
     997                                        proxy_for=src_prefix+name[:i+1],
     998                                        submodules={},
     999                                        docs_extracted_by='parser')
    1000          container = container.variables[identifier].value
    1001
    1002      # Add the variable to the container.
    1003      _add_import_var(src, name[-1], container)
    1004
    1005 -def _import_var_as(src, name, parent_docs):
    1006      """
    1007      Handle a statement of the form:
    1008          >>> import src as name
    1009
    1010      If L{IMPORT_HANDLING} is C{'parse'}, then first try to find
    1011      the value by parsing; and create an appropriate variable in
    1012      parentdoc.
    1013
    1014      Otherwise, create a variables with its C{imported_from} attribute
    1015      pointing to the imported object.
    1016      """
    1017      # This is redundant: already checked by caller.
    1018      if not isinstance(parent_docs[-1], NamespaceDoc): return
    1019
    1020      # If src is package-local, then convert it to a global name.
    1021      src = _global_name(src, parent_docs)
    1022
    1023      # Record the import
    1024      parent_docs[0].imports.append(src)
    1025
    1026      if IMPORT_HANDLING == 'parse':
    1027          # Parse the value and create a variable for it.
    1028          try: val_doc = _find(src)
    1029          except ImportError: val_doc = None
    1030          if val_doc is not None:
    1031              var_doc = VariableDoc(name=name, value=val_doc,
    1032                                    is_imported=True, is_alias=False,
    1033                                    imported_from=src,
    1034                                    docs_extracted_by='parser')
    1035              set_variable(parent_docs[-1], var_doc)
    1036              return
    1037
    1038      # If we got here, then either IMPORT_HANDLING='link', or we
    1039      # did not successfully find the value's docs by parsing; use a
    1040      # variable with a proxy value.
    1041      _add_import_var(src, name, parent_docs[-1])
    1042
    1043 -def _add_import_var(src, name, container):
    1044      """
    1045      Add a new imported variable named C{name} to C{container}, with
    1046      C{imported_from=src}.
    1047      """
    1048      var_doc = VariableDoc(name=name, is_imported=True, is_alias=False,
    1049                            imported_from=src, docs_extracted_by='parser')
    1050      set_variable(container, var_doc)
    1051      return var_doc
    1052
    1053 -def _global_name(name, parent_docs):
    1054      """
    1055      If the given name is package-local (relative to the current
    1056      context, as determined by C{parent_docs}), then convert it
    1057      to a global name.
    1058      """
    1059      # Get the containing package from parent_docs.
    1060      if parent_docs[0].is_package:
    1061          package = parent_docs[0]
    1062      else:
    1063          package = parent_docs[0].package
    1064
    1065      # Check each package (from closest to furthest) to see if it
    1066      # contains a module named name[0]; if so, then treat `name` as
    1067      # relative to that package.
    1068      while package not in (None, UNKNOWN):
    1069          try:
    1070              fp = imp.find_module(name[0], package.path)[0]
    1071              if fp is not None: fp.close()
    1072          except ImportError:
    1073              # No submodule found here; try the next package up.
    1074              package = package.package
    1075              continue
    1076          # A submodule was found; return its name.
    1077          return package.canonical_name + name
    1078
    1079      # We didn't find any package containing `name`; so just return
    1080      # `name` as-is.
    1081      return name
    1082
    1083  #/////////////////////////////////////////////////////////////////
    1084  # Line handler: assignment
    1085  #/////////////////////////////////////////////////////////////////
    1086
    1087 -def process_assignment(line, parent_docs, prev_line_doc, lineno,
    1088                         comments, decorators, encoding):
    1089      # Divide the assignment statement into its pieces.
    1090      pieces = split_on(line, (token.OP, '='))
    1091
    1092      lhs_pieces = pieces[:-1]
    1093      rhs = pieces[-1]
    1094
    1095      # Decide whether the variable is an instance variable or not.
    1096      # If it's an instance var, then discard the value.
    1097      is_instvar = lhs_is_instvar(lhs_pieces, parent_docs)
    1098
    1099      # if it's not an instance var, and we're not in a namespace,
    1100      # then it's just a local var -- so ignore it.
    1101      if not (is_instvar or isinstance(parent_docs[-1], NamespaceDoc)):
    1102          return None
    1103
    1104      # Evaluate the right hand side.
    1105      if not is_instvar:
    1106          rhs_val, is_alias = rhs_to_valuedoc(rhs, parent_docs)
    1107      else:
    1108          rhs_val, is_alias = UNKNOWN, False
    1109
    1110      # Assign the right hand side value to each left hand side.
    1111      # (Do the rightmost assignment first)
    1112      lhs_pieces.reverse()
    1113      for lhs in lhs_pieces:
    1114          # Try treating the LHS as a simple dotted name.
    1115          try: lhs_name = parse_dotted_name(lhs)
    1116          except: lhs_name = None
    1117          if lhs_name is not None:
    1118              lhs_parent = get_lhs_parent(lhs_name, parent_docs)
    1119              if lhs_parent is None: continue
    1120
    1121              # Skip a special class variable.
    1122              if lhs_name[-1] == '__slots__':
    1123                  continue
    1124
    1125              # Create the VariableDoc.
    1126              var_doc = VariableDoc(name=lhs_name[-1], value=rhs_val,
    1127                                    is_imported=False, is_alias=is_alias,
    1128                                    is_instvar=is_instvar,
    1129                                    docs_extracted_by='parser')
    1130              # Extract a docstring from the comments, when present,
    1131              # but only if there's a single LHS.
    1132              if len(lhs_pieces) == 1:
    1133                  add_docstring_from_comments(var_doc, comments)
    1134
    1135              # Assign the variable to the containing namespace,
    1136              # *unless* the variable is an instance variable
    1137              # without a comment docstring.  In that case, we'll
    1138              # only want to add it if we later discover that it's
    1139              # followed by a variable docstring.  If it is, then
    1140              # process_docstring will take care of adding it to the
    1141              # containing clas.  (This is a little hackish, but
    1142              # unfortunately is necessary because we won't know if
    1143              # this assignment line is followed by a docstring
    1144              # until later.)
    1145              if (not is_instvar) or comments:
    1146                  set_variable(lhs_parent, var_doc, True)
    1147
    1148              # If it's the only var, then return the VarDoc for use
    1149              # as the new `prev_line_doc`.
    1150              if (len(lhs_pieces) == 1 and
    1151                  (len(lhs_name) == 1 or is_instvar)):
    1152                  return var_doc
    1153
    1154          # Otherwise, the LHS must be a complex expression; use
    1155          # dotted_names_in() to decide what variables it contains,
    1156          # and create VariableDoc's for all of them (with UNKNOWN
    1157          # value).
    1158          else:
    1159              for lhs_name in dotted_names_in(lhs_pieces):
    1160                  lhs_parent = get_lhs_parent(lhs_name, parent_docs)
    1161                  if lhs_parent is None: continue
    1162                  var_doc = VariableDoc(name=lhs_name[-1],
    1163                                        is_imported=False,
    1164                                        is_alias=is_alias,
    1165                                        is_instvar=is_instvar,
    1166                                        docs_extracted_by='parser')
    1167                  set_variable(lhs_parent, var_doc, True)
    1168
    1169          # If we have multiple left-hand-sides, then all but the
    1170          # rightmost one are considered aliases.
    1171          is_alias = True
    1172
    1173
    1174 -def lhs_is_instvar(lhs_pieces, parent_docs):
    1175      if not isinstance(parent_docs[-1], RoutineDoc):
    1176          return False
    1177      # make sure that lhs_pieces is <self>.<name>, where <self> is
    1178      # the name of the first arg to the containing routinedoc, and
    1179      # <name> is a simple name.
    1180      posargs = parent_docs[-1].posargs
    1181      if posargs is UNKNOWN: return False
    1182      if not (len(lhs_pieces)==1 and len(posargs) > 0 and
    1183              len(lhs_pieces[0]) == 3 and
    1184              lhs_pieces[0][0] == (token.NAME, posargs[0]) and
    1185              lhs_pieces[0][1] == (token.OP, '.') and
    1186              lhs_pieces[0][2][0] == token.NAME):
    1187          return False
    1188      # Make sure we're in an instance method, and not a
    1189      # module-level function.
    1190      for i in range(len(parent_docs)-1, -1, -1):
    1191          if isinstance(parent_docs[i], ClassDoc):
    1192              return True
    1193          elif parent_docs[i] != parent_docs[-1]:
    1194              return False
    1195      return False
    1196
    1197 -def rhs_to_valuedoc(rhs, parent_docs):
    1198      # Dotted variable:
    1199      try:
    1200          rhs_name = parse_dotted_name(rhs)
    1201          rhs_val = lookup_value(rhs_name, parent_docs)
    1202          if rhs_val is not None and rhs_val is not UNKNOWN:
    1203              return rhs_val, True
    1204      except ParseError:
    1205          pass
    1206
    1207      # Decorators:
    1208      if (len(rhs)==2 and rhs[0][0] == token.NAME and
    1209          isinstance(rhs[1], list)):
    1210          arg_val, _ = rhs_to_valuedoc(rhs[1][1:-1], parent_docs)
    1211          if isinstance(arg_val, RoutineDoc):
    1212              doc = apply_decorator(DottedName(rhs[0][1]), arg_val)
    1213              doc.canonical_name = UNKNOWN
    1214              doc.parse_repr = pp_toktree(rhs)
    1215              return doc, False
    1216
    1217      # Nothing else to do: make a val with the source as its repr.
    1218      return GenericValueDoc(parse_repr=pp_toktree(rhs), toktree=rhs,
    1219                             defining_module=parent_docs[0],
    1220                             docs_extracted_by='parser'), False
    1221
    1222 -def get_lhs_parent(lhs_name, parent_docs):
    1223      assert isinstance(lhs_name, DottedName)
    1224
    1225      # For instance vars inside an __init__ method:
    1226      if isinstance(parent_docs[-1], RoutineDoc):
    1227          for i in range(len(parent_docs)-1, -1, -1):
    1228              if isinstance(parent_docs[i], ClassDoc):
    1229                  return parent_docs[i]
    1230          else:
    1231              raise ValueError("%r is not a namespace or method" %
    1232                               parent_docs[-1])
    1233
    1234      # For local variables:
    1235      if len(lhs_name) == 1:
    1236          return parent_docs[-1]
    1237
    1238      # For non-local variables:
    1239      return lookup_value(lhs_name.container(), parent_docs)
    1240
    1241  #/////////////////////////////////////////////////////////////////
    1242  # Line handler: single-line blocks
    1243  #/////////////////////////////////////////////////////////////////
    1244
    1245 -def process_one_line_block(line, parent_docs, prev_line_doc, lineno,
    1246                             comments, decorators, encoding):
    1247      """
    1248      The line handler for single-line blocks, such as:
    1249
    1250          >>> def f(x): return x*2
    1251
    1252      This handler calls L{process_line} twice: once for the tokens
    1253      up to and including the colon, and once for the remaining
    1254      tokens.  The comment docstring is applied to the first line
    1255      only.
    1256      @return: C{None}
    1257      """
    1258      i = line.index((token.OP, ':'))
    1259      doc1 = process_line(line[:i+1], parent_docs, prev_line_doc,
    1260                               lineno, comments, decorators, encoding)
    1261      doc2 = process_line(line[i+1:], parent_docs+[doc1],
    1262                               doc1, lineno, None, [], encoding)
    1263      return doc1
    1264
    1265  #/////////////////////////////////////////////////////////////////
    1266  # Line handler: semicolon-separated statements
    1267  #/////////////////////////////////////////////////////////////////
    1268
    1269 -def process_multi_stmt(line, parent_docs, prev_line_doc, lineno,
    1270                         comments, decorators, encoding):
    1271      """
    1272      The line handler for semicolon-separated statements, such as:
    1273
    1274          >>> x=1; y=2; z=3
    1275
    1276      This handler calls L{process_line} once for each statement.
    1277      The comment docstring is not passed on to any of the
    1278      sub-statements.
    1279      @return: C{None}
    1280      """
    1281      for statement in split_on(line, (token.OP, ';')):
    1282          if not statement: continue
    1283          doc = process_line(statement, parent_docs, prev_line_doc,
    1284                             lineno, None, decorators, encoding)
    1285          prev_line_doc = doc
    1286          decorators = []
    1287      return None
    1288
    1289  #/////////////////////////////////////////////////////////////////
    1290  # Line handler: delete statements
    1291  #/////////////////////////////////////////////////////////////////
    1292
    1293 -def process_del(line, parent_docs, prev_line_doc, lineno,
    1294                  comments, decorators, encoding):
    1295      """
    1296      The line handler for delete statements, such as:
    1297
    1298          >>> del x, y.z
    1299
    1300      This handler calls L{del_variable} for each dotted variable in
    1301      the variable list.  The variable list may be nested.  Complex
    1302      expressions in the variable list (such as C{x[3]}) are ignored.
    1303      @return: C{None}
    1304      """
    1305      # If we're not in a namespace, then ignore it.
    1306      parent_doc = parent_docs[-1]
    1307      if not isinstance(parent_doc, NamespaceDoc): return
    1308
    1309      var_list = split_on(line[1:], (token.OP, ','))
    1310      for var_name in dotted_names_in(var_list):
    1311          del_variable(parent_docs[-1], var_name)
    1312
    1313      return None
    1314
    1315  #/////////////////////////////////////////////////////////////////
    1316  # Line handler: docstrings
    1317  #/////////////////////////////////////////////////////////////////
    1318
    1319 -def process_docstring(line, parent_docs, prev_line_doc, lineno,
    1320                        comments, decorators, encoding):
    1321      """
    1322      The line handler for bare string literals.  If
    1323      C{prev_line_doc} is not C{None}, then the string literal is
    1324      added to that C{APIDoc} as a docstring.  If it already has a
    1325      docstring (from comment docstrings), then the new docstring
    1326      will be appended to the old one.
    1327      """
    1328      if prev_line_doc is None: return
    1329      docstring = parse_string(line)
    1330
    1331      # If the docstring is a str, then convert it to unicode.
    1332      # According to a strict reading of PEP 263, this might not be the
    1333      # right thing to do; but it will almost always be what the
    1334      # module's author intended.
    1335      if isinstance(docstring, str):
    1336          try:
    1337              docstring = docstring.decode(encoding)
    1338          except UnicodeDecodeError:
    1339              # If decoding failed, then fall back on using
    1340              # decode_with_backslashreplace, which will map e.g.
    1341              # "\xe9" -> u"\\xe9".
    1342              docstring = decode_with_backslashreplace(docstring)
    1343              log.warning("While parsing %s: docstring is not a unicode "
    1344                          "string, but it contains non-ascii data." %
    1345                          prev_line_doc.canonical_name)
    1346
    1347      # If the modified APIDoc is an instance variable, and it has
    1348      # not yet been added to its class's C{variables} list,
    1349      # then add it now.  This is done here, rather than in the
    1350      # process_assignment() call that created the variable, because
    1351      # we only want to add instance variables if they have an
    1352      # associated docstring.  (For more info, see the comment above
    1353      # the set_variable() call in process_assignment().)
    1354      added_instvar = False
    1355      if (isinstance(prev_line_doc, VariableDoc) and
    1356           prev_line_doc.is_instvar and
    1357           prev_line_doc.docstring in (None, UNKNOWN)):
    1358          for i in range(len(parent_docs)-1, -1, -1):
    1359              if isinstance(parent_docs[i], ClassDoc):
    1360                  set_variable(parent_docs[i], prev_line_doc, True)
    1361                  added_instvar = True
    1362                  break
    1363
    1364      if prev_line_doc.docstring not in (None, UNKNOWN):
    1365          log.warning("%s has both a comment-docstring and a normal "
    1366                      "(string) docstring; ignoring the comment-"
    1367                      "docstring." % prev_line_doc.canonical_name)
    1368
    1369      prev_line_doc.docstring = docstring
    1370      prev_line_doc.docstring_lineno = lineno
    1371
    1372      # If the modified APIDoc is an instance variable, and we added it
    1373      # to the class's variables list here, then it still needs to be
    1374      # grouped too; so return it for use as the new "prev_line_doc."
    1375      if added_instvar:
    1376          return prev_line_doc
    1377
    1378
    1379  #/////////////////////////////////////////////////////////////////
    1380  # Line handler: function declarations
    1381  #/////////////////////////////////////////////////////////////////
    1382
    1383 -def process_funcdef(line, parent_docs, prev_line_doc, lineno,
    1384                      comments, decorators, encoding):
    1385      """
    1386      The line handler for function declaration lines, such as:
    1387
    1388          >>> def f(a, b=22, (c,d)):
    1389
    1390      This handler creates and initializes a new C{VariableDoc}
    1391      containing a C{RoutineDoc}, adds the C{VariableDoc} to the
    1392      containing namespace, and returns the C{RoutineDoc}.
    1393      """
    1394      # Check syntax.
    1395      if len(line) != 4 or line[3] != (token.OP, ':'):
    1396          raise ParseError("Bad function definition line")
    1397
    1398      # If we're not in a namespace, then ignore it.
    1399      parent_doc = parent_docs[-1]
    1400      if not isinstance(parent_doc, NamespaceDoc): return
    1401
    1402      # Get the function's name
    1403      func_name = parse_name(line[1])
    1404      canonical_name = DottedName(parent_doc.canonical_name, func_name)
    1405
    1406      # Create the function's RoutineDoc.
    1407      func_doc = RoutineDoc(canonical_name=canonical_name,
    1408                            defining_module=parent_docs[0],
    1409                            lineno=lineno, docs_extracted_by='parser')
    1410
    1411      # Process the signature.
    1412      init_arglist(func_doc, line[2])
    1413
    1414      # If the preceeding comment includes a docstring, then add it.
    1415      add_docstring_from_comments(func_doc, comments)
    1416
    1417      # Apply any decorators.
    1418      func_doc.decorators = [pp_toktree(deco[1:]) for deco in decorators]
    1419      decorators.reverse()
    1420      for decorator in decorators:
    1421          try:
    1422              deco_name = parse_dotted_name(decorator[1:])
    1423          except ParseError:
    1424              deco_name = None
    1425          if func_doc.canonical_name is not UNKNOWN:
    1426              deco_repr = '%s(%s)' % (pp_toktree(decorator[1:]),
    1427                                      func_doc.canonical_name)
    1428          elif func_doc.parse_repr not in (None, UNKNOWN):
    1429              # [xx] this case should be improved.. when will func_doc
    1430              # have a known parse_repr??
    1431              deco_repr = '%s(%s)' % (pp_toktree(decorator[1:]),
    1432                                      func_doc.parse_repr)
    1433          else:
    1434              deco_repr = UNKNOWN
    1435          func_doc = apply_decorator(deco_name, func_doc)
    1436          func_doc.parse_repr = deco_repr
    1437          # [XX] Is there a reson the following should be done?  It
    1438          # causes the grouping code to break.  Presumably the canonical
    1439          # name should remain valid if we're just applying a standard
    1440          # decorator.
    1441          #func_doc.canonical_name = UNKNOWN
    1442
    1443      # Add a variable to the containing namespace.
    1444      var_doc = VariableDoc(name=func_name, value=func_doc,
    1445                            is_imported=False, is_alias=False,
    1446                            docs_extracted_by='parser')
    1447      set_variable(parent_doc, var_doc)
    1448
    1449      # Return the new ValueDoc.
    1450      return func_doc
    1451
    1452 -def apply_decorator(decorator_name, func_doc):
    1453      # [xx] what if func_doc is not a RoutineDoc?
    1454      if decorator_name == DottedName('staticmethod'):
    1455          return StaticMethodDoc(**func_doc.__dict__)
    1456      elif decorator_name == DottedName('classmethod'):
    1457          return ClassMethodDoc(**func_doc.__dict__)
    1458      elif DEFAULT_DECORATOR_BEHAVIOR == 'transparent':
    1459          return func_doc.__class__(**func_doc.__dict__) # make a copy.
    1460      elif DEFAULT_DECORATOR_BEHAVIOR == 'opaque':
    1461          return GenericValueDoc(docs_extracted_by='parser')
    1462      else:
    1463          raise ValueError, 'Bad value for DEFAULT_DECORATOR_BEHAVIOR'
    1464
    1465 -def init_arglist(func_doc, arglist):
    1466      if not isinstance(arglist, list) or arglist[0] != (token.OP, '('):
    1467          raise ParseError("Bad argument list")
    1468
    1469      # Initialize to defaults.
    1470      func_doc.posargs = []
    1471      func_doc.posarg_defaults = []
    1472      func_doc.vararg = None
    1473      func_doc.kwarg = None
    1474
    1475      # Divide the arglist into individual args.
    1476      args = split_on(arglist[1:-1], (token.OP, ','))
    1477
    1478      # Keyword argument.
    1479      if args and args[-1][0] == (token.OP, '**'):
    1480          if len(args[-1]) != 2 or args[-1][1][0] != token.NAME:
    1481              raise ParseError("Expected name after ** in argument list")
    1482          func_doc.kwarg = args[-1][1][1]
    1483          args.pop()
    1484
    1485      # Vararg argument.
    1486      if args and args[-1][0] == (token.OP, '*'):
    1487          if len(args[-1]) != 2 or args[-1][1][0] != token.NAME:
    1488              raise ParseError("Expected name after * in argument list")
    1489          func_doc.vararg = args[-1][1][1]
    1490          args.pop()
    1491
    1492      # Positional arguments.
    1493      for arg in args:
    1494          func_doc.posargs.append(parse_funcdef_arg(arg[0]))
    1495          if len(arg) == 1:
    1496              func_doc.posarg_defaults.append(None)
    1497          elif arg[1] != (token.OP, '=') or len(arg) == 2:
    1498              raise ParseError("Bad argument list")
    1499          else:
    1500              default_repr = pp_toktree(arg[2:], 'tight')
    1501              default_val = GenericValueDoc(parse_repr=default_repr,
    1502                                            docs_extracted_by='parser')
    1503              func_doc.posarg_defaults.append(default_val)
    1504
    1505  #/////////////////////////////////////////////////////////////////
    1506  # Line handler: class declarations
    1507  #/////////////////////////////////////////////////////////////////
    1508
    1509 -def process_classdef(line, parent_docs, prev_line_doc, lineno,
    1510                       comments, decorators, encoding):
    1511      """
    1512      The line handler for class declaration lines, such as:
    1513
    1514          >>> class Foo(Bar, Baz):
    1515
    1516      This handler creates and initializes a new C{VariableDoc}
    1517      containing a C{ClassDoc}, adds the C{VariableDoc} to the
    1518      containing namespace, and returns the C{ClassDoc}.
    1519      """
    1520      # Check syntax
    1521      if len(line)<3 or len(line)>4 or line[-1] != (token.OP, ':'):
    1522          raise ParseError("Bad class definition line")
    1523
    1524      # If we're not in a namespace, then ignore it.
    1525      parent_doc = parent_docs[-1]
    1526      if not isinstance(parent_doc, NamespaceDoc): return
    1527
    1528      # Get the class's name
    1529      class_name = parse_name(line[1])
    1530      canonical_name = DottedName(parent_doc.canonical_name, class_name)
    1531
    1532      # Create the class's ClassDoc & VariableDoc.
    1533      class_doc = ClassDoc(variables={}, sort_spec=[],
    1534                           bases=[], subclasses=[],
    1535                           canonical_name=canonical_name,
    1536                           defining_module=parent_docs[0],
    1537                           docs_extracted_by='parser')
    1538      var_doc = VariableDoc(name=class_name, value=class_doc,
    1539                            is_imported=False, is_alias=False,
    1540                            docs_extracted_by='parser')
    1541
    1542      # Add the bases.
    1543      if len(line) == 4:
    1544          if (not isinstance(line[2], list) or
    1545              line[2][0] != (token.OP, '(')):
    1546              raise ParseError("Expected base list")
    1547          try:
    1548              for base_name in parse_classdef_bases(line[2]):
    1549                  class_doc.bases.append(find_base(base_name, parent_docs))
    1550          except ParseError, e:
    1551              log.warning("Unable to extract the base list for %s: %s" %
    1552                          (canonical_name, e))
    1553              class_doc.bases = UNKNOWN
    1554      else:
    1555          class_doc.bases = []
    1556
    1557      # Register ourselves as a subclass to our bases.
    1558      if class_doc.bases is not UNKNOWN:
    1559          for basedoc in class_doc.bases:
    1560              if isinstance(basedoc, ClassDoc):
    1561                  # This test avoids that a subclass gets listed twice when
    1562                  # both introspection and parsing.
    1563                  # [XXX] This check only works because currently parsing is
    1564                  # always performed just after introspection of the same
    1565                  # class. A more complete fix shuld be independent from
    1566                  # calling order; probably the subclasses list should be
    1567                  # replaced by a ClassDoc set or a {name: ClassDoc} mapping.
    1568                  if (basedoc.subclasses
    1569                      and basedoc.subclasses[-1].canonical_name
    1570                          != class_doc.canonical_name):
    1571                      basedoc.subclasses.append(class_doc)
    1572
    1573      # If the preceeding comment includes a docstring, then add it.
    1574      add_docstring_from_comments(class_doc, comments)
    1575
    1576      # Add the VariableDoc to our container.
    1577      set_variable(parent_doc, var_doc)
    1578
    1579      return class_doc
    1580
    1581 -def _proxy_base(**attribs):
    1582      return ClassDoc(variables={}, sort_spec=[], bases=[], subclasses=[],
    1583                      docs_extracted_by='parser', **attribs)
    1584
    1585 -def find_base(name, parent_docs):
    1586      assert isinstance(name, DottedName)
    1587
    1588      # Find the variable containing the base.
    1589      base_var = lookup_variable(name, parent_docs)
    1590      if base_var is None:
    1591          # If we didn't find it, then it must have been imported.
    1592          # First, check if it looks like it's contained in any
    1593          # known imported variable:
    1594          if len(name) > 1:
    1595              src = lookup_name(name[0], parent_docs)
    1596              if (src is not None and
    1597                  src.imported_from not in (None, UNKNOWN)):
    1598                  base_src = DottedName(src.imported_from, name[1:])
    1599                  base_var = VariableDoc(name=name[-1], is_imported=True,
    1600                                         is_alias=False, imported_from=base_src,
    1601                                         docs_extracted_by='parser')
    1602          # Otherwise, it must have come from an "import *" statement
    1603          # (or from magic, such as direct manipulation of the module's
    1604          # dictionary), so we don't know where it came from.  So
    1605          # there's nothing left but to use an empty proxy.
    1606          if base_var is None:
    1607              return _proxy_base(parse_repr=str(name))
    1608              #raise ParseError("Could not find %s" % name)
    1609
    1610      # If the variable has a value, return that value.
    1611      if base_var.value is not UNKNOWN:
    1612          return base_var.value
    1613
    1614      # Otherwise, if BASE_HANDLING is 'parse', try parsing the docs for
    1615      # the base class; if that fails, or if BASE_HANDLING is 'link',
    1616      # just make a proxy object.
    1617      if base_var.imported_from not in (None, UNKNOWN):
    1618          if BASE_HANDLING == 'parse':
    1619              old_sys_path = sys.path
    1620              try:
    1621                  dirname = os.path.split(parent_docs[0].filename)[0]
    1622                  sys.path = [dirname] + sys.path
    1623                  try:
    1624                      return parse_docs(name=str(base_var.imported_from))
    1625                  except ParseError:
    1626                      log.info('Unable to parse base', base_var.imported_from)
    1627                  except ImportError:
    1628                      log.info('Unable to find base', base_var.imported_from)
    1629              finally:
    1630                  sys.path = old_sys_path
    1631
    1632          # Either BASE_HANDLING='link' or parsing the base class failed;
    1633          # return a proxy value for the base class.
    1634          return _proxy_base(proxy_for=base_var.imported_from)
    1635      else:
    1636          return _proxy_base(parse_repr=str(name))
    1637
    1638  #/////////////////////////////////////////////////////////////////
    1639  #{ Parsing
    1640  #/////////////////////////////////////////////////////////////////
    1641
    1642 -def dotted_names_in(elt_list):
    1643      """
    1644      Return a list of all simple dotted names in the given
    1645      expression.
    1646      """
    1647      names = []
    1648      while elt_list:
    1649          elt = elt_list.pop()
    1650          if len(elt) == 1 and isinstance(elt[0], list):
    1651              # Nested list: process the contents
    1652              elt_list.extend(split_on(elt[0][1:-1], (token.OP, ',')))
    1653          else:
    1654              try:
    1655                  names.append(parse_dotted_name(elt))
    1656              except ParseError:
    1657                  pass # complex expression -- ignore
    1658      return names
    1659
    1660 -def parse_name(elt, strip_parens=False):
    1661      """
    1662      If the given token tree element is a name token, then return
    1663      that name as a string.  Otherwise, raise ParseError.
    1664      @param strip_parens: If true, then if elt is a single name
    1665          enclosed in parenthases, then return that name.
    1666      """
    1667      if strip_parens and isinstance(elt, list):
    1668          while (isinstance(elt, list) and len(elt) == 3 and
    1669                 elt[0] == (token.OP, '(') and
    1670                 elt[-1] == (token.OP, ')')):
    1671              elt = elt[1]
    1672      if isinstance(elt, list) or elt[0] != token.NAME:
    1673          raise ParseError("Bad name")
    1674      return elt[1]
    1675
    1676 -def parse_dotted_name(elt_list, strip_parens=True, parent_name=None):
    1677      """
    1678      @param parent_name: canonical name of referring module, to resolve
    1679          relative imports.
    1680      @type parent_name: L{DottedName}
    1681      @bug: does not handle 'x.(y).z'
    1682      """
    1683      if len(elt_list) == 0: raise ParseError("Bad dotted name")
    1684
    1685      # Handle ((x.y).z).  (If the contents of the parens include
    1686      # anything other than dotted names, such as (x,y), then we'll
    1687      # catch it below and raise a ParseError.
    1688      while (isinstance(elt_list[0], list) and
    1689             len(elt_list[0]) >= 3 and
    1690             elt_list[0][0] == (token.OP, '(') and
    1691             elt_list[0][-1] == (token.OP, ')')):
    1692          elt_list[:1] = elt_list[0][1:-1]
    1693
    1694      # Convert a relative import into an absolute name.
    1695      prefix_name = None
    1696      if parent_name is not None and elt_list[0][-1] == '.':
    1697          items = 1
    1698          while len(elt_list) > items and elt_list[items][-1] == '.':
    1699              items += 1
    1700
    1701          elt_list = elt_list[items:]
    1702          prefix_name = parent_name[:-items]
    1703
    1704          # >>> from . import foo
    1705          if not elt_list:
    1706              if prefix_name == []:
    1707                  raise ParseError("Attempted relative import in non-package, "
    1708                                   "or beyond toplevel package")
    1709              return prefix_name
    1710
    1711      if len(elt_list) % 2 != 1: raise ParseError("Bad dotted name")
    1712      name = DottedName(parse_name(elt_list[0], True))
    1713      if prefix_name is not None:
    1714          name = prefix_name + name
    1715
    1716      for i in range(2, len(elt_list), 2):
    1717          dot, identifier = elt_list[i-1], elt_list[i]
    1718          if  dot != (token.OP, '.'):
    1719              raise ParseError("Bad dotted name")
    1720          name = DottedName(name, parse_name(identifier, True))
    1721      return name
    1722
    1723 -def split_on(elt_list, split_tok):
    1724      # [xx] add code to guarantee each elt is non-empty.
    1725      result = [[]]
    1726      for elt in elt_list:
    1727          if elt == split_tok:
    1728              if result[-1] == []: raise ParseError("Empty element from split")
    1729              result.append([])
    1730          else:
    1731              result[-1].append(elt)
    1732      if result[-1] == []: result.pop()
    1733      return result
    1734
    1735 -def parse_funcdef_arg(elt):
    1736      """
    1737      If the given tree token element contains a valid function
    1738      definition argument (i.e., an identifier token or nested list
    1739      of identifiers), then return a corresponding string identifier
    1740      or nested list of string identifiers.  Otherwise, raise a
    1741      ParseError.
    1742      """
    1743      if isinstance(elt, list):
    1744          if elt[0] == (token.OP, '('):
    1745              if len(elt) == 3:
    1746                  return parse_funcdef_arg(elt[1])
    1747              else:
    1748                  return [parse_funcdef_arg(e)
    1749                          for e in elt[1:-1]
    1750                          if e != (token.OP, ',')]
    1751          else:
    1752              raise ParseError("Bad argument -- expected name or tuple")
    1753      elif elt[0] == token.NAME:
    1754          return elt[1]
    1755      else:
    1756          raise ParseError("Bad argument -- expected name or tuple")
    1757
    1758 -def parse_classdef_bases(elt):
    1759      """
    1760      If the given tree token element contains a valid base list
    1761      (that contains only dotted names), then return a corresponding
    1762      list of L{DottedName}s.  Otherwise, raise a ParseError.
    1763
    1764      @bug: Does not handle either of::
    1765          - class A( (base.in.parens) ): pass
    1766          - class B( (lambda:calculated.base)() ): pass
    1767      """
    1768      if (not isinstance(elt, list) or
    1769          elt[0] != (token.OP, '(')):
    1770          raise ParseError("Bad base list")
    1771
    1772      return [parse_dotted_name(n)
    1773              for n in split_on(elt[1:-1], (token.OP, ','))]
    1774
    1775  # Used by: base list; 'del'; ...
    1776 -def parse_dotted_name_list(elt_list):
    1777      """
    1778      If the given list of tree token elements contains a
    1779      comma-separated list of dotted names, then return a
    1780      corresponding list of L{DottedName} objects.  Otherwise, raise
    1781      ParseError.
    1782      """
    1783      names = []
    1784
    1785      state = 0
    1786      for elt in elt_list:
    1787          # State 0 -- Expecting a name, or end of arglist
    1788          if state == 0:
    1789              # Make sure it's a name
    1790              if isinstance(elt, tuple) and elt[0] == token.NAME:
    1791                  names.append(DottedName(elt[1]))
    1792                  state = 1
    1793              else:
    1794                  raise ParseError("Expected a name")
    1795          # State 1 -- Expecting comma, period, or end of arglist
    1796          elif state == 1:
    1797              if elt == (token.OP, '.'):
    1798                  state = 2
    1799              elif elt == (token.OP, ','):
    1800                  state = 0
    1801              else:
    1802                  raise ParseError("Expected '.' or ',' or end of list")
    1803          # State 2 -- Continuation of dotted name.
    1804          elif state == 2:
    1805              if isinstance(elt, tuple) and elt[0] == token.NAME:
    1806                  names[-1] = DottedName(names[-1], elt[1])
    1807                  state = 1
    1808              else:
    1809                  raise ParseError("Expected a name")
    1810      if state == 2:
    1811          raise ParseError("Expected a name")
    1812      return names
    1813
    1814 -def parse_string(elt_list):
    1815      if len(elt_list) == 1 and elt_list[0][0] == token.STRING:
    1816          # [xx] use something safer here?  But it needs to deal with
    1817          # any string type (eg r"foo\bar" etc).
    1818          return eval(elt_list[0][1])
    1819      else:
    1820          raise ParseError("Expected a string")
    1821
    1822  # ['1', 'b', 'c']
    1823 -def parse_string_list(elt_list):
    1824      if (len(elt_list) == 1 and isinstance(elt_list, list) and
    1825          elt_list[0][0][1] in ('(', '[')):
    1826          elt_list = elt_list[0][1:-1]
    1827
    1828      string_list = []
    1829      for string_elt in split_on(elt_list, (token.OP, ',')):
    1830          string_list.append(parse_string(string_elt))
    1831
    1832      return string_list
    1833
    1834  #/////////////////////////////////////////////////////////////////
    1835  #{ Variable Manipulation
    1836  #/////////////////////////////////////////////////////////////////
    1837
    1838 -def set_variable(namespace, var_doc, preserve_docstring=False):
    1839      """
    1840      Add var_doc to namespace.  If namespace already contains a
    1841      variable with the same name, then discard the old variable.  If
    1842      C{preserve_docstring} is true, then keep the old variable's
    1843      docstring when overwriting a variable.
    1844      """
    1845      # Choose which dictionary we'll be storing the variable in.
    1846      if not isinstance(namespace, NamespaceDoc):
    1847          return
    1848
    1849      # This happens when the class definition has not been parsed, e.g. in
    1850      # sf bug #1693253 on ``Exception.x = y``
    1851      if namespace.sort_spec is UNKNOWN:
    1852          namespace.sort_spec = namespace.variables.keys()
    1853
    1854      # If we already have a variable with this name, then remove the
    1855      # old VariableDoc from the sort_spec list; and if we gave its
    1856      # value a canonical name, then delete it.
    1857      if var_doc.name in namespace.variables:
    1858          namespace.sort_spec.remove(var_doc.name)
    1859          old_var_doc = namespace.variables[var_doc.name]
    1860          if (old_var_doc.is_alias == False and
    1861              old_var_doc.value is not UNKNOWN):
    1862              old_var_doc.value.canonical_name = UNKNOWN
    1863          if (preserve_docstring and var_doc.docstring in (None, UNKNOWN) and
    1864              old_var_doc.docstring not in (None, UNKNOWN)):
    1865              var_doc.docstring = old_var_doc.docstring
    1866              var_doc.docstring_lineno = old_var_doc.docstring_lineno
    1867      # Add the variable to the namespace.
    1868      namespace.variables[var_doc.name] = var_doc
    1869      namespace.sort_spec.append(var_doc.name)
    1870      assert var_doc.container is UNKNOWN
    1871      var_doc.container = namespace
    1872
    1873 -def del_variable(namespace, name):
    1874      if not isinstance(namespace, NamespaceDoc):
    1875          return
    1876
    1877      if name[0] in namespace.variables:
    1878          if len(name) == 1:
    1879              var_doc = namespace.variables[name[0]]
    1880              namespace.sort_spec.remove(name[0])
    1881              del namespace.variables[name[0]]
    1882              if not var_doc.is_alias and var_doc.value is not UNKNOWN:
    1883                  var_doc.value.canonical_name = UNKNOWN
    1884          else:
    1885              del_variable(namespace.variables[name[0]].value, name[1:])
    1886
    1887  #/////////////////////////////////////////////////////////////////
    1888  #{ Name Lookup
    1889  #/////////////////////////////////////////////////////////////////
    1890
    1891 -def lookup_name(identifier, parent_docs):
    1892      """
    1893      Find and return the documentation for the variable named by
    1894      the given identifier.
    1895
    1896      @rtype: L{VariableDoc} or C{None}
    1897      """
    1898      # We need to check 3 namespaces: locals, globals, and builtins.
    1899      # Note that this is true even if we're in a version of python with
    1900      # nested scopes, because nested scope lookup does not apply to
    1901      # nested class definitions, and we're not worried about variables
    1902      # in nested functions.
    1903      if not isinstance(identifier, basestring):
    1904          raise TypeError('identifier must be a string')
    1905
    1906      # Locals
    1907      if isinstance(parent_docs[-1], NamespaceDoc):
    1908          if identifier in parent_docs[-1].variables:
    1909              return parent_docs[-1].variables[identifier]
    1910
    1911      # Globals (aka the containing module)
    1912      if isinstance(parent_docs[0], NamespaceDoc):
    1913          if identifier in parent_docs[0].variables:
    1914              return parent_docs[0].variables[identifier]
    1915
    1916      # Builtins
    1917      builtins = epydoc.docintrospecter.introspect_docs(__builtin__)
    1918      if isinstance(builtins, NamespaceDoc):
    1919          if identifier in builtins.variables:
    1920              return builtins.variables[identifier]
    1921
    1922      # We didn't find it; return None.
    1923      return None
    1924
    1925 -def lookup_variable(dotted_name, parent_docs):
    1926      assert isinstance(dotted_name, DottedName)
    1927      # If it's a simple identifier, use lookup_name.
    1928      if len(dotted_name) == 1:
    1929          return lookup_name(dotted_name[0], parent_docs)
    1930
    1931      # If it's a dotted name with multiple pieces, look up the
    1932      # namespace containing the var (=parent) first; and then
    1933      # look for the var in that namespace.
    1934      else:
    1935          parent = lookup_value(dotted_name[:-1], parent_docs)
    1936          if (isinstance(parent, NamespaceDoc) and
    1937              dotted_name[-1] in parent.variables):
    1938              return parent.variables[dotted_name[-1]]
    1939          else:
    1940              return None # var not found.
    1941
    1942 -def lookup_value(dotted_name, parent_docs):
    1943      """
    1944      Find and return the documentation for the value contained in
    1945      the variable with the given name in the current namespace.
    1946      """
    1947      assert isinstance(dotted_name, DottedName)
    1948      var_doc = lookup_name(dotted_name[0], parent_docs)
    1949
    1950      for i in range(1, len(dotted_name)):
    1951          if var_doc is None: return None
    1952
    1953          if isinstance(var_doc.value, NamespaceDoc):
    1954              var_dict = var_doc.value.variables
    1955          elif (var_doc.value is UNKNOWN and
    1956              var_doc.imported_from not in (None, UNKNOWN)):
    1957              src_name = var_doc.imported_from + dotted_name[i:]
    1958              # [xx] do I want to create a proxy here??
    1959              return GenericValueDoc(proxy_for=src_name,
    1960                                     parse_repr=str(dotted_name),
    1961                                     docs_extracted_by='parser')
    1962          else:
    1963              return None
    1964
    1965          var_doc = var_dict.get(dotted_name[i])
    1966
    1967      if var_doc is None: return None
    1968      return var_doc.value
    1969
    1970  #/////////////////////////////////////////////////////////////////
    1971  #{ Docstring Comments
    1972  #/////////////////////////////////////////////////////////////////
    1973
    1974 -def add_docstring_from_comments(api_doc, comments):
    1975      if api_doc is None or not comments: return
    1976      api_doc.docstring = '\n'.join([line for (line, lineno) in comments])
    1977      api_doc.docstring_lineno = comments[0][1]
    1978
    1979  #/////////////////////////////////////////////////////////////////
    1980  #{ Tree tokens
    1981  #/////////////////////////////////////////////////////////////////
    1982
    1983 -def _join_toktree(s1, s2):
    1984      # Join them.  s1 = left side; s2 = right side.
    1985      if (s2=='' or s1=='' or
    1986          s1 in ('-','`') or s2 in ('}',']',')','`',':') or
    1987          s2[0] in ('.',',') or s1[-1] in ('(','[','{','.','\n',' ') or
    1988          (s2[0] == '(' and s1[-1] not in (',','='))):
    1989          return '%s%s' % (s1,s2)
    1990      elif (spacing=='tight' and
    1991            s1[-1] in '+-*/=,' or s2[0] in '+-*/=,'):
    1992          return '%s%s' % (s1, s2)
    1993      else:
    1994          return '%s %s' % (s1, s2)
    1995
    1996 -def _pp_toktree_add_piece(spacing, pieces, piece):
    1997      s1 = pieces[-1]
    1998      s2 = piece
    1999
    2000      if (s2=='' or s1=='' or
    2001          s1 in ('-','`') or s2 in ('}',']',')','`',':') or
    2002          s2[0] in ('.',',') or s1[-1] in ('(','[','{','.','\n',' ') or
    2003          (s2[0] == '(' and s1[-1] not in (',','='))):
    2004          pass
    2005      elif (spacing=='tight' and
    2006            s1[-1] in '+-*/=,' or s2[0] in '+-*/=,'):
    2007          pass
    2008      else:
    2009          pieces.append(' ')
    2010
    2011      pieces.append(piece)
    2012
    2013 -def pp_toktree(elts, spacing='normal', indent=0):
    2014      pieces = ['']
    2015      _pp_toktree(elts, spacing, indent, pieces)
    2016      return ''.join(pieces)
    2017
    2018 -def _pp_toktree(elts, spacing, indent, pieces):
    2019      add_piece = _pp_toktree_add_piece
    2020
    2021      for elt in elts:
    2022          # Put a blank line before class & def statements.
    2023          if elt == (token.NAME, 'class') or elt == (token.NAME, 'def'):
    2024              add_piece(spacing, pieces, '\n%s' % ('    '*indent))
    2025
    2026          if isinstance(elt, tuple):
    2027              if elt[0] == token.NEWLINE:
    2028                  add_piece(spacing, pieces, '    '+elt[1])
    2029                  add_piece(spacing, pieces, '\n%s' % ('    '*indent))
    2030              elif elt[0] == token.INDENT:
    2031                  add_piece(spacing, pieces, '    ')
    2032                  indent += 1
    2033              elif elt[0] == token.DEDENT:
    2034                  assert pieces[-1] == '    '
    2035                  pieces.pop()
    2036                  indent -= 1
    2037              elif elt[0] == tokenize.COMMENT:
    2038                  add_piece(spacing, pieces, elt[1].rstrip() + '\n')
    2039                  add_piece('    '*indent)
    2040              else:
    2041                  add_piece(spacing, pieces, elt[1])
    2042          else:
    2043              _pp_toktree(elt, spacing, indent, pieces)
    2044
    2045  #/////////////////////////////////////////////////////////////////
    2046  #{ Helper Functions
    2047  #/////////////////////////////////////////////////////////////////
    2048
    2049 -def get_module_encoding(filename):
    2050      """
    2051      @see: U{PEP 263<http://www.python.org/peps/pep-0263.html>}
    2052      """
    2053      module_file = open(filename, 'rU')
    2054      try:
    2055          lines = [module_file.readline() for i in range(2)]
    2056          if lines[0].startswith('\xef\xbb\xbf'):
    2057              return 'utf-8'
    2058          else:
    2059              for line in lines:
    2060                  m = re.search("coding[:=]\s*([-\w.]+)", line)
    2061                  if m: return m.group(1)
    2062
    2063          # Fall back on Python's default encoding.
    2064          return 'iso-8859-1' # aka 'latin-1'
    2065      finally:
    2066          module_file.close()
    2067
    2068 -def _get_module_name(filename, package_doc):
    2069      """
    2070      Return (dotted_name, is_package)
    2071      """
    2072      name = re.sub(r'.py\w?$', '', os.path.split(filename)[1])
    2073      if name == '__init__':
    2074          is_package = True
    2075          name = os.path.split(os.path.split(filename)[0])[1]
    2076      else:
    2077          is_package = False
    2078
    2079      # [XX] if the module contains a script, then `name` may not
    2080      # necessarily be a valid identifier -- which will cause
    2081      # DottedName to raise an exception.  Is that what I want?
    2082      if package_doc is None:
    2083          dotted_name = DottedName(name)
    2084      else:
    2085          dotted_name = DottedName(package_doc.canonical_name, name)
    2086
    2087      # Check if the module looks like it's shadowed by a variable.
    2088      # If so, then add a "'" to the end of its canonical name, to
    2089      # distinguish it from the variable.
    2090      if package_doc is not None and name in package_doc.variables:
    2091          vardoc = package_doc.variables[name]
    2092          if (vardoc.value not in (None, UNKNOWN) and
    2093              vardoc.imported_from != dotted_name):
    2094              log.warning("Module %s might be shadowed by a variable with "
    2095                          "the same name." % dotted_name)
    2096              dotted_name = DottedName(str(dotted_name)+"'")
    2097
    2098      return dotted_name, is_package
    2099
    2100 -def flatten(lst, out=None):
    2101      """
    2102      @return: a flat list containing the leaves of the given nested
    2103          list.
    2104      @param lst: The nested list that should be flattened.
    2105      """
    2106      if out is None: out = []
    2107      for elt in lst:
    2108          if isinstance(elt, (list, tuple)):
    2109              flatten(elt, out)
    2110          else:
    2111              out.append(elt)
    2112      return out
    2113