Package epydoc :: Module docbuilder

[hide private]
[frames] | no frames]

# Source Code for Module epydoc.docbuilder

       1  # epydoc -- Documentation Builder
       2  #
       3  # Copyright (C) 2005 Edward Loper
       4  # Author: Edward Loper <edloper@loper.org>
       5  # URL: <http://epydoc.sf.net>
       6  #
       7  # $Id: docbuilder.py 1683 2008-01-29 22:17:39Z edloper $
       8
       9  """
      10  Construct data structures that encode the API documentation for Python
      11  objects.  These data structures are created using a series of steps:
      12
      13    1. B{Building docs}: Extract basic information about the objects,
      14       and objects that are related to them.  This can be done by
      15       introspecting the objects' values (with L{epydoc.docintrospecter}; or
      16       by parsing their source code (with L{epydoc.docparser}.
      17
      18    2. B{Merging}: Combine the information obtained from introspection &
      19       parsing each object into a single structure.
      20
      21    3. B{Linking}: Replace any 'pointers' that were created for imported
      22       variables by their target (if it's available).
      23
      24    4. B{Naming}: Chose a unique 'canonical name' for each
      25       object.
      26
      27    5. B{Docstring Parsing}: Parse the docstring of each object, and
      28       extract any pertinant information.
      29
      30    6. B{Inheritance}: Add information about variables that classes
      31       inherit from their base classes.
      32
      33  The documentation information for each individual object is
      34  represented using an L{APIDoc}; and the documentation for a collection
      35  of objects is represented using a L{DocIndex}.
      36
      37  The main interface to C{epydoc.docbuilder} consists of two functions:
      38
      39    - L{build_doc()} -- Builds documentation for a single item, and
      40      returns it as an L{APIDoc} object.
      41    - L{build_doc_index()} -- Builds documentation for a collection of
      42      items, and returns it as a L{DocIndex} object.
      43
      44  The remaining functions are used by these two main functions to
      45  perform individual steps in the creation of the documentation.
      46
      47  @group Documentation Construction: build_doc, build_doc_index,
      48      _get_docs_from_*, _report_valdoc_progress
      49  @group Merging: *MERGE*, *merge*
      50  @group Linking: link_imports
      51  @group Naming: _name_scores, _unreachable_names, assign_canonical_names,
      52      _var_shadows_self, _fix_self_shadowing_var, _unreachable_name_for
      53  @group Inheritance: inherit_docs, _inherit_info
      54  """
      55  __docformat__ = 'epytext en'
      56
      57  ######################################################################
      58  ## Contents
      59  ######################################################################
      60  ## 1. build_doc() & build_doc_index() -- the main interface.
      61  ## 2. merge_docs() -- helper, used to merge parse & introspect info
      62  ## 3. link_imports() -- helper, used to connect imported vars w/ values
      63  ## 4. assign_canonical_names() -- helper, used to set canonical names
      64  ## 5. inherit_docs() -- helper, used to inherit docs from base classes
      65
      66  ######################################################################
      67  ## Imports
      68  ######################################################################
      69
      70  import sys, os, os.path, __builtin__, imp, re, inspect
      71  from epydoc.apidoc import *
      72  from epydoc.docintrospecter import introspect_docs
      73  from epydoc.docparser import parse_docs, ParseError
      74  from epydoc.docstringparser import parse_docstring
      75  from epydoc import log
      76  from epydoc.util import *
      77  from epydoc.compat import * # Backwards compatibility
      78
      79  ######################################################################
      80  ## 1. build_doc()
      81  ######################################################################
      82
      83 -class BuildOptions:
      84      """
      85      Holds the parameters for a documentation building process.
      86      """
      87 -    def __init__(self, introspect=True, parse=True,
      88                   exclude_introspect=None, exclude_parse=None,
      89                   add_submodules=True):
      90          self.introspect = introspect
      91          self.parse = parse
      92          self.exclude_introspect = exclude_introspect
      93          self.exclude_parse = exclude_parse
      94          self.add_submodules = add_submodules
      95
      96          # Test for pattern syntax and compile them into pattern objects.
      97          try:
      98              self._introspect_regexp = (exclude_introspect
      99                  and re.compile(exclude_introspect) or None)
     100              self._parse_regexp = (exclude_parse
     101                  and re.compile(exclude_parse) or None)
     102          except Exception, exc:
     103              log.error('Error in regular expression pattern: %s' % exc)
     104              raise
     105
     106 -    def must_introspect(self, name):
     107          """
     108          Return C{True} if a module is to be introsepcted with the current
     109          settings.
     110
     111          @param name: The name of the module to test
     112          @type name: L{DottedName} or C{str}
     113          """
     114          return self.introspect \
     115              and not self._matches_filter(name, self._introspect_regexp)
     116
     117 -    def must_parse(self, name):
     118          """
     119          Return C{True} if a module is to be parsed with the current settings.
     120
     121          @param name: The name of the module to test
     122          @type name: L{DottedName} or C{str}
     123          """
     124          return self.parse \
     125              and not self._matches_filter(name, self._parse_regexp)
     126
     127 -    def _matches_filter(self, name, regexp):
     128          """
     129          Test if a module name matches a pattern.
     130
     131          @param name: The name of the module to test
     132          @type name: L{DottedName} or C{str}
     133          @param regexp: The pattern object to match C{name} against.
     134              If C{None}, return C{False}
     135          @type regexp: C{pattern}
     136          @return: C{True} if C{name} in dotted format matches C{regexp},
     137              else C{False}
     138          @rtype: C{bool}
     139          """
     140          if regexp is None: return False
     141
     142          if isinstance(name, DottedName):
     143              name = str(name)
     144
     145          return bool(regexp.search(name))
     146
     147
     148 -def build_doc(item, introspect=True, parse=True, add_submodules=True,
     149                exclude_introspect=None, exclude_parse=None):
     150      """
     151      Build API documentation for a given item, and return it as
     152      an L{APIDoc} object.
     153
     154      @rtype: L{APIDoc}
     155      @param item: The item to document, specified using any of the
     156          following:
     157            - A string, naming a python package directory
     158              (e.g., C{'epydoc/markup'})
     159            - A string, naming a python file
     160              (e.g., C{'epydoc/docparser.py'})
     161            - A string, naming a python object
     162              (e.g., C{'epydoc.docparser.DocParser'})
     163            - Any (non-string) python object
     164              (e.g., C{list.append})
     165      @param introspect: If true, then use introspection to examine the
     166          specified items.  Otherwise, just use parsing.
     167      @param parse: If true, then use parsing to examine the specified
     168          items.  Otherwise, just use introspection.
     169      """
     170      docindex = build_doc_index([item], introspect, parse, add_submodules,
     171                                 exclude_introspect=exclude_introspect,
     172                                 exclude_parse=exclude_parse)
     173      return docindex.root[0]
     174
     175 -def build_doc_index(items, introspect=True, parse=True, add_submodules=True,
     176                      exclude_introspect=None, exclude_parse=None):
     177      """
     178      Build API documentation for the given list of items, and
     179      return it in the form of a L{DocIndex}.
     180
     181      @rtype: L{DocIndex}
     182      @param items: The items to document, specified using any of the
     183          following:
     184            - A string, naming a python package directory
     185              (e.g., C{'epydoc/markup'})
     186            - A string, naming a python file
     187              (e.g., C{'epydoc/docparser.py'})
     188            - A string, naming a python object
     189              (e.g., C{'epydoc.docparser.DocParser'})
     190            - Any (non-string) python object
     191              (e.g., C{list.append})
     192      @param introspect: If true, then use introspection to examine the
     193          specified items.  Otherwise, just use parsing.
     194      @param parse: If true, then use parsing to examine the specified
     195          items.  Otherwise, just use introspection.
     196      """
     197      try:
     198          options = BuildOptions(parse=parse, introspect=introspect,
     199              exclude_introspect=exclude_introspect, exclude_parse=exclude_parse,
     200              add_submodules=add_submodules)
     201      except Exception, e:
     202          # log.error already reported by constructor.
     203          return None
     204
     205      # Get the basic docs for each item.
     206      doc_pairs = _get_docs_from_items(items, options)
     207
     208      # Merge the introspection & parse docs.
     209      if options.parse and options.introspect:
     210          log.start_progress('Merging parsed & introspected information')
     211          docs = []
     212          for i, (introspect_doc, parse_doc) in enumerate(doc_pairs):
     213              if introspect_doc is not None and parse_doc is not None:
     214                  if introspect_doc.canonical_name not in (None, UNKNOWN):
     215                      name = introspect_doc.canonical_name
     216                  else:
     217                      name = parse_doc.canonical_name
     218                  log.progress(float(i)/len(doc_pairs), name)
     219                  docs.append(merge_docs(introspect_doc, parse_doc))
     220              elif introspect_doc is not None:
     221                  docs.append(introspect_doc)
     222              elif parse_doc is not None:
     223                  docs.append(parse_doc)
     224          log.end_progress()
     225      elif options.introspect:
     226          docs = [doc_pair[0] for doc_pair in doc_pairs if doc_pair[0]]
     227      else:
     228          docs = [doc_pair[1] for doc_pair in doc_pairs if doc_pair[1]]
     229
     230      if len(docs) == 0:
     231          log.error('Nothing left to document!')
     232          return None
     233
     234      # Collect the docs into a single index.
     235      docindex = DocIndex(docs)
     236
     237      # Replace any proxy valuedocs that we got from importing with
     238      # their targets.
     239      if options.parse:
     240          log.start_progress('Linking imported variables')
     241          valdocs = sorted(docindex.reachable_valdocs(
     242              imports=False, submodules=False, packages=False, subclasses=False))
     243          for i, val_doc in enumerate(valdocs):
     244              _report_valdoc_progress(i, val_doc, valdocs)
     245              link_imports(val_doc, docindex)
     246          log.end_progress()
     247
     248      # Assign canonical names.
     249      log.start_progress('Indexing documentation')
     250      for i, val_doc in enumerate(docindex.root):
     251          log.progress(float(i)/len(docindex.root), val_doc.canonical_name)
     252          assign_canonical_names(val_doc, val_doc.canonical_name, docindex)
     253      log.end_progress()
     254
     255      # Set overrides pointers
     256      log.start_progress('Checking for overridden methods')
     257      valdocs = sorted(docindex.reachable_valdocs(
     258          imports=False, submodules=False, packages=False, subclasses=False))
     259      for i, val_doc in enumerate(valdocs):
     260          if isinstance(val_doc, ClassDoc):
     261              percent = float(i)/len(valdocs)
     262              log.progress(percent, val_doc.canonical_name)
     263              find_overrides(val_doc)
     264      log.end_progress()
     265
     266      # Parse the docstrings for each object.
     267      log.start_progress('Parsing docstrings')
     268      suppress_warnings = set(valdocs).difference(
     269          docindex.reachable_valdocs(
     270              imports=False, submodules=False, packages=False, subclasses=False,
     271              bases=False, overrides=True))
     272      for i, val_doc in enumerate(valdocs):
     273          _report_valdoc_progress(i, val_doc, valdocs)
     274          # the value's docstring
     275          parse_docstring(val_doc, docindex, suppress_warnings)
     276          # the value's variables' docstrings
     277          if (isinstance(val_doc, NamespaceDoc) and
     278              val_doc.variables not in (None, UNKNOWN)):
     279              for var_doc in val_doc.variables.values():
     280                  # Now we have a chance to propagate the defining module
     281                  # to objects for which introspection is not possible,
     282                  # such as properties.
     283                  if (isinstance(var_doc.value, ValueDoc)
     284                      and var_doc.value.defining_module is UNKNOWN):
     285                      var_doc.value.defining_module = val_doc.defining_module
     286                  parse_docstring(var_doc, docindex, suppress_warnings)
     287      log.end_progress()
     288
     289      # Take care of inheritance.
     290      log.start_progress('Inheriting documentation')
     291      for i, val_doc in enumerate(valdocs):
     292          if isinstance(val_doc, ClassDoc):
     293              percent = float(i)/len(valdocs)
     294              log.progress(percent, val_doc.canonical_name)
     295              inherit_docs(val_doc)
     296      log.end_progress()
     297
     298      # Initialize the groups & sortedvars attributes.
     299      log.start_progress('Sorting & Grouping')
     300      for i, val_doc in enumerate(valdocs):
     301          if isinstance(val_doc, NamespaceDoc):
     302              percent = float(i)/len(valdocs)
     303              log.progress(percent, val_doc.canonical_name)
     304              val_doc.init_sorted_variables()
     305              val_doc.init_variable_groups()
     306              if isinstance(val_doc, ModuleDoc):
     307                  val_doc.init_submodule_groups()
     308              val_doc.report_unused_groups()
     309      log.end_progress()
     310
     311      return docindex
     312
     313 -def _report_valdoc_progress(i, val_doc, val_docs):
     314      if (isinstance(val_doc, (ModuleDoc, ClassDoc)) and
     315          val_doc.canonical_name is not UNKNOWN and
     316          not val_doc.canonical_name[0].startswith('??')):
     317          log.progress(float(i)/len(val_docs), val_doc.canonical_name)
     318
     319  #/////////////////////////////////////////////////////////////////
     320  # Documentation Generation
     321  #/////////////////////////////////////////////////////////////////
     322
     323 -def _get_docs_from_items(items, options):
     324
     325      # Start the progress bar.
     326      log.start_progress('Building documentation')
     327      progress_estimator = _ProgressEstimator(items)
     328
     329      # Check for duplicate item names.
     330      item_set = set()
     331      for item in items[:]:
     332          if item in item_set:
     333              log.warning("Name %r given multiple times" % item)
     334              items.remove(item)
     335          item_set.add(item)
     336
     337      # Keep track of what top-level canonical names we've assigned, to
     338      # make sure there are no naming conflicts.  This dict maps
     339      # canonical names to the item names they came from (so we can print
     340      # useful error messages).
     341      canonical_names = {}
     342
     343      # Collect (introspectdoc, parsedoc) pairs for each item.
     344      doc_pairs = []
     345      for item in items:
     346          if isinstance(item, basestring):
     347              if is_module_file(item):
     348                  doc_pairs.append(_get_docs_from_module_file(
     349                      item, options, progress_estimator))
     350              elif is_package_dir(item):
     351                  pkgfile = os.path.abspath(os.path.join(item, '__init__'))
     352                  doc_pairs.append(_get_docs_from_module_file(
     353                      pkgfile, options, progress_estimator))
     354              elif os.path.isfile(item):
     355                  doc_pairs.append(_get_docs_from_pyscript(
     356                      item, options, progress_estimator))
     357              elif hasattr(__builtin__, item):
     358                  val = getattr(__builtin__, item)
     359                  doc_pairs.append(_get_docs_from_pyobject(
     360                      val, options, progress_estimator))
     361              elif is_pyname(item):
     362                  doc_pairs.append(_get_docs_from_pyname(
     363                      item, options, progress_estimator))
     364              elif os.path.isdir(item):
     365                  log.error("Directory %r is not a package" % item)
     366                  continue
     367              elif os.path.isfile(item):
     368                  log.error("File %s is not a Python module" % item)
     369                  continue
     370              else:
     371                  log.error("Could not find a file or object named %s" %
     372                            item)
     373                  continue
     374          else:
     375              doc_pairs.append(_get_docs_from_pyobject(
     376                  item, options, progress_estimator))
     377
     378          # Make sure there are no naming conflicts.
     379          name = (getattr(doc_pairs[-1][0], 'canonical_name', None) or
     380                  getattr(doc_pairs[-1][1], 'canonical_name', None))
     381          if name in canonical_names:
     382              log.error(
     383                  'Two of the specified items, %r and %r, have the same '
     384                  'canonical name ("%s").  This may mean that you specified '
     385                  'two different files that both use the same module name.  '
     386                  'Ignoring the second item (%r)' %
     387                  (canonical_names[name], item, name, canonical_names[name]))
     388              doc_pairs.pop()
     389          else:
     390              canonical_names[name] = item
     391
     392          # This will only have an effect if doc_pairs[-1] contains a
     393          # package's docs.  The 'not is_module_file(item)' prevents
     394          # us from adding subdirectories if they explicitly specify
     395          # a package's __init__.py file.
     396          if options.add_submodules and not is_module_file(item):
     397              doc_pairs += _get_docs_from_submodules(
     398                  item, doc_pairs[-1], options, progress_estimator)
     399
     400      log.end_progress()
     401      return doc_pairs
     402
     403 -def _get_docs_from_pyobject(obj, options, progress_estimator):
     404      progress_estimator.complete += 1
     405      log.progress(progress_estimator.progress(), repr(obj))
     406
     407      if not options.introspect:
     408          log.error("Cannot get docs for Python objects without "
     409                    "introspecting them.")
     410
     411      introspect_doc = parse_doc = None
     412      introspect_error = parse_error = None
     413      try:
     414          introspect_doc = introspect_docs(value=obj)
     415      except ImportError, e:
     416          log.error(e)
     417          return (None, None)
     418      if options.parse:
     419          if introspect_doc.canonical_name is not None:
     420              prev_introspect = options.introspect
     421              options.introspect = False
     422              try:
     423                  _, parse_docs = _get_docs_from_pyname(
     424                      str(introspect_doc.canonical_name), options,
     425                      progress_estimator, suppress_warnings=True)
     426              finally:
     427                  options.introspect = prev_introspect
     428
     429      # We need a name:
     430      if introspect_doc.canonical_name in (None, UNKNOWN):
     431          if hasattr(obj, '__name__'):
     432              introspect_doc.canonical_name = DottedName(
     433                  DottedName.UNREACHABLE, obj.__name__)
     434          else:
     435              introspect_doc.canonical_name = DottedName(
     436                  DottedName.UNREACHABLE)
     437      return (introspect_doc, parse_doc)
     438
     439 -def _get_docs_from_pyname(name, options, progress_estimator,
     440                            suppress_warnings=False):
     441      progress_estimator.complete += 1
     442      if options.must_introspect(name) or options.must_parse(name):
     443          log.progress(progress_estimator.progress(), name)
     444
     445      introspect_doc = parse_doc = None
     446      introspect_error = parse_error = None
     447      if options.must_introspect(name):
     448          try:
     449              introspect_doc = introspect_docs(name=name)
     450          except ImportError, e:
     451              introspect_error = str(e)
     452      if options.must_parse(name):
     453          try:
     454              parse_doc = parse_docs(name=name)
     455          except ParseError, e:
     456              parse_error = str(e)
     457          except ImportError, e:
     458              # If we get here, then there' probably no python source
     459              # available; don't bother to generate a warnining.
     460              pass
     461
     462      # Report any errors we encountered.
     463      if not suppress_warnings:
     464          _report_errors(name, introspect_doc, parse_doc,
     465                         introspect_error, parse_error)
     466
     467      # Return the docs we found.
     468      return (introspect_doc, parse_doc)
     469
     470 -def _get_docs_from_pyscript(filename, options, progress_estimator):
     471      # [xx] I should be careful about what names I allow as filenames,
     472      # and maybe do some munging to prevent problems.
     473
     474      introspect_doc = parse_doc = None
     475      introspect_error = parse_error = None
     476      if options.introspect:
     477          try:
     478              introspect_doc = introspect_docs(filename=filename, is_script=True)
     479              if introspect_doc.canonical_name is UNKNOWN:
     480                  introspect_doc.canonical_name = munge_script_name(filename)
     481          except ImportError, e:
     482              introspect_error = str(e)
     483      if options.parse:
     484          try:
     485              parse_doc = parse_docs(filename=filename, is_script=True)
     486          except ParseError, e:
     487              parse_error = str(e)
     488          except ImportError, e:
     489              parse_error = str(e)
     490
     491      # Report any errors we encountered.
     492      _report_errors(filename, introspect_doc, parse_doc,
     493                     introspect_error, parse_error)
     494
     495      # Return the docs we found.
     496      return (introspect_doc, parse_doc)
     497
     498 -def _get_docs_from_module_file(filename, options, progress_estimator,
     499                                 parent_docs=(None,None)):
     500      """
     501      Construct and return the API documentation for the python
     502      module with the given filename.
     503
     504      @param parent_docs: The C{ModuleDoc} of the containing package.
     505          If C{parent_docs} is not provided, then this method will
     506          check if the given filename is contained in a package; and
     507          if so, it will construct a stub C{ModuleDoc} for the
     508          containing package(s).  C{parent_docs} is a tuple, where
     509          the first element is the parent from introspection, and
     510          the second element is the parent from parsing.
     511      """
     512      # Record our progress.
     513      modulename = os.path.splitext(os.path.split(filename)[1])[0]
     514      if modulename == '__init__':
     515          modulename = os.path.split(os.path.split(filename)[0])[1]
     516      if parent_docs[0]:
     517          modulename = DottedName(parent_docs[0].canonical_name, modulename)
     518      elif parent_docs[1]:
     519          modulename = DottedName(parent_docs[1].canonical_name, modulename)
     520      if options.must_introspect(modulename) or options.must_parse(modulename):
     521          log.progress(progress_estimator.progress(),
     522                       '%s (%s)' % (modulename, filename))
     523      progress_estimator.complete += 1
     524
     525      # Normalize the filename.
     526      filename = os.path.normpath(os.path.abspath(filename))
     527
     528      # When possible, use the source version of the file.
     529      try:
     530          filename = py_src_filename(filename)
     531          src_file_available = True
     532      except ValueError:
     533          src_file_available = False
     534
     535      # Get the introspected & parsed docs (as appropriate)
     536      introspect_doc = parse_doc = None
     537      introspect_error = parse_error = None
     538      if options.must_introspect(modulename):
     539          try:
     540              introspect_doc = introspect_docs(
     541                  filename=filename, context=parent_docs[0])
     542              if introspect_doc.canonical_name is UNKNOWN:
     543                  introspect_doc.canonical_name = modulename
     544          except ImportError, e:
     545              introspect_error = str(e)
     546      if src_file_available and options.must_parse(modulename):
     547          try:
     548              parse_doc = parse_docs(
     549                  filename=filename, context=parent_docs[1])
     550          except ParseError, e:
     551              parse_error = str(e)
     552          except ImportError, e:
     553              parse_error = str(e)
     554
     555      # Report any errors we encountered.
     556      _report_errors(filename, introspect_doc, parse_doc,
     557                     introspect_error, parse_error)
     558
     559      # Return the docs we found.
     560      return (introspect_doc, parse_doc)
     561
     562 -def _get_docs_from_submodules(item, pkg_docs, options, progress_estimator):
     563      # Extract the package's __path__.
     564      if isinstance(pkg_docs[0], ModuleDoc) and pkg_docs[0].is_package:
     565          pkg_path = pkg_docs[0].path
     566          package_dir = os.path.split(pkg_docs[0].filename)[0]
     567      elif isinstance(pkg_docs[1], ModuleDoc) and pkg_docs[1].is_package:
     568          pkg_path = pkg_docs[1].path
     569          package_dir = os.path.split(pkg_docs[1].filename)[0]
     570      else:
     571          return []
     572
     573      module_filenames = {}
     574      subpackage_dirs = set()
     575      for subdir in pkg_path:
     576          if os.path.isdir(subdir):
     577              for name in os.listdir(subdir):
     578                  filename = os.path.join(subdir, name)
     579                  # Is it a valid module filename?
     580                  if is_module_file(filename):
     581                      basename = os.path.splitext(filename)[0]
     582                      if os.path.split(basename)[1] != '__init__':
     583                          module_filenames[basename] = filename
     584                  # Is it a valid package filename?
     585                  if is_package_dir(filename):
     586                      subpackage_dirs.add(filename)
     587
     588      # Update our estimate of the number of modules in this package.
     589      progress_estimator.revise_estimate(item, module_filenames.items(),
     590                                         subpackage_dirs)
     591
     592      docs = [pkg_docs]
     593      for module_filename in module_filenames.values():
     594          d = _get_docs_from_module_file(
     595              module_filename, options, progress_estimator, pkg_docs)
     596          docs.append(d)
     597      for subpackage_dir in subpackage_dirs:
     598          subpackage_file = os.path.join(subpackage_dir, '__init__')
     599          docs.append(_get_docs_from_module_file(
     600              subpackage_file, options, progress_estimator, pkg_docs))
     601          docs += _get_docs_from_submodules(
     602              subpackage_dir, docs[-1], options, progress_estimator)
     603      return docs
     604
     605 -def _report_errors(name, introspect_doc, parse_doc,
     606                     introspect_error, parse_error):
     607      hdr = 'In %s:\n' % name
     608      if introspect_doc == parse_doc == None:
     609          log.start_block('%sNo documentation available!' % hdr)
     610          if introspect_error:
     611              log.error('Import failed:\n%s' % introspect_error)
     612          if parse_error:
     613              log.error('Source code parsing failed:\n%s' % parse_error)
     614          log.end_block()
     615      elif introspect_error:
     616          log.start_block('%sImport failed (but source code parsing '
     617                          'was successful).' % hdr)
     618          log.error(introspect_error)
     619          log.end_block()
     620      elif parse_error:
     621          log.start_block('%sSource code parsing failed (but '
     622                          'introspection was successful).' % hdr)
     623          log.error(parse_error)
     624          log.end_block()
     625
     626
     627  #/////////////////////////////////////////////////////////////////
     628  # Progress Estimation (for Documentation Generation)
     629  #/////////////////////////////////////////////////////////////////
     630
     631 -class _ProgressEstimator:
     632      """
     633      Used to keep track of progress when generating the initial docs
     634      for the given items.  (It is not known in advance how many items a
     635      package directory will contain, since it might depend on those
     636      packages' __path__ values.)
     637      """
     638 -    def __init__(self, items):
     639          self.est_totals = {}
     640          self.complete = 0
     641
     642          for item in items:
     643              if is_package_dir(item):
     644                  self.est_totals[item] = self._est_pkg_modules(item)
     645              else:
     646                  self.est_totals[item] = 1
     647
     648 -    def progress(self):
     649          total = sum(self.est_totals.values())
     650          return float(self.complete) / total
     651
     652 -    def revise_estimate(self, pkg_item, modules, subpackages):
     653          del self.est_totals[pkg_item]
     654          for item in modules:
     655              self.est_totals[item] = 1
     656          for item in subpackages:
     657              self.est_totals[item] = self._est_pkg_modules(item)
     658
     659 -    def _est_pkg_modules(self, package_dir):
     660          num_items = 0
     661
     662          if is_package_dir(package_dir):
     663              for name in os.listdir(package_dir):
     664                  filename = os.path.join(package_dir, name)
     665                  if is_module_file(filename):
     666                      num_items += 1
     667                  elif is_package_dir(filename):
     668                      num_items += self._est_pkg_modules(filename)
     669
     670          return num_items
     671
     672  ######################################################################
     673  ## Doc Merger
     674  ######################################################################
     675
     676  MERGE_PRECEDENCE = {
     677      'repr': 'parse',
     678
     679      # The names we get from introspection match the names that users
     680      # can actually use -- i.e., they take magic into account.
     681      'canonical_name': 'introspect',
     682
     683      # Only fall-back on the parser for is_imported if the introspecter
     684      # isn't sure.  Otherwise, we can end up thinking that vars
     685      # containing modules are not imported, which can cause external
     686      # modules to show up in the docs (sf bug #1653486)
     687      'is_imported': 'introspect',
     688
     689      # The parser can tell if an assignment creates an alias or not.
     690      'is_alias': 'parse',
     691
     692      # The parser is better able to determine what text file something
     693      # came from; e.g., it can't be fooled by 'covert' imports.
     694      'docformat': 'parse',
     695
     696      # The parse should be able to tell definitively whether a module
     697      # is a package or not.
     698      'is_package': 'parse',
     699
     700      # Extract the sort spec from the order in which values are defined
     701      # in the source file.
     702      'sort_spec': 'parse',
     703
     704      'submodules': 'introspect',
     705
     706      # The filename used by 'parse' is the source file.
     707      'filename': 'parse',
     708
     709      # 'parse' is more likely to get the encoding right, but
     710      # 'introspect' will handle programatically generated docstrings.
     711      # Which is better?
     712      'docstring': 'introspect',
     713      }
     714  """Indicates whether information from introspection or parsing should be
     715  given precedence, for specific attributes.  This dictionary maps from
     716  attribute names to either C{'introspect'} or C{'parse'}."""
     717
     718  DEFAULT_MERGE_PRECEDENCE = 'introspect'
     719  """Indicates whether information from introspection or parsing should be
     720  given precedence.  Should be either C{'introspect'} or C{'parse'}"""
     721
     722  _attribute_mergefunc_registry = {}
     723 -def register_attribute_mergefunc(attrib, mergefunc):
     724      """
     725      Register an attribute merge function.  This function will be
     726      called by L{merge_docs()} when it needs to merge the attribute
     727      values of two C{APIDoc}s.
     728
     729      @param attrib: The name of the attribute whose values are merged
     730      by C{mergefunc}.
     731
     732      @param mergefunc: The merge function, whose sinature is:
     733
     734      >>> def mergefunc(introspect_val, parse_val, precedence, cyclecheck, path):
     735      ...     return calculate_merged_value(introspect_val, parse_val)
     736
     737      Where C{introspect_val} and C{parse_val} are the two values to
     738      combine; C{precedence} is a string indicating which value takes
     739      precedence for this attribute (C{'introspect'} or C{'parse'});
     740      C{cyclecheck} is a value used by C{merge_docs()} to make sure that
     741      it only visits each pair of docs once; and C{path} is a string
     742      describing the path that was taken from the root to this
     743      attribute (used to generate log messages).
     744
     745      If the merge function needs to call C{merge_docs}, then it should
     746      pass C{cyclecheck} and C{path} back in.  (When appropriate, a
     747      suffix should be added to C{path} to describe the path taken to
     748      the merged values.)
     749      """
     750      _attribute_mergefunc_registry[attrib] = mergefunc
     751
     752 -def merge_docs(introspect_doc, parse_doc, cyclecheck=None, path=None):
     753      """
     754      Merge the API documentation information that was obtained from
     755      introspection with information that was obtained from parsing.
     756      C{introspect_doc} and C{parse_doc} should be two C{APIDoc} instances
     757      that describe the same object.  C{merge_docs} combines the
     758      information from these two instances, and returns the merged
     759      C{APIDoc}.
     760
     761      If C{introspect_doc} and C{parse_doc} are compatible, then they will
     762      be I{merged} -- i.e., they will be coerced to a common class, and
     763      their state will be stored in a shared dictionary.  Once they have
     764      been merged, any change made to the attributes of one will affect
     765      the other.  The value for the each of the merged C{APIDoc}'s
     766      attributes is formed by combining the values of the source
     767      C{APIDoc}s' attributes, as follows:
     768
     769        - If either of the source attributes' value is C{UNKNOWN}, then
     770          use the other source attribute's value.
     771        - Otherwise, if an attribute merge function has been registered
     772          for the attribute, then use that function to calculate the
     773          merged value from the two source attribute values.
     774        - Otherwise, if L{MERGE_PRECEDENCE} is defined for the
     775          attribute, then use the attribute value from the source that
     776          it indicates.
     777        - Otherwise, use the attribute value from the source indicated
     778          by L{DEFAULT_MERGE_PRECEDENCE}.
     779
     780      If C{introspect_doc} and C{parse_doc} are I{not} compatible (e.g., if
     781      their values have incompatible types), then C{merge_docs()} will
     782      simply return either C{introspect_doc} or C{parse_doc}, depending on
     783      the value of L{DEFAULT_MERGE_PRECEDENCE}.  The two input
     784      C{APIDoc}s will not be merged or modified in any way.
     785
     786      @param cyclecheck, path: These arguments should only be provided
     787          when C{merge_docs()} is called by an attribute merge
     788          function.  See L{register_attribute_mergefunc()} for more
     789          details.
     790      """
     791      assert isinstance(introspect_doc, APIDoc)
     792      assert isinstance(parse_doc, APIDoc)
     793
     794      if cyclecheck is None:
     795          cyclecheck = set()
     796          if introspect_doc.canonical_name not in (None, UNKNOWN):
     797              path = '%s' % introspect_doc.canonical_name
     798          elif parse_doc.canonical_name not in (None, UNKNOWN):
     799              path = '%s' % parse_doc.canonical_name
     800          else:
     801              path = '??'
     802
     803      # If we've already examined this pair, then there's nothing
     804      # more to do.  The reason that we check id's here is that we
     805      # want to avoid hashing the APIDoc objects for now, so we can
     806      # use APIDoc.merge_and_overwrite() later.
     807      if (id(introspect_doc), id(parse_doc)) in cyclecheck:
     808          return introspect_doc
     809      cyclecheck.add( (id(introspect_doc), id(parse_doc)) )
     810
     811      # If these two are already merged, then we're done.  (Two
     812      # APIDoc's compare equal iff they are identical or have been
     813      # merged.)
     814      if introspect_doc == parse_doc:
     815          return introspect_doc
     816
     817      # If both values are GenericValueDoc, then we don't want to merge
     818      # them.  E.g., we don't want to merge 2+2 with 4.  So just copy
     819      # the parse_doc's parse_repr to introspect_doc, & return it.
     820      # (In particular, do *not* call merge_and_overwrite.)
     821      if type(introspect_doc) == type(parse_doc) == GenericValueDoc:
     822          if parse_doc.parse_repr is not UNKNOWN:
     823              introspect_doc.parse_repr = parse_doc.parse_repr
     824          introspect_doc.docs_extracted_by = 'both'
     825          return introspect_doc
     826
     827      # Perform several sanity checks here -- if we accidentally
     828      # merge values that shouldn't get merged, then bad things can
     829      # happen.
     830      mismatch = None
     831      if (introspect_doc.__class__ != parse_doc.__class__ and
     832          not (issubclass(introspect_doc.__class__, parse_doc.__class__) or
     833               issubclass(parse_doc.__class__, introspect_doc.__class__))):
     834          mismatch = ("value types don't match -- i=%r, p=%r." %
     835                      (introspect_doc.__class__, parse_doc.__class__))
     836      if (isinstance(introspect_doc, ValueDoc) and
     837          isinstance(parse_doc, ValueDoc)):
     838          if (introspect_doc.pyval is not UNKNOWN and
     839              parse_doc.pyval is not UNKNOWN and
     840              introspect_doc.pyval is not parse_doc.pyval):
     841              mismatch = "values don't match."
     842          elif (introspect_doc.canonical_name not in (None, UNKNOWN) and
     843              parse_doc.canonical_name not in (None, UNKNOWN) and
     844              introspect_doc.canonical_name != parse_doc.canonical_name):
     845              mismatch = "canonical names don't match."
     846      if mismatch is not None:
     847          log.info("Not merging the parsed & introspected values of %s, "
     848                   "since their %s" % (path, mismatch))
     849          if DEFAULT_MERGE_PRECEDENCE == 'introspect':
     850              return introspect_doc
     851          else:
     852              return parse_doc
     853
     854      # If one apidoc's class is a superclass of the other's, then
     855      # specialize it to the more specific class.
     856      if introspect_doc.__class__ is not parse_doc.__class__:
     857          if issubclass(introspect_doc.__class__, parse_doc.__class__):
     858              parse_doc.specialize_to(introspect_doc.__class__)
     859          if issubclass(parse_doc.__class__, introspect_doc.__class__):
     860              introspect_doc.specialize_to(parse_doc.__class__)
     861      assert introspect_doc.__class__ is parse_doc.__class__
     862
     863      # The posargs and defaults are tied together -- if we merge
     864      # the posargs one way, then we need to merge the defaults the
     865      # same way.  So check them first.  (This is a minor hack)
     866      if (isinstance(introspect_doc, RoutineDoc) and
     867          isinstance(parse_doc, RoutineDoc)):
     868          _merge_posargs_and_defaults(introspect_doc, parse_doc, path)
     869
     870      # Merge the two api_doc's attributes.
     871      for attrib in set(introspect_doc.__dict__.keys() +
     872                        parse_doc.__dict__.keys()):
     873          # Be sure not to merge any private attributes (especially
     874          # __mergeset or __has_been_hashed!)
     875          if attrib.startswith('_'): continue
     876          merge_attribute(attrib, introspect_doc, parse_doc,
     877                               cyclecheck, path)
     878
     879      # Set the dictionaries to be shared.
     880      return introspect_doc.merge_and_overwrite(parse_doc)
     881
     882 -def _merge_posargs_and_defaults(introspect_doc, parse_doc, path):
     883      # If either is unknown, then let merge_attrib handle it.
     884      if introspect_doc.posargs is UNKNOWN or parse_doc.posargs is UNKNOWN:
     885          return
     886
     887      # If the introspected doc just has '...', then trust the parsed doc.
     888      if introspect_doc.posargs == ['...'] and parse_doc.posargs != ['...']:
     889          introspect_doc.posargs = parse_doc.posargs
     890          introspect_doc.posarg_defaults = parse_doc.posarg_defaults
     891
     892      # If they are incompatible, then check the precedence.
     893      elif introspect_doc.posargs != parse_doc.posargs:
     894          log.info("Not merging the parsed & introspected arg "
     895                   "lists for %s, since they don't match (%s vs %s)"
     896                    % (path, introspect_doc.posargs, parse_doc.posargs))
     897          if (MERGE_PRECEDENCE.get('posargs', DEFAULT_MERGE_PRECEDENCE) ==
     898              'introspect'):
     899              parse_doc.posargs = introspect_doc.posargs
     900              parse_doc.posarg_defaults = introspect_doc.posarg_defaults
     901          else:
     902              introspect_doc.posargs = parse_doc.posargs
     903              introspect_doc.posarg_defaults = parse_doc.posarg_defaults
     904
     905 -def merge_attribute(attrib, introspect_doc, parse_doc, cyclecheck, path):
     906      precedence = MERGE_PRECEDENCE.get(attrib, DEFAULT_MERGE_PRECEDENCE)
     907      if precedence not in ('parse', 'introspect'):
     908          raise ValueError('Bad precedence value %r' % precedence)
     909
     910      if (getattr(introspect_doc, attrib) is UNKNOWN and
     911          getattr(parse_doc, attrib) is not UNKNOWN):
     912          setattr(introspect_doc, attrib, getattr(parse_doc, attrib))
     913      elif (getattr(introspect_doc, attrib) is not UNKNOWN and
     914            getattr(parse_doc, attrib) is UNKNOWN):
     915          setattr(parse_doc, attrib, getattr(introspect_doc, attrib))
     916      elif (getattr(introspect_doc, attrib) is UNKNOWN and
     917            getattr(parse_doc, attrib) is UNKNOWN):
     918          pass
     919      else:
     920          # Both APIDoc objects have values; we need to merge them.
     921          introspect_val = getattr(introspect_doc, attrib)
     922          parse_val = getattr(parse_doc, attrib)
     923          if attrib in _attribute_mergefunc_registry:
     924              handler = _attribute_mergefunc_registry[attrib]
     925              merged_val = handler(introspect_val, parse_val, precedence,
     926                                   cyclecheck, path)
     927          elif precedence == 'introspect':
     928              merged_val = introspect_val
     929          elif precedence == 'parse':
     930              merged_val = parse_val
     931
     932          setattr(introspect_doc, attrib, merged_val)
     933          setattr(parse_doc, attrib, merged_val)
     934
     935 -def merge_variables(varlist1, varlist2, precedence, cyclecheck, path):
     936      # Merge all variables that are in both sets.
     937      for varname, var1 in varlist1.items():
     938          var2 = varlist2.get(varname)
     939          if var2 is not None:
     940              var = merge_docs(var1, var2, cyclecheck, path+'.'+varname)
     941              varlist1[varname] = var
     942              varlist2[varname] = var
     943
     944      # Copy any variables that are not in varlist1 over.
     945      for varname, var in varlist2.items():
     946          varlist1.setdefault(varname, var)
     947
     948      return varlist1
     949
     950 -def merge_value(value1, value2, precedence, cyclecheck, path):
     951      assert value1 is not None and value2 is not None
     952      return merge_docs(value1, value2, cyclecheck, path)
     953
     954 -def merge_overrides(v1, v2, precedence, cyclecheck, path):
     955      return merge_value(v1, v2, precedence, cyclecheck, path+'.<overrides>')
     956 -def merge_fget(v1, v2, precedence, cyclecheck, path):
     957      return merge_value(v1, v2, precedence, cyclecheck, path+'.fget')
     958 -def merge_fset(v1, v2, precedence, cyclecheck, path):
     959      return merge_value(v1, v2, precedence, cyclecheck, path+'.fset')
     960 -def merge_fdel(v1, v2, precedence, cyclecheck, path):
     961      return merge_value(v1, v2, precedence, cyclecheck, path+'.fdel')
     962
     963 -def merge_proxy_for(v1, v2, precedence, cyclecheck, path):
     964      # Anything we got from introspection shouldn't have a proxy_for
     965      # attribute -- it should be the actual object's documentation.
     966      return v1
     967
     968 -def merge_bases(baselist1, baselist2, precedence, cyclecheck, path):
     969      # Be careful here -- if we get it wrong, then we could end up
     970      # merging two unrelated classes, which could lead to bad
     971      # things (e.g., a class that's its own subclass).  So only
     972      # merge two bases if we're quite sure they're the same class.
     973      # (In particular, if they have the same canonical name.)
     974
     975      # If the lengths don't match up, then give up.  This is most
     976      # often caused by __metaclass__.
     977      if len(baselist1) != len(baselist2):
     978          log.info("Not merging the introspected & parsed base lists "
     979                   "for %s, since their lengths don't match (%s vs %s)" %
     980                   (path, len(baselist1), len(baselist2)))
     981          if precedence == 'introspect': return baselist1
     982          else: return baselist2
     983
     984      # If any names disagree, then give up.
     985      for base1, base2 in zip(baselist1, baselist2):
     986          if ((base1.canonical_name not in (None, UNKNOWN) and
     987               base2.canonical_name not in (None, UNKNOWN)) and
     988              base1.canonical_name != base2.canonical_name):
     989              log.info("Not merging the parsed & introspected base "
     990                       "lists for %s, since the bases' names don't match "
     991                       "(%s vs %s)" % (path, base1.canonical_name,
     992                                       base2.canonical_name))
     993              if precedence == 'introspect': return baselist1
     994              else: return baselist2
     995
     996      for i, (base1, base2) in enumerate(zip(baselist1, baselist2)):
     997          base = merge_docs(base1, base2, cyclecheck,
     998                             '%s.__bases__[%d]' % (path, i))
     999          baselist1[i] = baselist2[i] = base
    1000
    1001      return baselist1
    1002
    1003 -def merge_posarg_defaults(defaults1, defaults2, precedence, cyclecheck, path):
    1004      if len(defaults1) != len(defaults2):
    1005          if precedence == 'introspect': return defaults1
    1006          else: return defaults2
    1007      defaults = []
    1008      for i, (d1, d2) in enumerate(zip(defaults1, defaults2)):
    1009          if d1 is not None and d2 is not None:
    1010              d_path = '%s.<default-arg-val>[%d]' % (path, i)
    1011              defaults.append(merge_docs(d1, d2, cyclecheck, d_path))
    1012          elif precedence == 'introspect':
    1013              defaults.append(d1)
    1014          else:
    1015              defaults.append(d2)
    1016      return defaults
    1017
    1018 -def merge_docstring(docstring1, docstring2, precedence, cyclecheck, path):
    1019      if docstring1 is None or docstring1 is UNKNOWN or precedence=='parse':
    1020          return docstring2
    1021      else:
    1022          return docstring1
    1023
    1024 -def merge_docs_extracted_by(v1, v2, precedence, cyclecheck, path):
    1025      return 'both'
    1026
    1027 -def merge_submodules(v1, v2, precedence, cyclecheck, path):
    1028      n1 = sorted([m.canonical_name for m in v1])
    1029      n2 = sorted([m.canonical_name for m in v2])
    1030      if (n1 != n2) and (n2 != []):
    1031          log.info('Introspector & parser disagree about submodules '
    1032                   'for %s: (%s) vs (%s)' % (path,
    1033                                             ', '.join([str(n) for n in n1]),
    1034                                             ', '.join([str(n) for n in n2])))
    1035          return v1 + [m for m in v2 if m.canonical_name not in n1]
    1036
    1037      return v1
    1038
    1039  register_attribute_mergefunc('variables', merge_variables)
    1040  register_attribute_mergefunc('value', merge_value)
    1041  register_attribute_mergefunc('overrides', merge_overrides)
    1042  register_attribute_mergefunc('fget', merge_fget)
    1043  register_attribute_mergefunc('fset', merge_fset)
    1044  register_attribute_mergefunc('fdel', merge_fdel)
    1045  register_attribute_mergefunc('proxy_for', merge_proxy_for)
    1046  register_attribute_mergefunc('bases', merge_bases)
    1047  register_attribute_mergefunc('posarg_defaults', merge_posarg_defaults)
    1048  register_attribute_mergefunc('docstring', merge_docstring)
    1049  register_attribute_mergefunc('docs_extracted_by', merge_docs_extracted_by)
    1050  register_attribute_mergefunc('submodules', merge_submodules)
    1051
    1052  ######################################################################
    1053  ## Import Linking
    1054  ######################################################################
    1055
    1056 -def link_imports(val_doc, docindex):
    1057      # Check if the ValueDoc has an unresolved proxy_for link.
    1058      # If so, then resolve it.
    1059      while val_doc.proxy_for not in (UNKNOWN, None):
    1060          # Find the valuedoc that the proxy_for name points to.
    1061          src_doc = docindex.get_valdoc(val_doc.proxy_for)
    1062
    1063          # If we don't have any valuedoc at that address, then
    1064          # set that address as its canonical name.
    1065          # [XXX] Do I really want to do this?
    1066          if src_doc is None:
    1067              val_doc.canonical_name = val_doc.proxy_for
    1068              return
    1069
    1070          # If we *do* have something at that address, then
    1071          # merge the proxy `val_doc` with it.
    1072          elif src_doc != val_doc:
    1073              # Copy any subclass information from val_doc->src_doc.
    1074              if (isinstance(val_doc, ClassDoc) and
    1075                  isinstance(src_doc, ClassDoc)):
    1076                  for subclass in val_doc.subclasses:
    1077                      if subclass not in src_doc.subclasses:
    1078                          src_doc.subclasses.append(subclass)
    1079              # Then overwrite val_doc with the contents of src_doc.
    1080              src_doc.merge_and_overwrite(val_doc, ignore_hash_conflict=True)
    1081
    1082          # If the proxy_for link points back at src_doc
    1083          # itself, then we most likely have a variable that's
    1084          # shadowing a submodule that it should be equal to.
    1085          # So just get rid of the variable.
    1086          elif src_doc == val_doc:
    1087              parent_name = val_doc.proxy_for[:-1]
    1088              var_name = val_doc.proxy_for[-1]
    1089              parent = docindex.get_valdoc(parent_name)
    1090              if parent is not None and var_name in parent.variables:
    1091                  del parent.variables[var_name]
    1092              src_doc.proxy_for = None
    1093
    1094  ######################################################################
    1095  ## Canonical Name Assignment
    1096  ######################################################################
    1097
    1098  _name_scores = {}
    1099  """A dictionary mapping from each C{ValueDoc} to the score that has
    1100  been assigned to its current cannonical name.  If
    1101  L{assign_canonical_names()} finds a canonical name with a better
    1102  score, then it will replace the old name."""
    1103
    1104  _unreachable_names = {DottedName(DottedName.UNREACHABLE):1}
    1105  """The set of names that have been used for unreachable objects.  This
    1106  is used to ensure there are no duplicate cannonical names assigned.
    1107  C{_unreachable_names} is a dictionary mapping from dotted names to
    1108  integer ids, where the next unused unreachable name derived from
    1109  dotted name C{n} is
    1110  C{DottedName('%s-%s' % (n, str(_unreachable_names[n]+1))}"""
    1111
    1112 -def assign_canonical_names(val_doc, name, docindex, score=0):
    1113      """
    1114      Assign a canonical name to C{val_doc} (if it doesn't have one
    1115      already), and (recursively) to each variable in C{val_doc}.
    1116      In particular, C{val_doc} will be assigned the canonical name
    1117      C{name} iff either:
    1118        - C{val_doc}'s canonical name is C{UNKNOWN}; or
    1119        - C{val_doc}'s current canonical name was assigned by this
    1120          method; but the score of the new name (C{score}) is higher
    1121          than the score of the current name (C{score_dict[val_doc]}).
    1122
    1123      Note that canonical names will even be assigned to values
    1124      like integers and C{None}; but these should be harmless.
    1125      """
    1126      # If we've already visited this node, and our new score
    1127      # doesn't beat our old score, then there's nothing more to do.
    1128      # Note that since score increases strictly monotonically, this
    1129      # also prevents us from going in cycles.
    1130      if val_doc in _name_scores and score <= _name_scores[val_doc]:
    1131          return
    1132
    1133      # Update val_doc's canonical name, if appropriate.
    1134      if (val_doc not in _name_scores and
    1135          val_doc.canonical_name is not UNKNOWN):
    1136          # If this is the first time we've seen val_doc, and it
    1137          # already has a name, then don't change that name.
    1138          _name_scores[val_doc] = sys.maxint
    1139          name = val_doc.canonical_name
    1140          score = 0
    1141      else:
    1142          # Otherwise, update the name iff the new score is better
    1143          # than the old one.
    1144          if (val_doc not in _name_scores or
    1145              score > _name_scores[val_doc]):
    1146              val_doc.canonical_name = name
    1147              _name_scores[val_doc] = score
    1148
    1149      # Recurse to any contained values.
    1150      if isinstance(val_doc, NamespaceDoc):
    1151          for var_doc in val_doc.variables.values():
    1152              # Set the variable's canonical name.
    1153              varname = DottedName(name, var_doc.name)
    1154              var_doc.canonical_name = varname
    1155
    1156              # If the value is unknown, or is a generic value doc, then
    1157              # the valuedoc doesn't get assigned a name; move on.
    1158              if (var_doc.value is UNKNOWN
    1159                  or isinstance(var_doc.value, GenericValueDoc)):
    1160                  continue
    1161
    1162              # [XX] After svn commit 1644-1647, I'm not sure if this
    1163              # ever gets used:  This check is for cases like
    1164              # curses.wrapper, where an imported variable shadows its
    1165              # value's "real" location.
    1166              if _var_shadows_self(var_doc, varname):
    1167                  _fix_self_shadowing_var(var_doc, varname, docindex)
    1168
    1169              # Find the score for this new name.
    1170              vardoc_score = score-1
    1171              if var_doc.is_imported is UNKNOWN: vardoc_score -= 10
    1172              elif var_doc.is_imported: vardoc_score -= 100
    1173              if var_doc.is_alias is UNKNOWN: vardoc_score -= 10
    1174              elif var_doc.is_alias: vardoc_score -= 1000
    1175
    1176              assign_canonical_names(var_doc.value, varname,
    1177                                     docindex, vardoc_score)
    1178
    1179      # Recurse to any directly reachable values.
    1180      for val_doc_2 in val_doc.apidoc_links(variables=False):
    1181          val_name, val_score = _unreachable_name_for(val_doc_2, docindex)
    1182          assign_canonical_names(val_doc_2, val_name, docindex, val_score)
    1183
    1184 -def _var_shadows_self(var_doc, varname):
    1185      return (var_doc.value not in (None, UNKNOWN) and
    1186              var_doc.value.canonical_name not in (None, UNKNOWN) and
    1187              var_doc.value.canonical_name != varname and
    1188              varname.dominates(var_doc.value.canonical_name))
    1189
    1190 -def _fix_self_shadowing_var(var_doc, varname, docindex):
    1191      # If possible, find another name for the shadowed value.
    1192      cname = var_doc.value.canonical_name
    1193      for i in range(1, len(cname)-1):
    1194          new_name = cname[:i] + (cname[i]+"'") + cname[i+1:]
    1195          val_doc = docindex.get_valdoc(new_name)
    1196          if val_doc is not None:
    1197              log.warning("%s shadows its own value -- using %s instead" %
    1198                       (varname, new_name))
    1199              var_doc.value = val_doc
    1200              return
    1201
    1202      # If we couldn't find the actual value, use an unreachable name.
    1203      name, score = _unreachable_name_for(var_doc.value, docindex)
    1204      log.warning('%s shadows itself -- using %s instead' % (varname, name))
    1205      var_doc.value.canonical_name = name
    1206
    1207 -def _unreachable_name_for(val_doc, docindex):
    1208      assert isinstance(val_doc, ValueDoc)
    1209
    1210      # [xx] (when) does this help?
    1211      if (isinstance(val_doc, ModuleDoc) and
    1212          len(val_doc.canonical_name)==1 and val_doc.package is None):
    1213          for root_val in docindex.root:
    1214              if root_val.canonical_name == val_doc.canonical_name:
    1215                  if root_val != val_doc:
    1216                      log.error("Name conflict: %r vs %r" %
    1217                                (val_doc, root_val))
    1218                  break
    1219          else:
    1220              return val_doc.canonical_name, -1000
    1221
    1222      # Assign it an 'unreachable' name:
    1223      if (val_doc.pyval is not UNKNOWN and
    1224            hasattr(val_doc.pyval, '__name__')):
    1225          try:
    1226              name = DottedName(DottedName.UNREACHABLE,
    1227                                val_doc.pyval.__name__, strict=True)
    1228          except DottedName.InvalidDottedName:
    1229              name = DottedName(DottedName.UNREACHABLE)
    1230      else:
    1231          name = DottedName(DottedName.UNREACHABLE)
    1232
    1233      # Uniquify the name.
    1234      if name in _unreachable_names:
    1235          _unreachable_names[name] += 1
    1236          name = DottedName('%s-%s' % (name, _unreachable_names[name]-1))
    1237      else:
    1238          _unreachable_names[name] = 1
    1239
    1240      return name, -10000
    1241
    1242  ######################################################################
    1243  ## Documentation Inheritance
    1244  ######################################################################
    1245
    1246 -def find_overrides(class_doc):
    1247      """
    1248      Set the C{overrides} attribute for all variables in C{class_doc}.
    1249      This needs to be done early (before docstring parsing), so we can
    1250      know which docstrings to suppress warnings for.
    1251      """
    1252      for base_class in list(class_doc.mro(warn_about_bad_bases=True)):
    1253          if base_class == class_doc: continue
    1254          if base_class.variables is UNKNOWN: continue
    1255          for name, var_doc in base_class.variables.items():
    1256              if ( not (name.startswith('__') and not name.endswith('__')) and
    1257                   base_class == var_doc.container and
    1258                   name in class_doc.variables and
    1259                   class_doc.variables[name].container==class_doc and
    1260                   class_doc.variables[name].overrides is UNKNOWN ):
    1261                  class_doc.variables[name].overrides = var_doc
    1262
    1263
    1264 -def inherit_docs(class_doc):
    1265      for base_class in list(class_doc.mro(warn_about_bad_bases=True)):
    1266          if base_class == class_doc: continue
    1267
    1268          # Inherit any groups.  Place them *after* this class's groups,
    1269          # so that any groups that are important to this class come
    1270          # first.
    1271          if base_class.group_specs not in (None, UNKNOWN):
    1272              class_doc.group_specs += [gs for gs in base_class.group_specs
    1273                                        if gs not in class_doc.group_specs]
    1274
    1275          # Inherit any variables.
    1276          if base_class.variables is UNKNOWN: continue
    1277          for name, var_doc in base_class.variables.items():
    1278              # If it's a __private variable, then don't inherit it.
    1279              if name.startswith('__') and not name.endswith('__'):
    1280                  continue
    1281
    1282              # Inhetit only from the defining class. Or else, in case of
    1283              # multiple inheritance, we may import from a grand-ancestor
    1284              # variables overridden by a class that follows in mro.
    1285              if base_class != var_doc.container:
    1286                  continue
    1287
    1288              # If class_doc doesn't have a variable with this name,
    1289              # then inherit it.
    1290              if name not in class_doc.variables:
    1291                  class_doc.variables[name] = var_doc
    1292
    1293              # Otherwise, class_doc already contains a variable
    1294              # that shadows var_doc.  But if class_doc's var is
    1295              # local, then record the fact that it overrides
    1296              # var_doc.
    1297              elif class_doc.variables[name].container==class_doc:
    1298                  class_doc.variables[name].overrides = var_doc
    1299                  _inherit_info(class_doc.variables[name])
    1300
    1301  _INHERITED_ATTRIBS = [
    1302      'descr', 'summary', 'metadata', 'extra_docstring_fields',
    1303      'type_descr', 'arg_descrs', 'arg_types', 'return_descr',
    1304      'return_type', 'exception_descrs']
    1305
    1306  _method_descriptor = type(list.append)
    1307
    1308 -def _inherit_info(var_doc):
    1309      """
    1310      Copy any relevant documentation information from the variable that
    1311      C{var_doc} overrides into C{var_doc} itself.
    1312      """
    1313      src_var = var_doc.overrides
    1314      src_val = var_doc.overrides.value
    1315      val_doc = var_doc.value
    1316
    1317      # Special case: if the source value and target values are both c
    1318      # extension methods, and the target value's signature is not
    1319      # specified, then inherit the source value's signature.
    1320      if (isinstance(val_doc, RoutineDoc) and
    1321          isinstance(src_val, RoutineDoc) and
    1322          (inspect.isbuiltin(val_doc.pyval) or
    1323           isinstance(val_doc.pyval, _method_descriptor)) and
    1324          (inspect.isbuiltin(src_val.pyval) or
    1325           isinstance(src_val.pyval, _method_descriptor)) and
    1326          val_doc.all_args() in (['...'], UNKNOWN) and
    1327          src_val.all_args() not in (['...'], UNKNOWN)):
    1328          for attrib in ['posargs', 'posarg_defaults', 'vararg',
    1329                         'kwarg', 'return_type']:
    1330              setattr(val_doc, attrib, getattr(src_val, attrib))
    1331
    1332      # If the new variable has a docstring, then don't inherit
    1333      # anything, even if the docstring is blank.
    1334      if var_doc.docstring not in (None, UNKNOWN):
    1335          return
    1336      # [xx] Do I want a check like this:?
    1337  #     # If it's a method and the signature doesn't match well enough,
    1338  #     # then give up.
    1339  #     if (isinstance(src_val, RoutineDoc) and
    1340  #         isinstance(val_doc, RoutineDoc)):
    1341  #         if (src_val.posargs != val_doc.posargs[:len(src_val.posargs)] or
    1342  #             src_val.vararg != None and src_val.vararg != val_doc.vararg):
    1343  #             log.docstring_warning(
    1344  #                 "The signature of %s does not match the signature of the "
    1345  #                 "method it overrides (%s); not inheriting documentation." %
    1346  #                 (var_doc.canonical_name, src_var.canonical_name))
    1347  #             return
    1348
    1349      # Inherit attributes!
    1350      for attrib in _INHERITED_ATTRIBS:
    1351          if (hasattr(var_doc, attrib) and hasattr(src_var, attrib) and
    1352              getattr(src_var, attrib) not in (None, UNKNOWN)):
    1353              setattr(var_doc, attrib, getattr(src_var, attrib))
    1354          elif (src_val is not None and
    1355                hasattr(val_doc, attrib) and hasattr(src_val, attrib) and
    1356                getattr(src_val, attrib) not in (None, UNKNOWN) and
    1357                getattr(val_doc, attrib) in (None, UNKNOWN, [])):
    1358              setattr(val_doc, attrib, getattr(src_val, attrib))
    1359