#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License # Version 1.1 (the "License"); you may not use this file except in # compliance with the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the # License for the specific language governing rights and limitations # under the License. # # The Original Code is Komodo code. # # The Initial Developer of the Original Code is ActiveState Software Inc. # Portions created by ActiveState Software Inc are Copyright (C) 2000-2007 # ActiveState Software Inc. All Rights Reserved. # # Contributor(s): # ActiveState Software Inc # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """New citree-based codeintel evaluation engine. A 'citree' is basically an ElementTree of a CIX document with some tweaks. The idea is to use these for completion/calltip evaluation instead of the CIDB. This is mainly about performance but also about fixing some limitations, bugs, and having a better code design (i.e. where lang-specific quirks can be dealt with cleanly). """ import sys from os.path import normpath import logging import re import ciElementTree as ET if not getattr(ET, "_patched_for_komodo_", False): import warnings warnings.warn("Not using codeintel's patched elementtree: " "this may cause problems") from codeintel2.common import * from codeintel2.citadel import CitadelEvaluator log = logging.getLogger("codeintel.tree") CIX_VERSION = "2.0" def tree_2_0_from_tree_0_1(tree): """Convert CIX 0.1 to CIX 2.0.""" # - update some of the no longer used attributes # - drop "generator" try: del tree[0].attrib["generator"] except KeyError: pass # - drop 'md5' and 'mtime' on the tag try: del tree[0].attrib["md5"] except KeyError: pass try: del tree[0].attrib["mtime"] except KeyError: pass # - move "language" attribute on to "lang" and to "lang" on # (New multi-lang CIX output will allow one file to # have modules of different langs.) for file in tree.getiterator("file"): lang = file.get("language") if lang is not None: file.set("lang", lang) for module in file.getiterator("module"): if module.get("lang") is None: module.set("lang", lang) try: del file.attrib["language"] except KeyError: # Be tolerant of transitional CIX. pass # - move and optional sub tags into parent # attribute #PERF: This could be done better. for tag in ("variable", "function", "class", "module", "interface", "argument", "classref", "interfaceref"): for node in tree.getiterator(tag): for child in reversed(node): # reversed() so can modify while iterating over if child.tag == "signature": if child.text: # be tolerant of node.set("signature", child.text) node.remove(child) elif child.tag == "doc": if child.text: # be tolerant of node.set("doc", child.text) node.remove(child) if not node: # no children now node.text = None # - move non-variable tags to attributes # (XXX currently and tags are not moved) for tag in ("variable", "argument", "classref", "interfaceref"): for node in tree.getiterator(tag): for child in reversed(node): if child.tag == "type": node.set("citdl", child.get("type")) node.remove(child) if not node: # no remaining children node.text = None if tag == "argument": node.tag = "variable" node.set("ilk", "argument") # - move to a attribute for node in tree.getiterator("function"): for child in reversed(node): #PERF: could just check last child if child.tag == "returns": assert child[0].tag == "type" node.set("returns", child[0].get("type")) node.remove(child) # - move classrefs and interfacerefs to attributes # Note: => "mixinrefs" attribute. # This is used by Ruby (though not used for eval, yet). for scope_ilk in ("class", "interface"): for node in tree.getiterator(scope_ilk): interfacerefs = [] classrefs = [] mixinrefs = [] for child in reversed(node): if child.tag == "classref": if "__mixin__" in child.get("attributes", ""): mixinrefs.append(child.get("citdl") or child.attrib["name"]) else: classrefs.append(child.get("citdl") or child.attrib["name"]) node.remove(child) elif child.tag == "interfaceref": interfacerefs.append(child.get("citdl") or child.attrib["name"]) node.remove(child) if classrefs: classrefs.reverse() assert not [c for c in classrefs if ' ' in c] node.set("classrefs", ' '.join(classrefs)) if interfacerefs: interfacerefs.reverse() assert not [i for i in interfacerefs if ' ' in i] node.set("interfacerefs", ' '.join(interfacerefs)) if mixinrefs: mixinrefs.reverse() assert not [m for m in mixinrefs if ' ' in m] node.set("mixinrefs", ' '.join(mixinrefs)) if len(node) == 0: node.text = None # - make all scope tags a "scope" tag (easier for elem.find() usage) for tag in ("class", "function", "interface", "module"): for node in tree.getiterator(tag): node.tag = "scope" if tag == "class" and "__namespace__" in node.get("attributes", ""): node.set("ilk", "namespace") attributes = node.get("attributes").split() attributes.remove("__namespace__") if not attributes: del node.attrib["attributes"] else: node.set("attributes", ' '.join(attributes)) elif tag == "module": node.set("ilk", "blob") else: node.set("ilk", tag) tree.set("version", "2.0") return tree def tree_from_cix_path(cix_path): """Return a (ci)tree for the CIX content in the given path. Raises pyexpat.ExpatError if the CIX content could not be parsed. """ tree = ET.parse(cix_path).getroot() version = tree.get("version") if version == CIX_VERSION: return tree elif version == "0.1": return tree_2_0_from_tree_0_1(tree) else: raise CodeIntelError("unknown CIX version: %r" % version) def tree_from_cix(cix): """Return a (ci)tree for the given CIX content. Raises pyexpat.ExpatError if the CIX content could not be parsed. """ if isinstance(cix, unicode): cix = cix.encode("UTF-8", "xmlcharrefreplace") tree = ET.XML(cix) version = tree.get("version") if version == CIX_VERSION: return tree elif version == "0.1": return tree_2_0_from_tree_0_1(tree) else: raise CodeIntelError("unknown CIX version: %r" % version) def pretty_tree_from_tree(tree, indent_width=2): """Add appropriate .tail and .text values to the given tree so that it will have a pretty serialization. Note: This modifies the tree *in-place*. Presumption: This is a CIX 2.0 tree. """ INDENT = ' '*indent_width def _prettify(elem, indent_level=0): if elem: # i.e. has children elem.text = '\n' + INDENT*(indent_level+1) for child in elem: _prettify(child, indent_level+1) elem[-1].tail = '\n' + INDENT*indent_level elem.tail = '\n' + INDENT*indent_level else: elem.text = None elem.tail = '\n' + INDENT*indent_level _prettify(tree) return tree def check_tree(tree): """Generate warnings/errors for common mistakes in CIX trees. Yields tuples of the form: ("warning|error", ) """ assert tree.tag == "codeintel",\ "can only check starting from element" assert tree.get("version") == CIX_VERSION, \ "can only check CIX v%s trees" % CIX_VERSION # - file 'lang' is set, not 'language' file = tree[0] if not file.get("lang"): yield ("error", "no 'lang' attr on element") if file.get("language"): yield ("warning", "'language' attr on element is obsolete," "use 'lang'") for blob in file: if blob.get("ilk") != "blob": yield ("error", "element under is not ilk=blob: %r" % blob) # - blob 'lang' is set if not blob.get("lang"): yield ("error", "no 'lang' attr on element: %r" % blob) # - classrefs are space separated, not with commas (warn) for class_elem in blob.getiterator("scope"): if class_elem.get("ilk") != "class": continue classrefs = class_elem.get("classrefs") if not classrefs: continue if ',' in classrefs: yield ("warning", "multiple class references in 'classrefs' " "attr on class scopes must be " "space-separated: %r may be using " "comma-separation: %r" % (class_elem, classrefs)) class TreeEvaluator(CitadelEvaluator): def get_start_scoperef(self): linenum = self.line + 1 # convert to 1-based try: blob = self.buf.blob_from_lang[self.trg.lang] except KeyError: raise EvalError("no %s scan info for %r" % (self.lang, self.buf)) return self.buf.scoperef_from_blob_and_line(blob, linenum) def eval(self, mgr): self.mgr = mgr self.citadel = mgr.citadel if self.ctlr.is_aborted(): self.ctlr.done("aborting") return self.ctlr.info("eval %s %s", self, self.trg) self.pre_eval() try: if self.trg.form == TRG_FORM_CPLN: cplns = self.eval_cplns() if cplns: cplns = self.post_process_cplns(cplns) self.info(" cplns: %r", cplns) if cplns: self.ctlr.set_cplns(cplns) elif self.trg.form == TRG_FORM_CALLTIP: calltips = self.eval_calltips() if calltips: calltips = self.post_process_calltips(calltips) self.info(" calltips: %r", calltips) if calltips: self.ctlr.set_calltips(calltips) else: # self.trg.form == TRG_FORM_DEFN defns = self.eval_defns() if defns: defns = Definition.unique_definitions(defns) defns = self.post_process_defns(defns) self.info(" defns: %r", defns) if defns: self.ctlr.set_defns(defns) self.ctlr.done("success") except CodeIntelError, ex: #XXX Should we have an error handling hook here? self.ctlr.error("evaluating %s: %s", self, ex) self.ctlr.done("eval error") except Exception: log.exception("Unexpected error with evaluator: %s", self) # Must still mark done on the ctlr to avoid leaks - bug 65502. self.ctlr.done("eval error") def scope_stack_from_tree_and_linenum(self, tree, linenum): """Get the start scope for the given line. "linenum" appears to be 0-based, however all CIX line data is 1-based so we'll convert here. Dev Notes: - XXX Add built-in scope. """ linenum += 1 # convert to 1-based #XXX This is presuming that the tree has only one blob. scope_stack = [tree.find("file/scope")] while True: next_scope_could_be = None # PERF: Could make this a binary search if a scope has *lots* of # subscopes. for scope in scope_stack[-1].findall("scope"): start = int(scope.get("line")) if start <= linenum \ and (not scope.get("lineend") or linenum <= int(scope.get("lineend"))): next_scope_could_be = scope elif start > linenum: break if next_scope_could_be is not None: scope_stack.append(next_scope_could_be) else: break return scope_stack #TODO: split out '()' as a separate token. def _tokenize_citdl_expr(self, citdl): for token in citdl.split('.'): yield token def _join_citdl_expr(self, tokens): return '.'.join(tokens) def str_elem(self, elem): if elem.tag == "scope": return "%s %s" % (elem.get("ilk"), elem.get("name")) else: return "%s %s" % (elem.tag, elem.get("name")) def str_elem_and_children(self, elem): s = [self.str_elem(elem)] for child in elem: s.append(self.str_elem(child)) return "%s: %s" % (self.str_elem(elem), ', '.join(self.str_elem(c) for c in elem)) def str_import(self, elem): # c.f. cb.py::getDescForImport() module = elem.get("module") symbol = elem.get("symbol") alias = elem.get("alias") if alias and symbol: s = "from %(module)s import %(symbol)s as %(alias)s" % locals() elif alias: s = "import %(module)s as %(alias)s" % locals() elif symbol: s = "from %(module)s import %(symbol)s" % locals() else: s = "import %(module)s" % locals() return s # logging funcs (perhaps best on controller) def log_start(self): self._log = [] def log(self, msg, *args, **kwargs): """ kwargs: "cached" (boolean) indicates if result was from cache """ log_indent = ' '*4 if True: # just print as we go if args: s = [msg % args] else: s = [msg] if kwargs.get("cached"): s.append(" (cached)") self.info('%s', ''.join(s)) else: # capture log for latter printing self._log.append(msg, args, kwargs) def pre_eval(self): self.curr_tree = self.buf.tree #ET.dump(self.curr_tree) def _eval_citdl_expr(self, expr, scope_stack): """Return the citree node for the given CITDL expression. os.environ.get() -> """ tokens = list(self._tokenize_citdl_expr(expr)) assert tokens, "have to adjust handling if no tokens" obj = self._eval_citdl_token(tokens[0], scope_stack) for i, token in enumerate(tokens[1:]): if token.endswith("()"): token = token[-2:] call = True else: call = False if obj.tag == "import": obj = self._eval_import_getattr(obj, token, self._join_citdl_expr(tokens[:i+2])) else: # XXX marky: this code does not appear to be used! # (nobody seems to define _eval_getattr) obj = self._eval_getattr(obj, token, self._join_citdl_expr(tokens[:i+2])) if call: raise CodeIntelError("_eval_citdl_expr(%r): not handling " "call on %r " % (expr, self.str_elem(obj))) if obj.tag == "import": raise CodeIntelError("_eval_citdl_expr: cannot return import " "<%s>: need to resolve it" % self.str_import(obj)) return obj def _resolve_import(self, module_name, symbol_name=None): """Return a loaded citree node for the given import info. "module_name" is the name of the module to import. "symbol_name" (if given) is the attribute on that module to return. """ #TODO: get logging right #XXX All the logging stuff should go on the controller and that # should get passed in here for appropriate logging of this # eval. #XXX Will this screw up for, e.g. in Python: # 'import codeintel.utils'? import_handler = self.citadel.import_handler_from_lang(self.lang) module = import_handler.import_blob_name( module_name, self.buf.libs, self.ctlr) self.log("module '%s' imports <%s>", module_name, self.str_elem(module)) if symbol_name: #XXX logging won't be right here return self._eval_getattr(module, symbol_name, "%s.%s" % (module_name, symbol_name)) #XXX Here is the _eval_getattr code to duplicate. # self.log("lookup '%s' on <%s>:", name, self.str_elem(obj)) # for child in obj: # if child.get("name") == name: # attr = child # self.log("'%s' is <%s>", citdl_expr, self.str_elem(child)) # return attr # else: # raise CodeIntelError("couldn't find '%s' attribute on <%s>" # % (name, self.str_elem(obj))) else: return module def _eval_import(self, imp, name): """Return the object imported, if any, with the given import node (in a citree) and name. Return value: If successful it returns the citree node imported. If 'name' was not found in a '*'-import then None is returned (e.g. it is not exceptional that 'from os import *' does not import 'fuzzywuzzy'). If the import could not be resolved, but it looks like it should have been, then an error is raised. """ # One of these: # 'os' may be from : # ... # 'os' is # 'os' is from : (cached) # #Python #if non-* import and matches: # 'os' is from # is from ? no # ... # is from ? yes: # 'os' is # # 'dirname' may be from : # is from ? no # ... # is from ? yes: # TOTEST: # - 'from xml import dom', does that get it right? I doubt it. module_name = imp.get("module") symbol_name = imp.get("symbol") alias = imp.get("alias") obj = None if alias: if alias == name: # or self.log("'%s' is from <%s>", name, self.str_import(imp)) return self._resolve_import(module_name, symbol_name) elif symbol_name: assert symbol_name != "**", "only Perl should be using '**' for imports" if symbol_name == "*": # self.log("'%s' may be from <%s>", name, imp) #XXX some variation of _resolve_import to specify just # importing the module. try: module = self._resolve_import(module_name) except CodeIntelError, ex: # use equivalent of NoModuleEntry? self.warn("could not resolve '%s' import to handle <%s>", module_name, self.str_import(imp)) return None #TODO: # see if name in export list (__all__ for Python, # @EXPORT for Perl, default all top-level symbols) # if so, eval getattr of name on module object self.warn("not handling <%s>!", self.str_import(imp)) if symbol_name == name: # self.log("'%s' is from <%s>", name, self.str_import(imp)) return self._resolve_import(module_name, symbol_name) elif module_name == name: # self.log("'%s' is from <%s>", name, self.str_import(imp)) return self._resolve_import(module_name) return None def _eval_citdl_token(self, token, scope_stack): start_scope_str = self.str_elem(scope_stack[-1]) self.log("eval '%s' at <%s>:", token, start_scope_str) while scope_stack: scope = scope_stack.pop() self.log("is '%s' accessible on <%s>?", token, self.str_elem(scope)) for child in reversed(scope): # Check children in reverse because last definition in # file wins. A correct refinement *for the top-level* # would be to skip anything defined later in the file # than the current start position. # TODO-PERF: The list of symbols on a scope should be a # dict to speed up this loop. This is complicated # by '*' imports. if child.tag == "import": obj = self._eval_import(child, token) if obj: return obj elif child.get("name") == token: obj = child if obj.tag == "variable": citdl = obj.get("citdl") if not citdl: self.log("'%s' is <%s> which is of unknown type", token, self.str_elem(obj)) raise CodeIntelError( "don't know type of <%s> on <%s>" % (self.str_elem(obj), self.str_elem(scope))) else: self.log("'%s' is <%s> which is '%s'", token, self.str_elem(obj), citdl) obj = self._eval_citdl_expr( citdl, scope_stack+[scope]) self.log("'%s' is <%s>", token, self.str_elem(obj)) else: self.log("'%s' is <%s>", token, self.str_elem(obj)) return obj else: continue else: raise CodeIntelError("couldn't resolve '%s' starting at %s" % (token, start_scope_str)) def _defn_from_hit(self, hit): elem, (blob, lpath) = hit #self.log("_defn_from_hit:: blob: %r", blob) #for attr_name, attr_value in blob.attrib.items(): # self.log("attr_name: %r, attr_value: %r", attr_name, attr_value) #self.log("_defn_from_hit:: elem: %r", elem) path = blob.get("src", None) name = elem.get("name", None) line = elem.get("line", 1) # e.g. for an import, just use the first line if line is not None: try: line = int(line) except ValueError: line = 1 ilk = elem.get("ilk") or elem.tag citdl = elem.get("citdl", None) doc = elem.get("doc", None) signature = elem.get("signature", None) attributes = elem.get("attributes", None) returns = elem.get("returns", None) try: scope = self._elem_from_scoperef((blob, lpath)) except AttributeError: scopestart = 1 scopeend = 0 else: def safe_int_get(attr, default_value): try: return int(scope.get(attr, default_value)) except (ValueError, AttributeError): return default_value scopestart = safe_int_get("line", 1) scopeend = safe_int_get("lineend", 0) # Only fixup paths that do not look like URIs. if path and not re.match(r"^\w+:\/\/", path): if sys.platform == "win32": path = path.replace('/', '\\') # unnormalize path path = normpath(path) # remove possible '.' and '..' elements defn = Definition(blob.get("lang"), path, blob.get("name"), lpath, name, line, ilk, citdl, doc, signature, attributes, returns, scopestart, scopeend) return defn class _infinite_recursion_checker(object): def __init__(self, evalr): self.evalr = evalr def __enter__(self): self.evalr._eval_count_all += 1 if self.evalr._eval_count_all >= self.evalr._SENTINEL_MAX_ALL_COUNT: raise EvalError("Too much recursion") def __exit__(self, exc_type, exc_value, traceback): self.evalr._eval_count_all -= 1 # The SENTINEL_MAX_EXPR_COUNT could probably be *reduced*. # Note: This is an approximation that we are infinitely looping # on the same evaluation. The *actual* appropriate key would be: # # (expr, scoperef) # # but that is overkill for now, I think. _SENTINEL_MAX_EXPR_COUNT = 10 _SENTINEL_MAX_ALL_COUNT = 100 _eval_count_from_expr = None _eval_count_all = 0 def _check_infinite_recursion(self, expr): if self._eval_count_from_expr is None: # Move this init into eval() when on TreeEvalutor. self._eval_count_from_expr = {} eval_count = self._eval_count_from_expr.get(expr, 0) eval_count += 1 if eval_count >= self._SENTINEL_MAX_EXPR_COUNT: raise EvalError("hit eval sentinel: expr '%s' eval count " "is %d (abort)" % (expr, eval_count)) self._eval_count_from_expr[expr] = eval_count return TreeEvaluator._infinite_recursion_checker(self) #---- internal support stuff def _dump_element(elem, indent=''): """Dump an element tree without using ET.dump() which (1) uses XML syntax, (2) can blow up if an attribute is set to None accidentally. This is only useful for debugging. """ s = "%selement '%s': %s" % (indent, elem.tag, elem.attrib) print s for child in elem: _dump_element(child, indent+' ')