#!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License # Version 1.1 (the "License"); you may not use this file except in # compliance with the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the # License for the specific language governing rights and limitations # under the License. # # The Original Code is Komodo code. # # The Initial Developer of the Original Code is ActiveState Software Inc. # Portions created by ActiveState Software Inc are Copyright (C) 2000-2007 # ActiveState Software Inc. All Rights Reserved. # # Contributor(s): # ActiveState Software Inc # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """JavaScript support for Code Intelligence This is a Language Engine for the Code Intelligence (codeintel) system. Code Intelligence XML format. See: http://specs.tl.activestate.com/kd/kd-0100.html#xml-based-import-export-syntax-cix Future ideas and changes for the JavaScript ciler: * give anonymous functions a unique identifier, so referencing code is able to look up the correct anonymous function. * create a class prototype structure * separate class prototype variables and functions from the local variables * use better push/pop handling """ import os from os.path import splitext, basename, exists, dirname, normpath import sys import types import logging import operator from cStringIO import StringIO import weakref from glob import glob from ciElementTree import Element, ElementTree, SubElement import SilverCity from SilverCity.Lexer import Lexer from SilverCity import ScintillaConstants from SilverCity.ScintillaConstants import ( SCE_C_COMMENT, SCE_C_COMMENTDOC, SCE_C_COMMENTDOCKEYWORD, SCE_C_COMMENTDOCKEYWORDERROR, SCE_C_COMMENTLINE, SCE_C_COMMENTLINEDOC, SCE_C_DEFAULT, SCE_C_IDENTIFIER, SCE_C_NUMBER, SCE_C_OPERATOR, SCE_C_STRING, SCE_C_CHARACTER, SCE_C_STRINGEOL, SCE_C_WORD, SCE_UDL_CSL_COMMENT, SCE_UDL_CSL_COMMENTBLOCK, SCE_UDL_CSL_DEFAULT, SCE_UDL_CSL_IDENTIFIER, SCE_UDL_CSL_NUMBER, SCE_UDL_CSL_OPERATOR, SCE_UDL_CSL_REGEX, SCE_UDL_CSL_STRING, SCE_UDL_CSL_WORD, ) from codeintel2.citadel import CitadelBuffer, ImportHandler, CitadelLangIntel from codeintel2.buffer import Buffer from codeintel2.tree_javascript import JavaScriptTreeEvaluator from codeintel2 import util from codeintel2.common import * from codeintel2.indexer import PreloadBufLibsRequest, PreloadLibRequest from codeintel2.jsdoc import JSDoc, JSDocParameter, jsdoc_tags from codeintel2.gencix_utils import * from codeintel2.database.langlib import LangDirsLib from codeintel2.udl import UDLBuffer, is_udl_csl_style, is_udl_css_style from codeintel2.accessor import AccessorCache from codeintel2.langintel import (ParenStyleCalltipIntelMixin, ProgLangTriggerIntelMixin, PythonCITDLExtractorMixin) if _xpcom_: from xpcom.server import UnwrapObject #---- globals lang = "JavaScript" # Setup the logger log = logging.getLogger("codeintel.javascript.lang") #log.setLevel(logging.DEBUG) #log.setLevel(logging.INFO) util.makePerformantLogger(log) # When enabled, will add a specific path attribute to each cix element for where # this element was originally created from, good for debugging gencix JavaScript # API catalogs (YUI, ExtJS). ADD_PATH_CIX_INFO = False #ADD_PATH_CIX_INFO = True # States used by JavaScriptScanner when parsing information S_DEFAULT = 0 S_IN_ARGS = 1 S_IN_ASSIGNMENT = 2 S_IGNORE_SCOPE = 3 S_OBJECT_ARGUMENT = 4 # Types used by JavaScriptScanner when parsing information TYPE_NONE = 0 TYPE_FUNCTION = 1 TYPE_VARIABLE = 2 TYPE_GETTER = 3 TYPE_SETTER = 4 TYPE_MEMBER = 5 TYPE_OBJECT = 6 TYPE_CLASS = 7 TYPE_PARENT = 8 TYPE_ALIAS = 9 #---- language support class JavaScriptLexer(Lexer): lang = lang def __init__(self, mgr): self._properties = SilverCity.PropertySet() self._lexer = SilverCity.find_lexer_module_by_id(ScintillaConstants.SCLEX_CPP) jsli = mgr.lidb.langinfo_from_lang(self.lang) self._keyword_lists = [ SilverCity.WordList(' '.join(jsli.keywords)), SilverCity.WordList(), SilverCity.WordList(), SilverCity.WordList(), SilverCity.WordList() ] class PureJavaScriptStyleClassifier: def __init__(self): self.is_udl = False self.operator_style = SCE_C_OPERATOR self.identifier_style = SCE_C_IDENTIFIER self.keyword_style = SCE_C_WORD self.comment_styles = (SCE_C_COMMENT, SCE_C_COMMENTDOC, SCE_C_COMMENTLINE, SCE_C_COMMENTLINEDOC, SCE_C_COMMENTDOCKEYWORD, SCE_C_COMMENTDOCKEYWORDERROR) self.string_styles = (SCE_C_STRING, SCE_C_CHARACTER, SCE_C_STRINGEOL) self.whitespace_style = SCE_C_DEFAULT self.ignore_styles = self.comment_styles + (self.whitespace_style, ) class UDLJavaScriptStyleClassifier: def __init__(self): self.is_udl = True self.operator_style = SCE_UDL_CSL_OPERATOR self.identifier_style = SCE_UDL_CSL_IDENTIFIER self.keyword_style = SCE_UDL_CSL_WORD self.comment_styles = (SCE_UDL_CSL_COMMENT, SCE_UDL_CSL_COMMENTBLOCK,) self.string_styles = (SCE_UDL_CSL_STRING, ) self.whitespace_style = SCE_UDL_CSL_DEFAULT self.ignore_styles = self.comment_styles + (self.whitespace_style, ) pureJSClassifier = PureJavaScriptStyleClassifier() udlJSClassifier = UDLJavaScriptStyleClassifier() class JavaScriptLangIntel(CitadelLangIntel, ParenStyleCalltipIntelMixin, ProgLangTriggerIntelMixin, PythonCITDLExtractorMixin): lang = lang _evaluatorClass = JavaScriptTreeEvaluator extraPathsPrefName = "javascriptExtraPaths" excludePathsPrefName = "javascriptExcludePaths" namespaceMappingPrefName = "javascriptNamespaceMapping" # The way namespacing is done with variables in JS means that grouping # global vars is just annoying. cb_group_global_vars = False # Define the trigger chars we use, used by ProgLangTriggerIntelMixin trg_chars = tuple(".(,@'\" ") calltip_trg_chars = tuple('(') # excluded ' ' for perf (bug 55497) # Define literal mapping to citdl member, used in PythonCITDLExtractorMixin citdl_from_literal_type = {"string": "String"} def cb_variable_data_from_elem(self, elem): """Use the 'namespace' image in the Code Browser for a variable acting as one. """ data = CitadelLangIntel.cb_variable_data_from_elem(self, elem) if len(elem) and data["img"].startswith("variable"): data["img"] = data["img"].replace("variable", "namespace") return data def _functionCalltipTrigger(self, ac, jsClassifier, pos, DEBUG=False): # Implicit calltip triggering from an arg separater ",", we trigger a # calltip if we find a function open paren "(" and function identifier # http://bugs.activestate.com/show_bug.cgi?id=93864 if DEBUG: print "Arg separater found, looking for start of function, pos: %r" % (pos, ) ac.getPrevPosCharStyle() # Move back to the open paren of the function paren_count = 0 p = pos min_p = max(0, p - 100) # look back max 100 chars while p > min_p: p, c, style = ac.getPrecedingPosCharStyle(ignore_styles=jsClassifier.comment_styles) if DEBUG: print ' p: %r, ch: %r, st: %d' % (p, c, style) loopcount = 0 while style == jsClassifier.operator_style and loopcount < 10: loopcount += 1 if c == ")": paren_count += 1 if DEBUG: print ' paren_count: %r' % (paren_count, ) elif c == "(": if DEBUG: print ' paren_count: %r' % (paren_count, ) if paren_count == 0: # We found the open brace of the func trg_from_pos = p+1 p, ch, style = ac.getPrevPosCharStyle() if DEBUG: print " function start found, pos: %d" % (p, ) if style in jsClassifier.ignore_styles: # Find previous non-ignored style then p, c, style = ac.getPrecedingPosCharStyle(style, jsClassifier.ignore_styles) if style == jsClassifier.identifier_style: # Ensure that this isn't a new function declaration. p, c, style = ac.getPrecedingPosCharStyle(style, jsClassifier.ignore_styles) if style == jsClassifier.keyword_style: p, prev_text = ac.getTextBackWithStyle(style, jsClassifier.ignore_styles, max_text_len=len("function")+1) if prev_text in ("function", ): # Don't trigger here return None return Trigger(lang, TRG_FORM_CALLTIP, "call-signature", trg_from_pos, implicit=True) return None else: paren_count -= 1 elif c in ";{}": # Gone too far and noting was found if DEBUG: print " no function found, hit stop char: %s at p: %d" % (c, p) return None p, c, style = ac.getPrevPosCharStyle() if DEBUG: print ' p: %r, ch: %r, st: %d' % (p, c, style) # Did not find the function open paren if DEBUG: print " no function found, ran out of chars to look at, p: %d" % (p,) return None def trg_from_pos(self, buf, pos, implicit=True, lang=None): DEBUG = False # not using 'logging' system, because want to be fast #DEBUG = True #if DEBUG: # print util.banner("JavaScript trg_from_pos(pos=%r, implicit=%r)" # % (pos, implicit)) if pos == 0: return None if lang is None: lang = self.lang if isinstance(buf, UDLBuffer): jsClassifier = udlJSClassifier else: jsClassifier = pureJSClassifier accessor = buf.accessor last_pos = pos - 1 last_char = accessor.char_at_pos(last_pos) last_style = accessor.style_at_pos(last_pos) if DEBUG: print " last_pos: %s" % last_pos print " last_ch: %r" % last_char print " last_style: %r" % last_style if (jsClassifier.is_udl and last_char == '/' and last_pos > 0 and accessor.char_at_pos(last_pos-1) == '<' and last_style not in (SCE_UDL_CSL_STRING, SCE_UDL_CSL_COMMENTBLOCK, SCE_UDL_CSL_COMMENT, SCE_UDL_CSL_REGEX)): # Looks like start of closing '' tag. While typing this # the styling will still be in the CSL range. return Trigger(buf.m_lang, TRG_FORM_CPLN, "end-tag", pos, implicit) # JSDoc completions elif last_char == "@" and last_style in jsClassifier.comment_styles: # If the preceeding non-whitespace character is a "*" or newline # then we complete for jsdoc tag names p = last_pos - 1 min_p = max(0, p - 50) # Don't bother looking more than 50 chars if DEBUG: print "Checking match for jsdoc completions" while p >= min_p and \ accessor.style_at_pos(p) in jsClassifier.comment_styles: ch = accessor.char_at_pos(p) p -= 1 #if DEBUG: # print "Looking at ch: %r" % (ch) if ch == "*" or ch in "\r\n": break elif ch not in " \t\v": # Not whitespace, not a valid tag then return None else: # Nothing found in the specified range if DEBUG: print "trg_from_pos: not a jsdoc" return None if DEBUG: print "Matched trigger for jsdoc completion" return Trigger(lang, TRG_FORM_CPLN, "jsdoc-tags", pos, implicit) # JSDoc calltip elif last_char in " \t" and last_style in jsClassifier.comment_styles: # whitespace in a comment, see if it matches for jsdoc calltip p = last_pos - 1 min_p = max(0, p - 50) # Don't bother looking more than 50 chars if DEBUG: print "Checking match for jsdoc calltip" ch = None ident_found_pos = None while p >= min_p and \ accessor.style_at_pos(p) in jsClassifier.comment_styles: ch = accessor.char_at_pos(p) p -= 1 if ident_found_pos is None: #print "jsdoc: Looking for identifier, at ch: %r" % (ch) if ch in " \t": pass elif _isident(ch): ident_found_pos = p+1 else: if DEBUG: print "No jsdoc, whitespace not preceeded by an " \ "identifer" return None elif ch == "@": # This is what we've been looking for! jsdoc_field = accessor.text_range(p+2, ident_found_pos+1) if DEBUG: print "Matched trigger for jsdoc calltip: '%s'" % (jsdoc_field, ) return Trigger(lang, TRG_FORM_CALLTIP, "jsdoc-tags", ident_found_pos, implicit, jsdoc_field=jsdoc_field) elif not _isident(ch): if DEBUG: print "No jsdoc, identifier not preceeded by an '@'" # Not whitespace, not a valid tag then return None # Nothing found in the specified range if DEBUG: print "No jsdoc, ran out of characters to look at." elif last_char not in self.trg_chars: # Check if this could be a 'complete-names' trigger, this is # an implicit 3-character trigger i.e. " win<|>" or an explicit # trigger on 3 or more characters. We cannot support less than than # 3-chars without involving a big penalty hit (as our codeintel # database uses a 3-char index). if last_pos >= 2 and (last_style == jsClassifier.identifier_style or last_style == jsClassifier.keyword_style): if DEBUG: print "Checking for 'names' three-char-trigger" # The previous two characters must be the same style. p = last_pos - 1 min_p = max(0, p - 1) citdl_expr = [last_char] while p >= min_p: if accessor.style_at_pos(p) != last_style: if DEBUG: print "No 'names' trigger, inconsistent style: " \ "%d, pos: %d" % (accessor.style_at_pos(p), p) break citdl_expr += accessor.char_at_pos(p) p -= 1 else: # Now check the third character back. # "p < 0" is for when we are at the start of a document. if p >= 0: ac = None style = accessor.style_at_pos(p) if style == last_style: if implicit: if DEBUG: print "No 'names' trigger, third char " \ "style: %d, pos: %d" % (style, p) return None else: # explicit can be longer than 3-chars, skip over # the rest of the word/identifier. ac = AccessorCache(accessor, p) p, ch, style = ac.getPrecedingPosCharStyle(last_style, jsClassifier.ignore_styles, max_look_back=80) # Now we know that we are three identifier characters # preceeded by something different, which is not that # common, we now need to be a little more cpu-intensive # with our search to ensure we're not preceeded by a # dot ".", as this would mean a different cpln type! if style == jsClassifier.whitespace_style: # Find out what occurs before the whitespace, # ignoring whitespace and comments. if ac is None: ac = AccessorCache(accessor, p) p, ch, style = ac.getPrecedingPosCharStyle(style, jsClassifier.ignore_styles, max_look_back=80) if style is not None: ch = accessor.char_at_pos(p) if ch == ".": if DEBUG: print "No 'names' trigger, third char " \ "is a dot" return None elif style == jsClassifier.keyword_style: p, prev_text = ac.getTextBackWithStyle(style, jsClassifier.ignore_styles, max_text_len=len("function")+1) if prev_text in ("function", ): # We don't trigger after function, this is # defining a new item that does not exist. if DEBUG: print "No 'names' trigger, preceeding "\ "text is 'function'" return None if DEBUG: print "triggering 'javascript-complete-names' at " \ "pos: %d" % (last_pos - 2, ) return Trigger(self.lang, TRG_FORM_CPLN, "names", last_pos - 2, implicit, citdl_expr="".join(reversed(citdl_expr))) if DEBUG: print "trg_from_pos: no: %r is not in %r" % ( last_char, "".join(self.trg_chars), ) return None elif last_style == jsClassifier.operator_style: # Go back and check what we are operating on, should be # an identifier or a close brace type ")]}". p = last_pos - 1 while p >=0: style = accessor.style_at_pos(p) if style == jsClassifier.identifier_style: break elif style == jsClassifier.keyword_style and last_char == ".": break elif style == jsClassifier.operator_style and \ last_char == "." and accessor.char_at_pos(p) in ")]}": break elif style in jsClassifier.string_styles: break elif style not in jsClassifier.ignore_styles: # Else, wrong style for calltip if DEBUG: print "not a trigger: unexpected style: %d at pos: %d" \ % (style, p) return None p -= 1 else: # Did not find the necessary style, no completion/calltip then return None if last_char == ".": if style in jsClassifier.string_styles: return Trigger(lang, TRG_FORM_CPLN, "literal-members", pos, implicit, citdl_expr="String") elif style == jsClassifier.keyword_style: # Check if it's a "this." expression isThis = False if last_pos >= 4: word = [] p = last_pos - 1 p_end = last_pos - 5 while p > p_end: word.insert(0, accessor.char_at_pos(p)) p -= 1 if "".join(word) == "this": isThis = True if not isThis: return None return Trigger(lang, TRG_FORM_CPLN, "object-members", pos, implicit) elif last_char in "(,": # p is now at the end of the identifier, go back and check # that we are not defining a function ac = AccessorCache(accessor, p) # Implicit calltip triggering from an arg separater "," # http://bugs.activestate.com/show_bug.cgi?id=93864 if implicit and last_char == ',': return self._functionCalltipTrigger(ac, jsClassifier, p, DEBUG) # Get the previous style, if it's a keyword style, check that # the keyword is not "function" prev_pos, prev_char, prev_style = ac.getPrecedingPosCharStyle(jsClassifier.identifier_style, jsClassifier.ignore_styles) if prev_style == jsClassifier.keyword_style: p, prev_text = ac.getTextBackWithStyle(prev_style, jsClassifier.ignore_styles, max_text_len=len("function")+1) if prev_text in ("function", ): # Don't trigger here return None return Trigger(lang, TRG_FORM_CALLTIP, "call-signature", pos, implicit) elif last_style in jsClassifier.string_styles and last_char in "\"'": prev_pos = last_pos - 1 prev_char = accessor.char_at_pos(prev_pos) if prev_char != '[': prev_style = accessor.style_at_pos(prev_pos) ac = AccessorCache(accessor, prev_pos) if prev_style in jsClassifier.ignore_styles: # Look back further. prev_pos, prev_char, prev_style = ac.getPrevPosCharStyle(ignore_styles=jsClassifier.ignore_styles) if prev_char == '[': # We're good to go. if DEBUG: print "Matched trigger for array completions" return Trigger(lang, TRG_FORM_CPLN, "array-members", pos, implicit, bracket_pos=prev_pos, trg_char=last_char) return None def preceding_trg_from_pos(self, buf, pos, curr_pos, preceding_trg_terminators=None, DEBUG=False): DEBUG = False if DEBUG: print "pos: %d" % (pos, ) print "ch: %r" % (buf.accessor.char_at_pos(pos), ) print "curr_pos: %d" % (curr_pos, ) # Check if we can match on either of the 3-character trigger or on the # normal preceding_trg_terminators. # Try the default preceding_trg_from_pos handler if pos != curr_pos and self._last_trg_type == "names": # The last trigger type was a 3-char trigger "names", we must try # triggering from the same point as before to get other available # trigger types defined at the same poisition or before. trg = ProgLangTriggerIntelMixin.preceding_trg_from_pos( self, buf, pos+2, curr_pos, preceding_trg_terminators, DEBUG=DEBUG) else: trg = ProgLangTriggerIntelMixin.preceding_trg_from_pos( self, buf, pos, curr_pos, preceding_trg_terminators, DEBUG=DEBUG) # Now try the 3-char trigger, if we get two triggers, take the closest # match. names_trigger = None if isinstance(buf, UDLBuffer): jsClassifier = udlJSClassifier else: jsClassifier = pureJSClassifier style = None if pos > 0: accessor = buf.accessor if pos == curr_pos: # We actually care about whats left of the cursor. pos -= 1 style = accessor.style_at_pos(pos) if style in (jsClassifier.identifier_style, jsClassifier.keyword_style): ac = AccessorCache(accessor, pos) prev_pos, prev_ch, prev_style = ac.getPrecedingPosCharStyle(style) if prev_style is not None and (pos - prev_pos) > 3: # We need at least 3 character for proper completion handling. names_trigger = self.trg_from_pos(buf, prev_pos + 4, implicit=False) if DEBUG: print "trg: %r" % (trg, ) print "names_trigger: %r" % (names_trigger, ) print "last_trg_type: %r" % (self._last_trg_type, ) if names_trigger: if not trg: trg = names_trigger # Two triggers, choose the best one. elif trg.pos == names_trigger.pos: if self._last_trg_type != "names": # The names trigger gets priority over the other trigger # types, unless the previous trigger was also a names trg. trg = names_trigger elif trg.pos < names_trigger.pos: trg = names_trigger elif trg is None and style in jsClassifier.comment_styles: # Check if there is a JSDoc to provide a calltip for, example: # /** @param foobar {sometype} This is field for <|> if DEBUG: print "\njs preceding_trg_from_pos::jsdoc: check for calltip" comment = accessor.text_range(max(0, curr_pos-200), curr_pos) at_idx = comment.rfind("@") if at_idx >= 0: if DEBUG: print "\njs preceding_trg_from_pos::jsdoc: contains '@'" space_idx = comment[at_idx:].find(" ") if space_idx >= 0: # Trigger after the space character. trg_pos = (curr_pos - len(comment)) + at_idx + space_idx + 1 if DEBUG: print "\njs preceding_trg_from_pos::jsdoc: calltip at %d" % (trg_pos, ) trg = self.trg_from_pos(buf, trg_pos, implicit=False) if trg: self._last_trg_type = trg.type return trg _jsdoc_cplns = [ ("variable", t) for t in sorted(jsdoc_tags) ] def async_eval_at_trg(self, buf, trg, ctlr): if _xpcom_: trg = UnwrapObject(trg) ctlr = UnwrapObject(ctlr) ctlr.start(buf, trg) # JSDoc completions if trg.id == (self.lang, TRG_FORM_CPLN, "jsdoc-tags"): #TODO: Would like a "javadoc tag" completion image name. ctlr.set_cplns(self._jsdoc_cplns) ctlr.done("success") return # JSDoc calltip elif trg.id == (self.lang, TRG_FORM_CALLTIP, "jsdoc-tags"): #TODO: Would like a "javadoc tag" completion image name. jsdoc_field = trg.extra.get("jsdoc_field") if jsdoc_field: #print "jsdoc_field: %r" % (jsdoc_field, ) calltip = jsdoc_tags.get(jsdoc_field) if calltip: ctlr.set_calltips([calltip]) ctlr.done("success") return if trg.type == "literal-members": # We could leave this to citdl_expr_from_trg, but this is a # little bit faster, since we already know the citdl expr. citdl_expr = trg.extra.get("citdl_expr") elif trg.type == "names": # We could leave this to citdl_expr_from_trg, but since we already # know the citdl expr, use it. citdl_expr = trg.extra.get("citdl_expr") else: try: citdl_expr = self.citdl_expr_from_trg(buf, trg) except CodeIntelError, ex: ctlr.error(str(ex)) ctlr.done("error") return line = buf.accessor.line_from_pos(trg.pos) evalr = self._evaluatorClass(ctlr, buf, trg, citdl_expr, line) buf.mgr.request_eval(evalr) def _extra_dirs_from_env(self, env): extra_dirs = set() exclude_dirs = set() include_project = env.get_pref("codeintel_scan_files_in_project", True) if include_project: proj_base_dir = env.get_proj_base_dir() if proj_base_dir is not None: extra_dirs.add(proj_base_dir) # Bug 68850. for pref in env.get_all_prefs(self.extraPathsPrefName): if not pref: continue extra_dirs.update(d.strip() for d in pref.split(os.pathsep) if exists(d.strip())) for pref in env.get_all_prefs(self.excludePathsPrefName): if not pref: continue exclude_dirs.update(d.strip() for d in pref.split(os.pathsep) if exists(d.strip())) if extra_dirs: log.debug("%s extra lib dirs: %r minus %r", self.lang, extra_dirs, exclude_dirs) max_depth = env.get_pref("codeintel_max_recursive_dir_depth", 10) # Always include common JS associations - bug 91915. js_assocs = env.assoc_patterns_from_lang("JavaScript") if self.lang != "JavaScript": # Add any language specific associations. js_assocs = set(js_assocs) js_assocs.update(env.assoc_patterns_from_lang(self.lang)) js_assocs = list(js_assocs) extra_dirs = tuple( util.gen_dirs_under_dirs(extra_dirs, max_depth=max_depth, interesting_file_patterns=js_assocs, exclude_dirs=exclude_dirs) ) else: extra_dirs = () # ensure retval is a tuple return extra_dirs def _get_stdlibs_from_env(self, env=None): return [self.mgr.db.get_stdlib(self.lang)] def libs_from_buf(self, buf): env = buf.env # A buffer's libs depend on its env and the buf itself so # we cache it on the env and key off the buffer. if "javascript-buf-libs" not in env.cache: env.cache["javascript-buf-libs"] = weakref.WeakKeyDictionary() cache = env.cache["javascript-buf-libs"] # -> if buf not in cache: env.add_pref_observer(self.extraPathsPrefName, self._invalidate_cache_and_rescan_extra_dirs) env.add_pref_observer(self.excludePathsPrefName, self._invalidate_cache_and_rescan_extra_dirs) env.add_pref_observer("codeintel_selected_catalogs", self._invalidate_cache) env.add_pref_observer("codeintel_max_recursive_dir_depth", self._invalidate_cache) env.add_pref_observer("codeintel_scan_files_in_project", self._invalidate_cache) db = self.mgr.db libs = [] # - extradirslib extra_dirs = self._extra_dirs_from_env(env) if extra_dirs: libs.append( db.get_lang_lib(self.lang, "extradirslib", extra_dirs) ) # Warn the user if there is a huge number of import dirs that # might slow down completion. num_import_dirs = len(extra_dirs) if num_import_dirs > 100: msg = "This buffer is configured with %d %s import dirs: " \ "this may result in poor completion performance" % \ (num_import_dirs, self.lang) self.mgr.report_message(msg, "\n".join(extra_dirs)) if buf.lang == self.lang: # - curdirlib (before extradirslib; only if pure JS file) cwd = dirname(normpath(buf.path)) if cwd not in extra_dirs: libs.insert(0, db.get_lang_lib(self.lang, "curdirlib", [cwd])) # - cataloglibs if buf.lang == "HTML5": # Implicit HTML 5 catalog additions. libs.append(db.get_catalog_lib("JavaScript", ["html5"])) catalog_selections = env.get_pref("codeintel_selected_catalogs") libs.append(db.get_catalog_lib(self.lang, catalog_selections)) # - stdlibs libs += self._get_stdlibs_from_env(env) cache[buf] = libs return cache[buf] def _invalidate_cache(self, env, pref_name): if "javascript-buf-libs" in env.cache: log.debug("invalidate 'javascript-buf-libs' cache on %r", env) del env.cache["javascript-buf-libs"] def _invalidate_cache_and_rescan_extra_dirs(self, env, pref_name): self._invalidate_cache(env, pref_name) extra_dirs = self._extra_dirs_from_env(env) if extra_dirs: extradirslib = self.mgr.db.get_lang_lib( self.lang, "extradirslib", extra_dirs) request = PreloadLibRequest(extradirslib) self.mgr.idxr.stage_request(request, 1.0) def lpaths_from_blob(self, blob): """Return for this blob where, is a set of externally referencable lookup-paths, e.g. [("YAHOO",), ("YAHOO", "util"), ...] Note: The jury is out on whether this should include imports. However, currently this is only being used for JS (no imports) so it doesn't yet matter. """ return set(lpath for child in blob for lpath in _walk_js_symbols(child)) def citdl_expr_from_trg(self, buf, trg): """Return a CITDL expression preceding the given trigger. This is the JavaScript specialization; all it does is set array_as_attr to True. """ DEBUG=False if trg.form != TRG_FORM_DEFN: if trg.type == 'array-members': # Get everything before the bracket position. pos = trg.extra.get('bracket_pos') - 1 else: pos = trg.pos - 2 # skip past the trigger char return self._citdl_expr_from_pos(buf, pos, implicit=trg.implicit, DEBUG=DEBUG, trg=trg, array_as_attr=True) return PythonCITDLExtractorMixin.citdl_expr_from_trg(self, buf, trg) class JavaScriptBuffer(CitadelBuffer): lang = lang # Fillup chars for JavaScript: basically, any non-identifier char. # XXX - '@' removed in order to better support XPCOM completions: # Components.interfaces['@mozilla.] # Whilst not an ideal solution, as when the '.' is hit we run into the # same problem again... the ideal solution would be to override the # cpln_fillup_chars to be only "\"'" for the 'array-members' trigger # event. But this is not yet possible... # - dropped ' ' It gets in the way of common usage: "var " (bug 77950). cpln_fillup_chars = "~`!#%^&*()-=+{}[]|\\;:'\",.<>?/" cpln_stop_chars = "~`!@#%^&*()-=+{}[]|\\;:'\",.<>?/ " sce_prefixes = ["SCE_C_"] cb_show_if_empty = True @property def libs(self): return self.langintel.libs_from_buf(self) @property def stdlib(self): return self.libs[-1] def scoperef_from_blob_and_line(self, blob, line): """Return the scope for the given position in this buffer. "line" is 1-based. See CitadelBuffer.scoperef_from_pos() for details. JavaScript has two differences here: - 's are scopes if they have child tags. This CIX technique is used in JavaScript to define customized object instances. - Currently a JavaScript "class" line range may not include its methods in some cases. function Foo() { } Foo.prototype.bar = function() { } Class "Foo" has a line range that does not include method "bar". c.f. test javascript/cpln/intermixed_class_definitions """ DEBUG = False if DEBUG: print "scoperef_from_pos: look for line %d in %r" % (line, blob) best_fit_lpath = None for scope, lpath in _walk_js_scopes(blob): start = int(scope.get("line")) # JS CIX should alway have lineend. The default is # because JS 's with content (i.e. anonymous # custom Object instances) do not typically have lineend. # Note: not sure the fallback is correct. end = int(scope.get("lineend", start)) if DEBUG: print "scoperef_from_pos: scope %r (%r-%r)?"\ % (scope, start, end), if line < start: if DEBUG: print "no, before start" continue elif line > end: if DEBUG: print "no, after end" continue elif line <= end: if DEBUG: print "yes, could be" best_fit_lpath = lpath else: if DEBUG: print "no, passed end" if best_fit_lpath is not None: break if best_fit_lpath is not None: return (blob, best_fit_lpath) else: return (blob, []) class JavaScriptImportHandler(ImportHandler): # The file extensions that this import handler will use when importing. import_file_extensions = (".js", ) def _findScannableFiles(self, (files, searchedDirs), dirname, names): if sys.platform.startswith("win"): cpath = dirname.lower() else: cpath = dirname if cpath in searchedDirs: while names: del names[0] return else: searchedDirs[cpath] = 1 for i in range(len(names)-1, -1, -1): # backward so can del from list path = os.path.join(dirname, names[i]) if os.path.isdir(path): pass elif os.path.splitext(names[i])[1] in self.import_file_extensions: #XXX The list of extensions should be settable on # the ImportHandler and Komodo should set whatever is # set in prefs. #XXX This check for files should probably include # scripts, which might likely not have the # extension: need to grow filetype-from-content smarts. files.append(path) def find_importables_in_dir(self, dir): """See citadel.py::ImportHandler.find_importables_in_dir() for details. Importables for JavaScript look like this: {"foo.js": ("foo.js", None, False), "somedir": (None, None, True)} TODO: log the fs-stat'ing a la codeintel.db logging. """ from os.path import join, isdir, splitext if dir == "": #TODO: stop these getting in here. return {} #TODO: log the fs-stat'ing a la codeintel.db logging. try: names = os.listdir(dir) except OSError, ex: return {} dirs, nondirs = set(), set() for name in names: try: if isdir(join(dir, name)): dirs.add(name) else: nondirs.add(name) except UnicodeDecodeError: # Hit a filename that cannot be encoded in the default encoding. # Just skip it. (Bug 82268) pass importables = {} for name in nondirs: base, ext = splitext(name) if ext not in self.import_file_extensions: continue if base in dirs: importables[base] = (name, None, True) dirs.remove(base) else: importables[base] = (name, None, False) for name in dirs: importables[name] = (None, None, True) return importables class JavaScriptCILEDriver(CILEDriver): lang = lang def scan_purelang(self, buf): #print >> sys.stderr, buf.path log.info("scan_purelang: path: %r lang: %s", buf.path, buf.lang) norm_path = buf.path if sys.platform == "win32": # CIX requires a normalized path. norm_path = norm_path.replace('\\', '/') mtime = "XXX" jscile = JavaScriptCiler(self.mgr, norm_path, mtime, lang=buf.lang) # Profiling code: BEGIN #import hotshot, hotshot.stats #profiler = hotshot.Profile("%s.prof" % (__file__)) #profiler.runcall(jscile.scan_puretext, buf.accessor.text) # Profiling code: END jscile.scan_puretext(buf.accessor.text) tree = createCixRoot() jscile.convertToElementTreeFile(tree, file_lang=self.lang) return tree def scan_multilang(self, buf, csl_cile_driver=None, css_cile_driver=None): """Given the buffer, scan the buffer tokens for CSL UDL tokens.""" #print >> sys.stderr, buf.path log.info("scan_multilang: path: %r", buf.path) norm_path = buf.path if sys.platform == "win32": # CIX requires a normalized path. norm_path = norm_path.replace('\\', '/') #XXX Remove mtime when move to CIX 2.0. mtime = "XXX" jscile = JavaScriptCiler(self.mgr, norm_path, mtime) css_tokens = [] jscile.setStyleValues(wordStyle=SCE_UDL_CSL_WORD, identiferStyle=SCE_UDL_CSL_IDENTIFIER, operatorStyle=SCE_UDL_CSL_OPERATOR, stringStyles=(SCE_UDL_CSL_STRING, ), numberStyle=SCE_UDL_CSL_NUMBER, commentStyles=jscile.UDL_COMMENT_STYLES) for token in buf.accessor.gen_tokens(): # The tokens can be a generator of mixed UDL tokens (CSL, SSL, CSS # etc.). Need to parse out the tokens that are not CSL. if is_udl_csl_style(token['style']): jscile.token_next(**token) elif is_udl_css_style(token['style']): css_tokens.append(token) # Ensure we take notice of any text left in the ciler jscile._endOfScanReached() # We've parsed up the JavaScript, fix any variables types jscile.cile.updateAllScopeNames() tree = createCixRoot() cixfile = createCixFile(tree, norm_path, lang=buf.lang) cixblob = createCixModule(cixfile, os.path.basename(norm_path), self.lang, src=norm_path) jscile.convertToElementTreeModule(cixblob) # Hand off the CSS tokens, if any. if css_cile_driver and css_tokens: css_cile_driver.scan_css_tokens(cixfile, norm_path, css_tokens) return tree def scan_csl_tokens(self, file_elem, blob_name, csl_tokens): """csl_tokens are pure JavaScript UDL tokens. There is no need to parse out other types of tokens. """ #print >> sys.stderr, file_elem.get("path") log.info("scan_csl_tokens: %r", file_elem.get("path")) blob_elem = createCixModule(file_elem, blob_name, lang, src=file_elem.get("path")) jscile = JavaScriptCiler(self.mgr) jscile.setStyleValues(wordStyle=SCE_UDL_CSL_WORD, identiferStyle=SCE_UDL_CSL_IDENTIFIER, operatorStyle=SCE_UDL_CSL_OPERATOR, stringStyles=(SCE_UDL_CSL_STRING, ), numberStyle=SCE_UDL_CSL_NUMBER, commentStyles=jscile.UDL_COMMENT_STYLES) for csl_token in csl_tokens: jscile.token_next(**csl_token) # Ensure we take notice of any text left in the ciler jscile._endOfScanReached() # We've parsed up the JavaScript, fix any variables types jscile.cile.updateAllScopeNames() jscile.convertToElementTreeModule(blob_elem) # Everything is JS is an object.... MUMUHAHAHAHAHAHAAAA....... class JSObject: def __init__(self, name, parent, lineno, depth, type=None, doc=None, isLocal=False, isHidden=False, path=None, pos=None): self.name = name self.parent = parent self.cixname = self.__class__.__name__[2:].lower() self.line = lineno self.lineend = -1 self.depth = depth self.type = type self.path = path self.pos = pos # Used for argument positions - 0 indexed. self._class = None # Used when part of a class self.classes = {} # declared sub-classes self.members = {} # all private member variables used in class self.variables = {} # all variables used in class self.functions = {} self.anonymous_functions = [] # anonymous functions declared in scope self.attributes = [] # Special attributes for object self.returnTypes = [] # List of possible return values self.constructor = None self.classrefs = [] self.doc = doc self.metadata = None self.isHidden = isHidden # Special case, should not be output to cix if isLocal: # XXX: TODO: It may be appropriate to just use private..., although # my feeling of the difference between the two names # is that private elements should still be listed in # completions from the class itself, whereas local # should not... # # Local has a special meaning within the javascript tree evaluator, # elements with a "__local__" attribute will not be included in js # codeintel completion results. self.attributes.append("__local__") # Private has a special meaning within the code browser, # an element with a "private" attribute shows a small lock icon. # Private also has special meaning for jsdoc purposes, where it # means not to show documentation for these elements. self.attributes.append("private") self.doc = doc self.jsdoc = None if self.doc: # Turn the doc list into a JSDoc object self.jsdoc = JSDoc("".join(self.doc)) def setParent(self, parent): # Validate the parent/child relationship. This is to avoid possible # recursion errors - bug 85481. seen_scopes = [self] parent_scope = parent while parent_scope: if parent_scope in seen_scopes: raise CodeIntelError("setParent:: recursion error, scope: %r, " "parent: %r" % (self.name, parent_scope.name)) seen_scopes.append(parent_scope) parent_scope = parent_scope.parent self.parent = parent def getFullPath(self): if self.parent: return self.parent.getFullPath() + [self.name] return [self.name] def addAttribute(self, attr): if attr not in self.attributes: self.attributes.append(attr) def removeAttribute(self, attr): while attr in self.attributes: self.attributes.remove(attr) def hasChildren(self): return self.variables or self.functions or self.classes or self.members def isAnonymous(self): return self.name.startswith("(anonymous") def addClassRef(self, baseclass): assert isinstance(baseclass, (str, unicode)), "baseclass %r is not a str" % (baseclass,) if baseclass not in self.classrefs: self.classrefs.append(baseclass) def _mergeVariables(self, existingvar, newvar): # If there is no citdl type yet, assign it the given type if newvar.type and not existingvar.type: log.debug("marging VAR:%s, setting type: %r", existingvar.name, newvar.type) existingvar.type = newvar.type # TODO: We could choose the simpler type, i.e. if types were "foo" or # "int", then we should choose "int" as it's a base JS type. #if newvar.type and newvar.type != existingvar.type: # print " %r existing type: %r, new type: %r" % (existingvar.name, existingvar.type, newvar.type) # See if the locality has changed - bug 93726. if "__file_local__" in existingvar.attributes and \ not "__file_local__" in newvar.attributes: existingvar.removeAttribute("__file_local__") # This should be the main definition of this variable - so prefer # other fields (like citdl) too. if newvar.type: existingvar.type = newvar.type def addVariable(self, name, value=None, metadata=None): """Add a variable in this scope (possibly as a property) @param name {unicode} The name of the variable @param value {JSObject} The value of the variable @param metadata {dict} Extra metadata about the assignment @returns {JSObject} The resulting variable (possibly different from |value| if one with the given |name| exists) """ v = self.variables.get(name, None) if v is None: v = self.members.get(name, None) if v is None: log.info("VAR:%s, line:%d, type:%r, scope:%r, meta:%r", name, value.line, value.type, self.name, metadata) v = value self.variables[name] = v else: self._mergeVariables(v, value) if metadata: if v.metadata is None: v.metadata = metadata else: v.metadata.update(metadata) return v def addMemberVariable(self, name, value): """Add a member variable to this object @param name {unicode} The name of the member @param value {JSObject} The value to add @returns {JSObject} The resulting member (possibly different from |value| if one with the given |name| exists) """ assert isinstance(value, JSObject), \ "addMemberVariable: bad value%r" % (value,) v = self.members.get(name, None) if v is None: v = self.variables.get(name, None) if v is None: log.info("CLASSMBR: %r, in %s %r", name, self.cixname, self.name) self.members[name] = v = value else: self._mergeVariables(v, value) return v def getReturnType(self): """Get the JS return type for this function, JSDoc gets precedence.""" bestType = None if self.jsdoc and self.jsdoc.returns: bestType = self.jsdoc.returns.paramtype elif len(self.returnTypes) > 0: d = {} bestCount = 0 bestType = None for rtype in self.returnTypes: if isinstance(rtype, (str, unicode)): count = d.get(rtype, 0) + 1 d[rtype] = count if count > bestCount: bestType = rtype if bestType: bestType = standardizeJSType(bestType) return bestType def __repr__(self): return "\n".join(self.outline()) def outline(self, depth=0): result = [] if self.cixname == "function": s = "%s%s %s(%s)" % (" " * depth, self.cixname, self.name, ", ".join(self.args)) r = self.getReturnType() if r: s += " => %s" % (r, ) result.append(s) elif self.cixname == "class" and self.classrefs: result.append("%s%s %s [%s]" % (" " * depth, self.cixname, self.name, self.classrefs)) elif self.cixname == "variable" and (self.type or (self.jsdoc and self.jsdoc.type)): result.append("%s%s %s [%s]" % (" " * depth, self.cixname, self.name, self.type or (self.jsdoc and self.jsdoc.type))) else: result.append("%s%s %s" % (" " * depth, self.cixname, self.name)) for attrname in ("classes", "members", "functions", "variables"): d = getattr(self, attrname, {}) for v in d.values(): result += v.outline(depth + 2) return result def toElementTree(self, cixelement): if not self.name: log.info("%s has no name, line: %d, ignoring it.", self.cixname, self.line) return None if self.cixname == "function": cixobject = createCixFunction(cixelement, self.name) elif self.cixname in ("object", "variable"): cixobject = createCixVariable(cixelement, self.name) elif self.cixname in ("class"): cixobject = createCixClass(cixelement, self.name) #else: # print "self.cixname: %r" %(self.cixname) cixobject.attrib["line"] = str(self.line) if self.lineend >= 0: cixobject.attrib["lineend"] = str(self.lineend) if ADD_PATH_CIX_INFO and self.path: cixobject.attrib["path"] = self.path jsdoc = self.jsdoc if jsdoc: #print "jsdoc: %r" % (jsdoc) # the docstring #docElem.text = self.doc attributeDocs = [] if jsdoc.isDeprecated(): attributeDocs.append("DEPRECATED") self.attributes.append("deprecated") if jsdoc.isPrivate(): attributeDocs.append("PRIVATE") if "private" not in self.attributes: self.attributes.append("private") if jsdoc.isStatic(): attributeDocs.append("STATIC") if "__static__" not in self.attributes: self.attributes.append("__static__") if jsdoc.isConstant(): attributeDocs.append("CONSTANT") if "constant" not in self.attributes: self.attributes.append("constant") if jsdoc.isConstructor() and self.cixname == "function": attributeDocs.append("CONSTRUCTOR") if "__ctor__" not in self.attributes: self.attributes.append("__ctor__") if jsdoc.is__local__(): attributeDocs.append("__LOCAL__") if "__local__" not in self.attributes: self.attributes.append("__local__") if jsdoc.tags: cixobject.attrib["tags"] = jsdoc.tags if jsdoc.doc: if attributeDocs: setCixDoc(cixobject, "%s: %s" % (" ".join(attributeDocs), jsdoc.doc)) else: setCixDoc(cixobject, jsdoc.doc) # Additional one-off attributes if self.attributes: cixobject.attrib["attributes"] = " ".join(self.attributes) # Additional meta-data. if self.metadata: for key, value in self.metadata.items(): cixobject.attrib[key] = value # Add the type information, JSDoc overrides whatever the ciler found if jsdoc and jsdoc.type: # Convert the value into a standard name addCixType(cixobject, standardizeJSType(jsdoc.type)) elif self.type: assert isinstance(self.type, (str, unicode)), \ "self.type %r is not a str" % (self.type) addCixType(cixobject, standardizeJSType(self.type)) if isinstance(self, JSFunction): signature = "%s(" % (self.name) # Add function arguments if self.args: signature += ", ".join(self.args) # Add function arguments to tree # Add signature - calltip signature += ")" cixobject.attrib["signature"] = signature # Add return type for functions, JSDoc gets precedence returnType = self.getReturnType() if returnType: addCixReturns(cixobject, returnType) # Add a "this" member for class functions if self._class: createCixVariable(cixobject, "this", vartype=self._class.name) elif self.parent and self.parent.cixname in ("object", "variable"): createCixVariable(cixobject, "this", vartype=self.parent.name) if self.cixname == "class": for baseclass in self.classrefs: addClassRef(cixobject, baseclass) if self.jsdoc and self.jsdoc.baseclasses: for baseclass in self.jsdoc.baseclasses: if baseclass not in self.classrefs: addClassRef(cixobject, baseclass) # Note that arguments must be kept in the order they were defined. variables = self.variables.values() arguments = [x for x in variables if isinstance(x, JSArgument)] variables = [x for x in variables if not isinstance(x, JSArgument)] allValues = sorted(arguments, key=operator.attrgetter("pos", "name")) + \ sorted(self.functions.values() + self.members.values() + \ self.classes.values() + variables + \ self.anonymous_functions, key=operator.attrgetter("line", "name")) # If this is a variable with child elements, yet has a citdl type of # something that is not an "Object", don't bother to adding these child # elements, as we will just go with what the citdl information holds. # http://bugs.activestate.com/show_bug.cgi?id=78484 # Ideally the ciler should include this information and have the tree # handler combine the completions from the citdl and also the child # elements, but this is not yet possible. if allValues and self.cixname == 'variable' and \ cixobject.get("citdl") and cixobject.get("citdl") not in ("Object", "require()"): log.info("Variable of type: %r contains %d child elements, " "ignoring them.", cixobject.get("citdl"), len(allValues)) return None # Sort and include contents for v in allValues: if not v.isHidden: v.toElementTree(cixobject) return cixobject class JSVariable(JSObject): def __init__(self, name, parent, line, depth, vartype='', doc=None, isLocal=False, path=None, pos=None): if isinstance(vartype, list): vartype = ".".join(vartype) JSObject.__init__(self, name, parent, line, depth, type=vartype, doc=doc, isLocal=isLocal, path=path, pos=pos) class JSAlias(JSVariable): """An alias, which is a simple assignment from a variable (or possibly a class).""" def __init__(self, *args, **kwargs): """Initialize the alias @param target {list of unicode} The type name expression (e.g. ["Array"] or ["window", "document"]) to alias to @param scope {JSObject} The scope in which the assignment takes place, used to resolve the target @see JSVariable.__init__ - all other arguments are inherited """ target = kwargs.pop("target", None) assert target is not None, "JSAlias requires a target= keyword argument" scope = kwargs.pop("scope", None) assert scope is not None, "JSAlias requires a scope= keyword argument" JSVariable.__init__(self, *args, **kwargs) self.cixname = "variable" self.target = target self.scope = scope class JSArgument(JSVariable): """An argument for a function (or constructor)""" def __init__(self, *args, **kwargs): JSVariable.__init__(self, *args, **kwargs) self.cixname = "variable" if self.name.startswith("..."): # ES6 rest parameter (..arg) self.type = "Array" self.name = self.name[3:] if not self.type and ENABLE_HEURISTICS: if self.name == 'event': # assume that variables named event are Events log.debug("JSArgument: assuming argument named event is a Event") self.type = "Event" def outline(self, depth=0): result = [] if self.type or (self.jsdoc and self.jsdoc.type): result.append("%sargument %s [%s]" % (" " * depth, self.name, self.type or (self.jsdoc and self.jsdoc.type))) else: result.append("%sargument %s" % (" " * depth, self.name)) for attrname in ("classes", "members", "functions", "variables"): d = getattr(self, attrname, {}) for v in d.values(): result += v.outline(depth + 2) return result def toElementTree(self, *args, **kwargs): cixelement = JSVariable.toElementTree(self, *args, **kwargs) if cixelement is not None: cixelement.attrib["ilk"] = "argument" return cixelement class JSFunction(JSObject): """A JavaScript function""" def __init__(self, funcname, parent, args, lineno, depth=0, doc=None, isLocal=False, isHidden=False, path=None): """Initialize the function @param funcname {unicode} The name of the function @param parent {JSObject} The parent scope @param args {JSArgs} The arguments to the function @param lineo {int} The line the function starts at @param depth {int} @param doc {list of unicode} Documentation comment for the function @param isLocal {bool} Whether this function is local @param isHidden {bool} Whether this function should be hidden (not output into the cix file) @param path {list of unicode} The path to the function """ # funcname: string # args: list (or None) JSObject.__init__(self, funcname, parent, lineno, depth, doc=doc, isLocal=isLocal, isHidden=isHidden, path=path) if isinstance(parent, JSClass): self._class = parent self._parent_assigned_vars = [] self._callers = set() self.args = list(args or []) for pos, arg in enumerate(self.args): self.addVariable(arg, JSArgument(name=arg, parent=self, line=lineno, depth=depth, pos=pos)) ## # @rtype {string or JSObject} add this possible return type def addReturnType(self, rtype): self.returnTypes.append(rtype) def addCaller(self, caller, pos, line): """Add caller information to this function""" if isinstance(caller, (list, tuple)): caller = ".".join(caller) self._callers.add((pos, caller, line)) def toElementTree(self, cixelement): if self.jsdoc: # fix up argument info from jsdoc for jsdocParam in self.jsdoc.params: var = self.variables.get(jsdocParam.paramname) if isinstance(var, JSArgument): var.type = standardizeJSType(jsdocParam.paramtype) # fake a JSDoc (since it's already all parsed) var.jsdoc = JSDoc() var.jsdoc.doc = jsdocParam.doc cixobject = JSObject.toElementTree(self, cixelement) if not cixobject: return cixobject for pos, caller, line in self._callers: SubElement(cixobject, "caller", citdl=caller, pos=str(pos), line=str(line), attributes="__hidden__") return cixobject class JSClass(JSObject): """A JavaScript class object (a function with a non-default .prototype)""" def __init__(self, name, parent, lineno, depth, doc=None, path=None): """Initialize the class @see JSObject.__init__ """ JSObject.__init__(self, name, parent, lineno, depth, doc=doc, path=path) self.constructor = name class JSFile: """CIX specifies that a tag have zero or more children. In JavaScript this is a one-to-one relationship, so this class represents both (and emits the XML tags for both). """ def __init__(self, path, mtime=None): self.path = path self.name = os.path.basename(path) self.parent = None self.cixname = self.__class__.__name__[2:].lower() #XXX Drop mtime when move to CIX 2.0. if mtime is None: mtime = "XXX" self.mtime = mtime self.functions = {} # functions declared in file self.anonymous_functions = [] # anonymous functions declared in file self.classes = {} # classes declared in file self.variables = {} # all variables used in file def __repr__(self): return "\n".join(self.outline()) def getFullPath(self): return [self.name] def isAnonymous(self): return False def outline(self): result = ["File: %r" % (self.name) ] for attrname in ("classes", "functions", "variables"): d = getattr(self, attrname, {}) for v in d.values(): result += v.outline(2) return result def hasChildren(self): return self.variables or self.functions or self.classes def _findScopeWithName(self, name, scopeStack, type="variables"): """Find a object somewhere on the given scope stack with the given name @param name {unicode} The name to look for @param scopeStack {list of JSObject} The scope stack, with the outermost (and therefore last searched) scope at the lowest index @param type {str} The type of property to look for, e.g. "variables", "classs", "functions" @returns {JSObject or None} The object found, or None """ if not name: return None log.debug("_findScopeWithName: %r with name:%r in scopeStack:%r", type, name, scopeStack[-1].name) # Work up the scope stack looking for the name #for scopePos in range(len(scope) - 1, -1, -1): # currentScope = scope[scopePos] for scopePos in range(len(scopeStack) - 1, -1, -1): currentScope = scopeStack[scopePos] #print "Looking in scope %r" % (currentScope.name) #print "Looking in %s: %r" % (currentScope.__class__.__name__, # currentScope.name) namesDict = getattr(currentScope, type, None) if namesDict: foundScope = namesDict.get(name) if foundScope: log.debug("Found %r in scope:%r(%s)", name, currentScope.name, currentScope.cixname) return foundScope log.debug("NO scope found for: %r", name) return None def _lookupVariableType(self, varType, jsobject, scopeStack, depth=0): """Resolves a variable type string to a JSObject representing a well- known JavaScript type @param varType {str} The type to look up as a dot-separated string, e.g. "Foo.Bar.baz" or "Array" @param jsobject {JSObject} The starting JSObject candidate (in case varType is well-known) @param scopeStack {list} The scope stack, with the outermost (and therefore last searched) scope at the lowest index @param depth {int} (For internal use to prevent deep recursion) @returns {JSObject or None} The variable type, or None """ #print "Looking for varType:%r in scope:%r" % (varType, scopeStack[-1].name) assert not varType or isinstance(varType, (str, unicode)), \ "varType %r is not a string" % varType if depth < 10 and varType: # Don't look any further if it's a known type if varType.lower() in known_javascript_types: return jsobject sp = varType.split(".") #print "sp: %r" % (sp) namePos = 0 while namePos < len(sp): name = sp[namePos] #print "sp[%d]: %r" % (namePos, name) foundScope = self._findScopeWithName(name, scopeStack, type="variables") alternateScopeStack = scopeStack while foundScope and isinstance(foundScope, JSArgument) and \ not foundScope.type and foundScope.parent in alternateScopeStack: log.debug("found untyped argument %r on %r, trying next", foundScope.name, foundScope.parent.name) alternateScopeStack[alternateScopeStack.index(foundScope.parent):] = [] foundScope = self._findScopeWithName(name, alternateScopeStack, type="variables") if not foundScope: #print "Trying member variables" # Then look for a class members with this name foundScope = self._findScopeWithName(name, scopeStack, type="members") #if foundScope: # print "Found a member variable with this name" if not foundScope: # Then look for a class with this name #print "Trying class" foundScope = self._findScopeWithName(name, scopeStack, type="classes") if foundScope: #print "Found a class with this name" # Only search this scope now scopeStack.append(foundScope) if not foundScope: break # returns None #print "Found scope" if isinstance(foundScope, JSVariable): #print "Recursively searching scope" assert foundScope.type is None or isinstance(foundScope.type, (str, unicode)), \ "foundScope %r has invalid type %r" % (foundScope, foundScope.type) foundScope = self._lookupVariableType(foundScope.type, foundScope, scopeStack, depth+1) #return self._lookupVariableType(foundType, scopeStack) namePos += 1 #print "Returning: %s" % foundScope return foundScope return None #print "jsobject:%r" % (jsobject) #print "jsobject.type:%r" % (jsobject.type) def _lookupVariableTypes(self, jstypelist, scopeStack): """Work out variable types according to their namespace @param jstypelist {list of JSObject} The list of types that shold be examined @param scopeStack {list} The scope stack, with the outermost (and therefore last searched) scope at the lowest index @returns {None} """ for jstype in jstypelist: if hasattr(jstype, "classes"): # Recursive lookup for the class variables self._lookupVariableTypes(jstype.classes.values(), scopeStack + [jstype]) if hasattr(jstype, "functions"): # Recursive lookup for the function variables self._lookupVariableTypes(jstype.functions.values(), scopeStack + [jstype]) if hasattr(jstype, "variables"): for jsvariable in jstype.variables.values(): varType = jsvariable.type if varType: actualType = self._lookupVariableType(varType, jsvariable, scopeStack + [jstype]) if actualType and actualType != jsvariable: if isinstance(actualType, JSVariable) and not actualType.hasChildren(): log.debug("variable %r: replacing type %r with %r", jsvariable.name, jsvariable.type, actualType.type) jsvariable.type = actualType.type else: log.debug("variable %r: replacing type %r with %r", jsvariable.name, jsvariable.type, actualType.name) jsvariable.type = actualType.name # Lookup function return type values if isinstance(jstype, JSFunction): for i in range(len(jstype.returnTypes)): returnType = jstype.returnTypes[i] #print "Looking up function return type: %r" % (returnType, ) if isinstance(returnType, (str, unicode)): actualType = self._lookupVariableType(returnType, jstype, scopeStack + [jstype]) if actualType and actualType != jstype: #print "actualType: %r" % (actualType, ) # Use the variable name if it's type is "Object" if isinstance(actualType, JSVariable) and \ actualType.type != "Object": #print "ActualType is: %r" % (actualType.type) jstype.returnTypes[i] = actualType.type else: #print "ActualType is: %r" % (actualType.name) jstype.returnTypes[i] = actualType.name def _updateClassConstructors(self, jsobject): """Recursively update the class constructor name of the given class @param jsobject {list of JSObject} The JSClasses (or things things containing the JSClasses) to find constructors to mark as ctors """ if isinstance(jsobject, JSClass): if jsobject.constructor: jsfunc = self._findScopeWithName(jsobject.constructor, [jsobject], type='functions') if jsfunc and "__ctor__" not in jsfunc.attributes: log.debug("Making function:%r the constructor for class:%r", jsfunc.name, jsobject.name) jsfunc.attributes.append("__ctor__") allObjects = jsobject.functions.values() + jsobject.classes.values() + \ jsobject.variables.values() if not isinstance(jsobject, JSFile): allObjects += jsobject.members.values() for subobj in allObjects: self._updateClassConstructors(subobj) def updateAllScopeNames(self): """We've gathered as much information as possible, update all scope names as best as possible.""" log.info("****************************************") log.info("Finished scanning, updating all scope names") self._lookupVariableTypes([self], []) log.info("Updating all class constructor names") self._updateClassConstructors(self) def addVariable(self, name, value=None, metadata=None): """Add the given variable to this file @see JSObject.addVariable """ assert value is not None, "no value given" v = self.variables.get(name, None) if v is None: log.info("VAR: %s on line %d, type:%r (class %r)", name, value.line, value.type, type(value)) self.variables[name] = v = value # Else if there is no citdl type yet, assign it the given type elif value.type and not v.type: log.debug("existing VAR:%s, setting type: %r", name, value.type) v.type = value.type if metadata: if v.metadata is None: v.metadata = metadata else: v.metadata.update(metadata) return v def convertToElementTreeModule(self, cixmodule): """Convert this file to a in the cix (DOM) tree @param cixmodule {Element} The to write to """ # Sort and include contents allValues = self.functions.values() + self.variables.values() + \ self.classes.values() + self.anonymous_functions for v in sorted(allValues, key=operator.attrgetter("line", "name")): if not v.isHidden: v.toElementTree(cixmodule) def convertToElementTreeFile(self, cixelement, file_lang, module_lang=None): """Convert this file to a new .cix file @param cixelement {Element} The root element in a CIX file @param file_lang The language of this file (possibly not JavaScript in a mutli-language file) @param module_lang The language of this module (defaults to file_lang) """ if module_lang is None: module_lang = file_lang cixfile = createCixFile(cixelement, self.path, lang=file_lang, mtime=str(self.mtime)) cixmodule = createCixModule(cixfile, self.name, lang=module_lang, src=self.path) self.convertToElementTreeModule(cixmodule) class JavaScriptCiler: JS_COMMENT_STYLES = (SCE_C_COMMENT, SCE_C_COMMENTDOC, SCE_C_COMMENTLINE, SCE_C_COMMENTLINEDOC, SCE_C_COMMENTDOCKEYWORD, SCE_C_COMMENTDOCKEYWORDERROR) UDL_COMMENT_STYLES = (SCE_UDL_CSL_COMMENT, SCE_UDL_CSL_COMMENTBLOCK) def __init__(self, mgr, path="", mtime=None, lang="JavaScript"): """Initialize the ciler @param mgr {codeintel2.manager.Manager} codeintel manager @param path {str} Normalized filesystem path to the file being scanned @param mtime {str} The last modified timestamp @param lang {str} The language being scanned """ self.mgr = mgr self.path = path self.lang = lang # hook up the lexical matches to a function that handles the token # Working variables, used in conjunction with state self.lineno = 0 self.last_lineno = 0 self.depth = 0 self.styles = [] self.text = [] self.in_variable_definition = False # for multi variable assignment self.comment = [] self.last_comment_and_jsdoc = [None, None] self.argumentPosition = 0 self.argumentTextPosition = 0 # keep track of arg position in self.text self.objectArguments = [] # state : used to store the current JS lexing state # state_stack : used to store JS state to return to self.state = S_DEFAULT self.state_stack = [] # JScile will store all references for what we scan in self.cile = JSFile(path, mtime) # Cile information, used to store code structure self.currentScope = self.cile self._scopeStack = [self.currentScope] self.objectStack = [self.currentScope] self.currentClass = None # Used for determining Javascript closures self.bracket_depth = 0 self.lastText = [] self.lastScope = None self._metadata = {} self._anonid = 0 self.anonFuncPending = False # Document styles used for deciding what to do # Note: Can be customized by calling setStyleValues() self.JS_WORD = SCE_C_WORD self.JS_IDENTIFIER = SCE_C_IDENTIFIER self.JS_OPERATOR = SCE_C_OPERATOR self.JS_STRINGS = (SCE_C_STRING, SCE_C_CHARACTER, ) self.JS_NUMBER = SCE_C_NUMBER # js_cile styles are styles that the ciler uses self.JS_CILE_STYLES = self.JS_STRINGS + \ (self.JS_WORD, self.JS_IDENTIFIER, self.JS_OPERATOR, self.JS_NUMBER) # Allows to change styles used by scanner # Needed for UDL languages etc... where the style bits are different def setStyleValues(self, wordStyle = SCE_C_WORD, identiferStyle = SCE_C_IDENTIFIER, operatorStyle = SCE_C_OPERATOR, stringStyles = (SCE_C_STRING, SCE_C_CHARACTER, ), numberStyle = SCE_C_NUMBER, commentStyles = None): self.JS_WORD = wordStyle self.JS_IDENTIFIER = identiferStyle self.JS_OPERATOR = operatorStyle self.JS_STRINGS = stringStyles self.JS_NUMBER = numberStyle self.JS_CILE_STYLES = self.JS_STRINGS + \ (self.JS_WORD, self.JS_IDENTIFIER, self.JS_OPERATOR, self.JS_NUMBER) if commentStyles: self.JS_COMMENT_STYLES = commentStyles def _logVariables(self): """Helper method to log about the current state""" if log.level >= logging.DEBUG: log.debug(" lineno:%r, state:%r, depth:%r", self.lineno, self.state, self.depth) log.debug(" currentScope: %r", self.currentScope) log.debug("") def incBlock(self): """Increment the block (scope) count""" self.depth = self.depth+1 log.info("incBlock: depth:%d, line:%d, currentScope:%r", self.depth, self.lineno, self.currentScope.name) if not self.currentScope: log.info("incBlock:: No currentScope available. Defaulting to global file scope.") # Use the global file scope then self.currentScope = self.cile if len(self.objectStack) == 0 or self.currentScope != self.objectStack[-1]: # Not the same scope... self.objectStack.append(self.currentScope) self._scopeStack.append(self.currentScope) def decBlock(self): """Decrement the block (scope) count""" log.info("decBlock: depth:%d, line:%d, leavingScope:%r", self.depth, self.lineno, self.currentScope.name) if self.depth > 0: self.depth = self.depth-1 self.lastScope = self.currentScope # Update lineend for scope if hasattr(self.currentScope, "lineend"): self.currentScope.lineend = self.lineno if isinstance(self.currentScope, JSClass) and \ len(self.currentScope.functions) == 1: jsfunc = self.currentScope.functions.values()[0] if jsfunc.depth == self.depth and jsfunc.lineend == -1: jsfunc.lineend = self.lineno log.debug("Setting lineend: %d for scope %r", self.lineno, jsfunc) elif isinstance(self.currentScope, JSFunction) and \ self.currentScope.isAnonymous(): # Anonymous functions may be immediately invoked. # In these cases the contents actually belong to the # previous scope. For example, a top-level, invoked # anonymous function's contents belong in the file scope. self.anonFuncPending = True log.debug("Anonymous function %r's contents may apply to prior scope. Analyzing subsequent tokens for invocation...", self.currentScope.name) log.debug("Setting lineend: %d for scope %r", self.lineno, self.currentScope.name) else: log.debug("Current scope does not have a lineend: %r", self.currentScope.name) self._scopeStack.pop() #assert(len(self._scopeStack) > 0) if self._scopeStack[-1] != self.objectStack[-1]: self.objectStack.pop() #assert(len(self.objectStack) > 0) self.currentScope = self._scopeStack[-1] log.debug("decBlock: currentScope:%r", self.currentScope.name) if not self.currentScope: log.info("decBlock:: No currentScope available. Defaulting to global file scope.") # Use the global file scope then self.currentScope = self.cile return # Update currentClass variable oldCurrentClass = self.currentClass if isinstance(self.currentScope, JSClass): self.currentClass = self.currentScope log.debug("Currentclass now: %r", self.currentClass.name) elif isinstance(self.currentScope, JSFunction): self.currentClass = self.currentScope._class if self.currentClass: log.debug("Currentclass now: %r", self.currentClass.name) else: log.debug("Currentclass now: %r", self.currentClass) else: self.currentClass = None log.debug("Currentclass now: %r", self.currentClass) # Update line number for the current class if it doesn't have one already if oldCurrentClass and oldCurrentClass.lineend == -1 and \ oldCurrentClass != self.currentClass: oldCurrentClass.lineend = self.lineno else: # Likely there is a syntax error in the document log.debug("decBlock:: Scope already at 0. Document has syntax errors.") def _findInScope(self, name, attrlist=("variables", ), scope=None): """Find the object of the given name in the given scope @param name {str} The name of the object to look for @param attrlist {seq of str} The attributes to look into (e.g., "variables", "functions", "classes") @param scope {JSObject} The scope in which to find the object @returns {JSObject or None} The found object, an immediate child of the scope. """ assert scope is not None, "Missing scope" for attr in attrlist: namesDict = getattr(scope, attr, None) if namesDict: subscope = namesDict.get(name) if subscope: log.debug("_findInScope: Found a scope for: %r in %s.%s:", name, scope.name, attr) return subscope # Not found return None def _resolveAlias(self, namelist, scope=None): """Resolve the given alias @param namelist {seq of str} The path to the alias to resolve @param scope {JSObject} The starting scope; defaults to the current scope @returns {tuple of scope, {seq of str}} The scope and namelist for the (recursively) resolved namelist; alternatively, tuple (None, None) if the alias could not be resolved (e.g. not an alias) """ if not namelist: return (None, None) namelist = namelist[:] if scope is None: scope = self.currentScope lastScope, lastNamelist = scope, namelist[:] log.debug("_resolveAlias: Resolving alias %r in scope %r", namelist, scope.name) aliasDepth = 0 # prevent infinite loop found = False while namelist: log.debug("_resolveAlias: Looking for %r in scope %r", namelist, scope.name) namesDict = getattr(scope, "variables", None) if not namesDict: # this scope has no variables log.debug("_resolveAlias: scope %r has no variables", scope.name) break name = namelist.pop(0) foundVar = namesDict.get(name) if foundVar is None: # can't find the wanted name log.debug("_resolveAlias: scope %r does not have variable %r", scope.name, name) break if isinstance(foundVar, JSAlias) and aliasDepth < 10: lastScope = scope = foundVar.scope namelist = foundVar.target + namelist if namelist[-1].endswith("()"): # resolved to a function call; drop the call syntax namelist[-1] = namelist[-1][:-2] lastNamelist = namelist[:] aliasDepth += 1 log.debug("_resolveAlias: found alias %r on %r; new names %r", foundVar.name, scope.name, namelist) else: # found a non-alias variable scope = foundVar log.debug("_resolveAlias: finished resolve: scope %r namelist %r depth %r", lastScope.name, lastNamelist, aliasDepth) if aliasDepth == 0: return (None, None) return (lastScope, lastNamelist) def _locateScopeForName(self, namelist, attrlist=("variables", ), scope=None): """Find the scope referenced by namelist, or if it is not itself a scope, the containing scope @param namelist {seq of str} The path to the object, e.g. ["foo", "bar"] @param attrlist {seq of str} The types of attributes to look at @param scope {JSObject} The starting scope; defaults to the current scope @returns {JSObject} The object, if it is a scope; otherwise, the scope containing the object, if it is a variable. """ if not namelist: return None namelist = namelist[:] # copy if scope is None: scope = self.currentScope log.debug("Finding in scope: %s.%r with names: %r", scope.name, attrlist, namelist) # If variables are defined, also search members, which is the # correstponding group for class scopes. if "variables" in attrlist: attrlist += ("members", ) noVariables = False else: # add variables because aliases are stored there attrlist += ("variables", ) noVariables = True # Work up the scope stack looking for the classname aliasDepth = 0 firstAliasFound = None while scope: currentScope = scope log.debug("Looking in scope %r", currentScope.name) foundScope = None namePos = -1; while namePos < len(namelist) - 1: namePos = namePos + 1 name = namelist[namePos] #attrToLookIn = "variables" #if len(namelist) == 0: for attrToLookIn in attrlist: log.debug("Looking for name:%r in %s of scope with name:%r", name, attrToLookIn, currentScope.name) namesDict = getattr(currentScope, attrToLookIn, None) if namesDict: foundScope = namesDict.get(name) if isinstance(foundScope, JSAlias) and aliasDepth < 10: if firstAliasFound is None: firstAliasFound = foundScope # This is an alias; look again with its target namelist[:namePos+1] = foundScope.target currentScope = scope = foundScope.scope namePos = -1 aliasDepth += 1 log.debug("_locateScopeForName: encountered alias %r, restarting with %r on %r", name, namelist, scope.name) break # continue outer for loop elif attrToLookIn == "variables" and noVariables: # we didn't originally want to pick up variables continue if foundScope: log.debug("_locateScopeForName: Found scope %r for: %r", foundScope.name, name) # Look in this sub-scope if we have more names to check if type != "variables" or namePos < len(namelist) - 1: currentScope = foundScope # else we've located the scope we want break # goes to else: of outer for loop at end of namelist else: # Not found break else: log.debug("Found %r in scope:%s.%s", namelist, currentScope.name, attrToLookIn) return currentScope # Try parent scope scope = scope.parent if firstAliasFound: log.debug("Found alias, but no alias target, returning just the alias: %r", firstAliasFound.name) return firstAliasFound log.debug("NO scope found for: %r", namelist) return None ## # Create a JSFunction and add it to the current scope # @param namelist {list} list of names for the function # @param args {list} list of arguments for the function # @param doc {list} list of comment strings for given scope # def addFunction(self, namelist, args=None, doc=None, isLocal=False, isHidden=False): log.debug("AddFunction: %s(%s)", namelist, args) funcName = namelist[-1] toScope = self.currentScope if len(namelist) > 1: isLocal = False scopeNames = namelist[:-1] if "prototype" in namelist: pIndex = namelist.index("prototype") scopeNames = namelist[:pIndex] jsclass = self._addClassPart(funcName, self.ADD_CLASS_FUNCTION, scopeNames, args=args, doc=doc, path=self.path) # Ensure we onte the currentClass which we'll be working with self.currentClass = jsclass return else: toScope = self._findOrCreateScope(namelist[:-1], ('variables', 'classes', 'functions')) elif isinstance(toScope, JSFile): isLocal = False log.info("FUNC: %s(%s) isLocal:%r adding to %s %r", funcName, args, isLocal, toScope.cixname, toScope.name) #log.debug("jsdoc: %r", JSDoc("".join(doc))) fn = JSFunction(funcName, toScope, args, self.lineno, self.depth, doc=doc, isLocal=isLocal, isHidden=isHidden) toScope.functions[fn.name] = fn self.currentScope = fn # Special jsdoc parameter telling us explicitly that it's a class jsdoc_says_class = False if fn.jsdoc and fn.jsdoc.isClass(): jsdoc_says_class = True # Also check the last comment, sometimes it's meant for this scope if not jsdoc_says_class and self.last_comment_and_jsdoc[0]: last_jsdoc = self.last_comment_and_jsdoc[1] if last_jsdoc is None: last_jsdoc = JSDoc("".join(self.last_comment_and_jsdoc[0])) self.last_comment_and_jsdoc[1] = last_jsdoc if last_jsdoc.isClass() and \ fn.name == last_jsdoc.classname: # Name is same, check the namespace as well if it exists nspc = reversed(last_jsdoc.namespace.split(".")) scope = fn.parent for name in nspc: if scope is None or name != scope.name: break scope = scope.parent else: jsdoc_says_class = True fn.jsdoc = last_jsdoc log.debug("last_jsdoc classname: %r, namespace: %r", last_jsdoc.classname, last_jsdoc.namespace) if fn.name and jsdoc_says_class: # Ick, this is really a class constructor jsclass = self._convertFunctionToClass(fn) jsclass.doc = None jsclass.jsdoc = None def _createAnonymousFunctionName(self): self._anonid += 1 return "(anonymous %d)" % (self._anonid, ) ## # Create an anonymous JSFunction and add it to the current scope. # @param args {list} list of arguments for the function # @param doc {list} list of comment strings for given scope # def addAnonymousFunction(self, args=None, doc=None, isHidden=False): name = self._createAnonymousFunctionName() log.info("addAnonymousFunction: %s(%s)", name, args) toScope = self.currentScope fn = JSFunction(name, toScope, args, self.lineno, self.depth, doc=doc, isLocal=True, isHidden=isHidden, path=self.path) toScope.anonymous_functions.append(fn) self.currentScope = fn return fn ## # Create a JSFunction and add it to the current scope # @param namelist {list} list of names for the function # @param args {list} list of arguments for the function # @param doc {list} list of comment strings for given scope # def addClassFunction(self, namelist, args=None, doc=None): log.debug("AddClassFunction: %s(%s)", namelist, args) toScope = self.currentClass if not toScope: # See if it's a function, we'll convert it into a class then if isinstance(self.currentScope, JSFunction): toScope = self._convertFunctionToClass(self.currentScope) if not toScope or len(namelist) > 1: self.addFunction(namelist, args, doc) else: funcName = namelist[-1] log.info("FUNC: %s(%s) on line %d", funcName, args, self.lineno) fn = JSFunction(funcName, toScope, args, self.lineno, self.depth, doc=doc) toScope.functions[fn.name] = fn self.currentScope = fn ADD_CLASS = 0 ADD_CLASS_MEMBER = 1 ADD_CLASS_VARIABLE = 2 ADD_CLASS_FUNCTION = 3 ADD_CLASS_PARENT = 4 ADD_CLASS_CONSTRUCTOR = 5 def _addClassPart(self, partName, addType, scopeNames=None, args=None, doc=None, path=None, varCtor=JSVariable): """Add something to this class @param partName {unicode} The name of the thing to add @param addType {int} What sort of thing to add; can be one of ADD_CLASS (add a new child class), ADD_CLASS_MEMBER (a member varaible), ADD_CLASS_VARIABLE (a member variable), ADD_CLASS_FUNCTION (a method), ADD_CLASS_PARENT (an object on this class's prototype chain), ADD_CLASS_CONSTRUCTOR (a constructor function) @param scopeNames {list of unicode} The expression to locate the part @param args {JSArguments} The arguments to a method when using ADD_CLASS_FUNCTION @param doc {list of unicode} The documentation comment for this part @param path {list of unicode} @param varCtor {callable} The constructor function to create a variable for use with ADD_CLASS_MEMBER / ADD_CLASS_VARIABLE; the signature should match that of JSVariable """ log.debug("_addClassPart: partName:%r, addType:%r, scopeNames:%r, args:%r", partName, addType, scopeNames, args) jsclass = None fn = None # Find the class to place this part into #jsclass = self._findClassWithNames(scopeNames) if scopeNames: # Look for the class first, then if we don't find it look for # a function or variable: bug 70324 jsclass = self._locateScopeForName(scopeNames, attrlist=("classes", )) # Note: We could have an alias object, in that case we look for # something better, see test "variable_aliasing_komodo.js" if jsclass is None or not isinstance(jsclass, JSClass): jsclass = self._locateScopeForName(scopeNames, attrlist=("classes", "functions", "variables", )) if isinstance(jsclass, JSFunction): # Convert it to a class jsclass = self._convertFunctionToClass(jsclass) else: jsclass = self.currentClass if not jsclass and scopeNames: if len(scopeNames) > 1: toScope = self._findOrCreateScope(scopeNames, attrlist=("classes", "functions", "variables", )) else: toScope = self.currentScope className = scopeNames[-1] jsclass = JSClass(className, toScope, self.lineno, self.depth, doc=doc, path=path) self.currentScope.classes[jsclass.name] = jsclass log.info("CLASS: %r on line %d in %r at depth %d", jsclass.name, jsclass.line, self.currentScope.name, self.depth) self.currentScope = jsclass if addType == self.ADD_CLASS_FUNCTION: log.info("CLASS_FUNC: %s(%s) on line %d", partName, args, self.lineno) fn = JSFunction(partName, jsclass, args, self.lineno, self.depth, doc=doc) fn._class = jsclass jsclass.functions[fn.name] = fn #print "num functions: %d" % (len(jsclass.functions)) self.currentScope = fn elif addType == self.ADD_CLASS_MEMBER: if partName not in jsclass.variables: log.info("CLASS_MBR added: %r", partName) v = varCtor(partName, jsclass, self.lineno, self.depth, doc=doc, path=self.path) jsclass.variables[partName] = v else: log.info("CLASS_MBR already exists: %r", partName) elif addType == self.ADD_CLASS_VARIABLE: if partName not in jsclass.variables: log.info("CLASS_VBR added: %r", partName) v = varCtor(partName, jsclass, self.lineno, self.depth, doc=doc, path=self.path) jsclass.variables[partName] = v else: log.info("CLASS_MBR already exists: %r", partName) elif addType == self.ADD_CLASS_PARENT: log.info("CLASS_PARENT: %r", partName) jsclass.addClassRef(partName) elif addType == self.ADD_CLASS_CONSTRUCTOR: log.info("CLASS_CTOR: %r", partName) jsclass.constructor = partName if jsclass: self.currentClass = jsclass if addType == self.ADD_CLASS: self.currentScope = jsclass elif addType == self.ADD_CLASS_PARENT and partName == "Object": self.currentScope = jsclass return jsclass # a class part using prototype.name = function #def addClassPart(self): # self._addClassPart() # a class using classname.prototype = { ... } def addClass(self, namelist, doc=None, path=None, varCtor=JSVariable): """Add a new class using scope.classname.prototype = {...} @param namelist {list of unicode} The path to get to the class; in the example above, ["scope", "classname"] @param doc {list of unicode} Documentation comment associated with this class @param varCtor {callable} The constructor function to create a variable for use with ADD_CLASS_MEMBER / ADD_CLASS_VARIABLE; the signature should match that of JSVariable """ jsclass = self._addClassPart(namelist[-1], self.ADD_CLASS, scopeNames=namelist, doc=doc, path=path, varCtor=varCtor) return jsclass def addAnonymousClass(self, namelist, doc=None): # Example syntax: c.prototype = { rows: { return this._rows.length; } } self.addClass(namelist[:-1], doc=doc) def addClassOrVariableMember(self, namelist, typeNames, scope=None, doc=None, assignAsCurrentScope=False, isLocal=False, varCtor=JSVariable): """Add the variable to the given scope or current scope If the scope is a * variable or object: add as a variable to this scope. * class: add as a member to this class. * function: then this is a little more tricky: * If it's class function, then add as a member for the class. * If it's a function inside a variable/object, then add as a variable to the variable/object. * If it's just a function on it's own, turn the function into a class and then add a member variable for the class. """ if not scope: scope = self.currentScope log.debug("addClassOrVariableMember: namelist:%r, type:%r, isLocal:%r, scope (%s):%s", namelist, typeNames, isLocal, scope.cixname, scope.name) memberName = namelist[-1] if len(namelist) > 2 and "prototype" in namelist: pIndex = namelist.index("prototype") scopeNames = namelist[:pIndex] log.debug("Adding class prototype. class name: %r, variable: %r", scopeNames, memberName) scope = self._addClassPart(memberName, self.ADD_CLASS_MEMBER, scopeNames=scopeNames, args=None, doc=doc, path=self.path, varCtor=varCtor) v = scope elif len(namelist) >= 2: # Find the scope to apply to. scope = self._findOrCreateScope(namelist[:-1], attrlist=("variables", "classes"), fromScope=scope) v = varCtor(memberName, scope, self.lineno, self.depth, vartype=typeNames, doc=doc, isLocal=isLocal, path=self.path) v = scope.addVariable(memberName, value=v) if assignAsCurrentScope: self.currentScope = v elif scope.cixname in ("object", "variable"): if isLocal: log.warn("addClassOrVariableMember: %s:%d Trying to add %r as " "a local member variable??", self.cile.name, self.lineno, namelist) return v = varCtor(memberName, scope, self.lineno, self.depth, vartype=typeNames, doc=doc, isLocal=isLocal, path=self.path) v = scope.addVariable(memberName, value=v) if assignAsCurrentScope: self.currentScope = v # Special case - classes and anonymous functions, but not anonymous # classes (which as a parent of an anon function will have the same # name). elif isinstance(scope, JSClass) or \ (scope.isAnonymous() and not scope.parent.name == scope.name): v = varCtor(memberName, scope, self.lineno, self.depth, vartype=typeNames, doc=doc, isLocal=isLocal, path=self.path) v = scope.addMemberVariable(memberName, value=v) if assignAsCurrentScope: self.currentScope = v elif isinstance(scope, JSFunction): # If it's a function already within a class, then thats okay parentScope = scope.parent if not parentScope: log.debug("addClassOrVariableMember: ignoring assignment %r " "into a dummy function", namelist) return None log.debug("ParentScope is: %s (%s)", parentScope.name, parentScope.cixname) if isinstance(parentScope, JSClass): self.currentClass = parentScope log.debug("Assigning to parent class: %r:%r", parentScope.cixname, parentScope.name) v = varCtor(memberName, parentScope, self.lineno, self.depth, typeNames, doc=doc, isLocal=isLocal, path=self.path) v = parentScope.addMemberVariable(memberName, value=v) if assignAsCurrentScope: self.currentScope = v # If it's a function within a variable, then thats okay too elif parentScope and parentScope.cixname in ("object", "variable"): log.debug("Assigning to parent scope: %r:%r", parentScope.cixname, parentScope.name) v = varCtor(memberName, parentScope, self.lineno, self.depth, vartype=typeNames, doc=doc, isLocal=isLocal, path=self.path) v = parentScope.addVariable(memberName, value=v) # We need to keep track of what we assign in this particular # case, as we may later turn this function into it's own class, # and then we'll need to grab these "this." variables back! # Example code: # var ko = {} # ko.f1 = function() { this.x = 1; } // x assigned to ko # ko.f1.prototype.run = function() {} // convert f1 to class # // Now we want ko.x to move into class ko.f1 scope._parent_assigned_vars.append(v) if assignAsCurrentScope: self.currentScope = v # Convert the function to class then else: # If the class name exists already, assign to that class func = scope funcName = func.name jsclass = self._locateScopeForName([funcName], attrlist=("classes", ), scope=scope) if not jsclass: log.debug("Creating class %r, function %r now ctor", funcName, funcName) # Turn function into a constructor for the class jsclass = self._convertFunctionToClass(func) else: # Update the function class information self._convertFunctionToClassContructor(func, jsclass) v = varCtor(memberName, jsclass, self.lineno, self.depth, typeNames, doc=doc, isLocal=isLocal, path=self.path) v = jsclass.addMemberVariable(memberName, value=v) if assignAsCurrentScope: self.currentScope = v elif isinstance(scope, JSFile): v = self.addVariable(namelist, typeNames, scope, doc, assignAsCurrentScope, isLocal, varCtor=varCtor) else: log.info("addClassOrVariableMember:: Invalid scope type. Could not add %r to scope: %r - %r", namelist, scope.cixname, scope.name) v = None return v def addClassParent(self, namelist, typeNames): log.debug("addClassParent: namelist:%r, typeNames:%r", namelist, typeNames) self._addClassPart(".".join(typeNames), self.ADD_CLASS_PARENT, namelist[:-1], path=self.path) def addGetter(self, namelist, typeNames, scopeNames=None, doc=None): log.debug("addGetter: namelist:%r, type: %r, scopeNames: %r", namelist, typeNames, scopeNames) if scopeNames: toScope = self._locateScopeForName(scopeNames, attrlist=("variables", "classes")) if not toScope: log.info("addGetter:: Not adding getter. Could not find scope for: %r", scopeNames) return self.currentScope = toScope else: toScope = self.currentScope self.addClassOrVariableMember(namelist, typeNames, toScope, doc=doc) def addSetter(self, namelist, scopeNames=None, doc=None): log.debug("addSetter: namelist:%r, scopeNames: %r", namelist, scopeNames) if scopeNames: toScope = self._locateScopeForName(scopeNames, attrlist=("variables", "classes")) if not toScope: log.info("addSetter:: Not adding setter. Could not find scope for: %r", scopeNames) return self.currentScope = toScope else: toScope = self.currentScope self.addClassOrVariableMember(namelist, [], toScope, doc=doc) def _convertFunctionToClassContructor(self, jsfunc, jsclass): # Mark it as a constructor if it's not already so marked funcName = jsfunc.name # Copy attributes across, except for "__ctor__" class_attributes = jsfunc.attributes[:] if "__ctor__" not in jsfunc.attributes: jsfunc.attributes.append("__ctor__") else: class_attributes.remove("__ctor__") jsclass.attributes = class_attributes # Might already be the contructor for the class if funcName not in jsclass.functions: parentScope = jsfunc.parent log.debug("Converting function: %r into a class contructor for: %r", funcName, jsclass.name) jsclass.functions[funcName] = jsfunc # Update references if jsfunc.isAnonymous(): parentScope.anonymous_functions.remove(jsfunc) parentScope.anonymous_functions.append(jsclass) else: parentScope.functions.pop(funcName) parentScope.classes[funcName] = jsclass # Fix starting line number if jsfunc.line < jsclass.line: jsclass.line = jsfunc.line # Copy over non-argument variables from the function to the class. # All the local variables are closures and should be in the class # scope. for varName, v in jsfunc.variables.items(): if not isinstance(v, JSArgument): # Add to class and remove from the function v.setParent(jsclass) jsclass.variables[varName] = v del jsfunc.variables[varName] parent = jsfunc.parent for var in jsfunc._parent_assigned_vars: log.debug("Converting function: Moved parent assigned variable %r " "into the class instance", var.name) jsclass.members[var.name] = var parent.variables.pop(var.name, None) # Copy across all non-local members, bug 88549. for name, jsobject in jsfunc.variables.items(): if '__local__' not in jsobject.attributes and not isinstance(jsobject, JSArgument): jsclass.variables[name] = jsobject jsfunc.variables.pop(name, None) for name, jsobject in jsfunc.functions.items(): if '__local__' not in jsobject.attributes: jsclass.functions[name] = jsobject jsfunc.functions.pop(name, None) for name, jsobject in jsfunc.classes.items(): if '__local__' not in jsobject.attributes: jsclass.classes[name] = jsobject jsfunc.classes.pop(name, None) jsfunc._parent_assigned_vars = [] jsfunc._class = jsclass jsfunc.setParent(jsclass) return jsclass def _convertFunctionToClass(self, jsfunc): """Convert the provided JSFunction into a JSClass and return it.""" funcName = jsfunc.name log.debug("Creating class %r, from function %r", funcName, funcName) jsclass = JSClass(funcName, jsfunc.parent, jsfunc.line, self.depth - 1, jsfunc.doc) self._convertFunctionToClassContructor(jsfunc, jsclass) if self.currentScope == jsfunc: self.currentClass = jsclass # Copy across the classrefs from the jsdoc (if any). if jsfunc.jsdoc and jsfunc.jsdoc.baseclasses: for baseclass in jsfunc.jsdoc.baseclasses: jsclass.addClassRef(baseclass) jsfunc.jsdoc.baseclasses = [] return jsclass def _convertFunctionToClosureVariable(self, jsfunc): funcName = jsfunc.name log.info("Creating variable %r, from function closure %r", funcName, funcName) jsvariable = JSVariable(funcName, jsfunc.parent, jsfunc.line, jsfunc.depth, jsfunc.type, jsfunc.doc, path=self.path) if jsfunc.returnTypes: jsro = jsfunc.returnTypes[0] #print jsro if isinstance(jsro, JSVariable): # Convert this object into the variable jsro.setParent(jsfunc.parent) jsro.line = jsfunc.line jsro.name = funcName jsvariable = jsro parent = jsfunc.parent if not jsfunc.isAnonymous(): parent.functions.pop(funcName) parent.variables[funcName] = jsvariable return jsvariable def _findOrCreateScope(self, namelist, attrlist=("variables", ), fromScope=None, isLocal=False): # Don't create a window scope - bug 87442. global_var = { "JavaScript": "window", "Node.js": "global", }.get(self.lang) if namelist[0] == global_var: fromScope = self.cile namelist = namelist[1:] if not namelist: return fromScope # Ensure the scope exists, else create it # Find the base scope first if fromScope is None: fromScope = self.currentScope log.debug("_findOrCreateScope: %r, attrlist: %r, from scope: %s", namelist, attrlist, fromScope.name) name = namelist[0] # Determine where variables get added when they are not found if isLocal: applyToScope = fromScope else: applyToScope = self.cile # Global file level resolvedScope, resolvedNamelist = self._resolveAlias(namelist, fromScope) if resolvedScope is not None and resolvedNamelist: # don't use the resolved namelist if it resolves to an undeclared # global object; doing so can cause us to shadow things found in the # standard library if resolvedScope is not self.cile or \ self._findInScope(resolvedNamelist[0], attrlist, resolvedScope) is not None: fromScope, namelist = resolvedScope, resolvedNamelist isTheFirstName = True for name in namelist: # When looking for the first name of the scope, traverse the parent # chain, subsequent names *must* reside within the current scope # being checked! if isTheFirstName: isTheFirstName = False scope = self._locateScopeForName([name], attrlist, fromScope) else: scope = self._findInScope(name, attrlist, fromScope) if not scope: v = JSVariable(name, applyToScope, self.lineno, self.depth, vartype="Object", path=self.path) scope = applyToScope.addVariable(name, value=v) log.info("Could not find %r in scope: %r, creating variable (type=Object) for it on scope %r!!!", name, fromScope.name, applyToScope.name) scope.attributes.append("__file_local__") fromScope = scope applyToScope = scope return fromScope def addVariable(self, namelist, typeNames, toScope=None, doc=None, assignAsCurrentScope=False, isLocal=False, value=None, varCtor=JSVariable): varName = namelist[-1] if toScope is None: toScope = self.currentScope log.debug("addVariable: %r, typeNames:%r, isLocal: %r, scope: %r", namelist, typeNames, isLocal, toScope.name) if len(namelist) > 1: if namelist[-2] == "prototype": # Adding to an existing class then toScope = self._locateScopeForName(namelist[:-2], attrlist=("classes", )) if not toScope: # Create a class for it then log.debug("Creating class now: %r", namelist[:-2]) toScope = self.addClass(namelist[:-2], doc=doc, path=self.path, varCtor=varCtor) #raise CodeIntelError("Could not find scope for: %r" % (namelist[:-2], )) if varName == "constructor": if isinstance(typeNames, JSObject): ctorName = typeNames.type else: ctorName = ".".join(typeNames) func = self._locateScopeForName([ctorName], attrlist=("functions", )) if func: return self._convertFunctionToClassContructor(func, toScope) else: return self._addClassPart(ctorName, self.ADD_CLASS_CONSTRUCTOR, namelist[:-2], doc=doc, path=self.path, varCtor=varCtor) else: return self._addClassPart(varName, self.ADD_CLASS_VARIABLE, namelist[:-2], doc=doc, path=self.path, varCtor=varCtor) else: # Find or create the parent scope toScope = self._findOrCreateScope(namelist[:-1], ('variables', 'classes', 'functions'), fromScope=toScope, isLocal=isLocal) elif not isLocal: # Try and find the scope we are assigning to, should be in # a parent scope somewhere! #print("addVariable: namelist:%r, typeNames:%r, isLocal: %r, line: %d" % (namelist, typeNames, isLocal, self.lineno)) fromscope = toScope toScope = self._locateScopeForName(namelist, ('variables', 'classes', 'functions'), toScope) if toScope is None: #if self.text[0] not in ("var", "const"): # sys.stderr.write("Undeclared var in %s:%d, %r in %s %r\n" % ( # self.cile.name, # self.lineno, varName, fromscope.cixname, fromscope.name)) # Place it at the global level then toScope = self.cile else: toScope = toScope.parent # Add it to scope if it's not already in there if toScope: if isinstance(toScope, JSVariable): # toScope.type can be None if it's an implicitly defined scope # (i.e. ones we picked up by observing properties being set on it) if toScope.type is not None and toScope.type.lower() not in ("object", ): # Not going to add sub-variables, as it's likely a class # object already, which has this variable information set #if not toScope.type: # msg = "Assignment to a unknown type, %s:%d, %r (%s)" % (self.filename, self.lineno, ".".join(namelist), toScope.type) # print >> sys.stderr, msg return None #if not isLocal and varName not in toScope.variables: # print("addVariable: namelist:%r, typeNames:%r, isLocal: %r, line: %d" % (namelist, typeNames, isLocal, self.lineno)) if value is None: value = varCtor(varName, toScope, self.lineno, self.depth, vartype=typeNames, doc=doc, isLocal=isLocal, path=self.path) v = toScope.addVariable(varName, value=value, metadata=self._metadata) if assignAsCurrentScope: self.currentScope = v return v # We kinda lost the scope somewhere return None def addObjectVariable(self, namelist, toScope=None, doc=None, isLocal=False): if not toScope: toScope = self.currentScope log.debug("addObjectVariable: namelist:%r, scope:%r", namelist, toScope.name) varName = namelist[-1] if len(namelist) > 1: # Ensure the scope exists, else create it if "prototype" in namelist: classnames = namelist[:namelist.index("prototype")] toScope = self.addClass(classnames, doc=self.comment) else: toScope = self._findOrCreateScope(namelist[:-1], ("variables", "classes", "functions"), toScope) # Assignment to a function, outside the function scope... create a class for it if isinstance(toScope, JSFunction): toScope = self._convertFunctionToClass(toScope) # Add it to scope if it's not already in there v = JSVariable(varName, toScope, self.lineno, self.depth, vartype=["Object"], doc=doc, isLocal=isLocal, path=self.path) v = toScope.addVariable(varName, value=v) self.currentScope = v def addReturnObject(self, doc=None): log.debug("addReturnObject: scope:%r", self.currentScope.name) jsro = JSVariable("", self.currentScope, self.lineno, self.depth, vartype="Object", doc=doc, path=self.path) if isinstance(self.currentScope, JSFunction): self.currentScope.addReturnType(jsro) # else: # TODO: This is ignoring the return type for function getters. self.currentScope = jsro return jsro def addFunctionReturnType(self, typeNames, doc=None): if isinstance(self.currentScope, JSFunction): log.debug("addFunctionReturnType: type: %r, scope:%r", typeNames, self.currentScope.name) self.currentScope.addReturnType(".".join(typeNames)) ## # Read everything up to and including the matching close paren # @param styles list # @param text list # @param p int position in the styles and text list # @param paren string type of parenthesis def _getParenArguments(self, styles, text, p, paren=None): # Examples: # (arg1, arg2) { # => [(arg1, arg2)] # [row][this.columns[column][0]]; # => [row] if paren is None: paren = text[p] parenMatches = { '{': '}', '[': ']', '(': ')' } args = [] oppParen = parenMatches.get(paren) if oppParen is None: log.info("_getParenArguments:: No matching paren for: %r, " \ "ignoring arguments.", paren) return args, p parenCount = 0 while p < len(styles): args.append(text[p]) if styles[p] == self.JS_OPERATOR: if text[p] == paren: parenCount += 1 elif text[p] == oppParen: parenCount -= 1 if parenCount <= 0: p += 1 break p += 1 return args, p def _skipOverParenArguments(self, styles, text, p, paren="("): args, p = self._getParenArguments(styles, text, p, paren) return p # Skip over all the variable assignment details. Returns position at # the end of the assignment, usually a "," or a ";" character. # # Examples: # var nGroupIndex = typeof <|>p_nGroupIndex=="number" ?p_nGroupIndex :0, # aGroup = this._getItemGroup(nGroupIndex); # should skip to "aGroup = this._getItemGroup(nGroupIndex);" def _skipToEndOfVariableAssignment(self, styles, text, p): old_p = p while p < len(styles): style = styles[p] if style == self.JS_OPERATOR: t = text[p] if t in '([{': p = self._skipOverParenArguments(styles, text, p, t) continue elif t in ',;': break p += 1 if old_p == p: # Ensure we at least move somewhere (avoid recursion) p += 1 log.debug("_skipToEndOfVariableAssignment:: skipped text %r, p: %d", text[old_p:p], p) return p def _getArgumentsFromPos(self, styles, text, pos): log.debug("_getArgumentsFromPos: text: %r", text[pos:]) if pos < len(styles) and styles[pos] == self.JS_OPERATOR and text[pos] == "(": ids = [] pos += 1 start_pos = pos while pos < len(styles): if styles[pos] == self.JS_IDENTIFIER: ids.append(text[pos]) if text[pos - 1] == '.': ids[-1] = '...' + ids[-1] # ES6 rest operator (...arg) elif styles[pos] == self.JS_OPERATOR and text[pos] == '=': # Skip over default argument value. # TODO: complicated default values that include commas will # not be parsed correctly. while pos < len(styles) and not \ (styles[pos] == self.JS_OPERATOR and \ (text[pos] == ',' or text[pos] == ')')): pos += 1 continue elif styles[pos] == self.JS_OPERATOR and text[pos] == '.' and \ pos + 2 <= len(styles) and text[pos + 1] == '.' and \ text[pos + 2] == '.': pos += 2 # handle ES6 rest operator (...arg) elif styles[pos] != self.JS_OPERATOR or text[pos] != ",": break pos += 1 return ids, pos return None, pos def _getIdentifiersFromPos(self, styles, text, pos): log.debug("_getIdentifiersFromPos: text: %r", text[pos:]) start_pos = pos ids = [] last_style = self.JS_OPERATOR while pos < len(styles): style = styles[pos] if style in (self.JS_IDENTIFIER, self.JS_WORD): if last_style != self.JS_OPERATOR: break ids.append(text[pos]) elif style == self.JS_OPERATOR and text[pos] == "[" and \ len(styles) - pos > 2 and styles[pos + 1] in self.JS_STRINGS and \ text[pos + 2] == "]": # blah["string here"] part = text[pos + 1] if len(part) > 1 and part[0] in ("'", '"') and part[-1] == part[0]: part = part[1:-1] # trim quotes ids.append(part) pos += 2 elif style != self.JS_OPERATOR or text[pos] != "." or \ last_style != self.JS_IDENTIFIER: break pos += 1 last_style = style return ids, pos ## # Grab all necessary citdl information from the given text # @param styles list # @param text list # @param p int position in the styles and text list # @return the citdl list and the position after the last item swallowed def _getCitdlTypeInfo(self, styles, text, p): log.debug("_getCitdlTypeInfo:: text: %r", text[p:]) citdl = [] last_style = self.JS_OPERATOR while p < len(styles): style = styles[p] #log.debug("p: %d, text[p]: %r", p, text[p]) #print "style: %d, last_style: %d" % (style, last_style) if style in (self.JS_IDENTIFIER, self.JS_WORD): if last_style != self.JS_OPERATOR: break citdl.append(text[p]) style = self.JS_IDENTIFIER elif style == self.JS_OPERATOR and last_style == self.JS_IDENTIFIER: if text[p] == ".": pass elif text[p] == ")": # Collect after the closing brace - bug 80581. style = last_style elif text[p] == "(": paren_pos = p if citdl == ['require']: # Deal with CommonJS (NodeJS) require statements. args, p = self._getParenArguments(styles, text, p) if len(args) >= 3 and styles[paren_pos+1] in self.JS_STRINGS: self._metadata['required_library_name'] = self._unquoteJsString(args[1]) log.debug("Dealing with CommonJS require(%s)", self._metadata['required_library_name']) else: p = self._skipOverParenArguments(styles, text, p) if citdl: if len(citdl) > 1 and 'QueryInterface' == citdl[-1]: # QueryInterface is specific to xpcom interfaces. citdl = [] # Don't want the "." items in the citdl for t in text[paren_pos+1:p-1]: if t != ".": citdl.append(t) else: citdl[-1] = citdl[-1] + "()" style = self.JS_IDENTIFIER p -= 1 # Are at the pos after the paren, move back to it elif text[p] == "[": # Arrays, just read in the arguments and add it to the citdl args, p = self._getParenArguments(styles, text, p, "[") if args and citdl: # Check if this is an xpcom component. if citdl in (["CC"], ["Cc"], ["Components", "classes"]) and \ (p+2) < len(styles) and \ text[p] == "." and \ text[p+1] in ("getService", "createInstance") and \ text[p+2] == "(": # Add the xpcom interface information. # TODO: Change this once array completions are # supported citdl, p = self._getArgumentsFromPos(styles, text, p+2) else: citdl[-1] = citdl[-1] + "".join(args) style = self.JS_IDENTIFIER p -= 1 # We are are after the last "]", move back else: break else: break p += 1 last_style = style return citdl, p def _getVariableType(self, styles, text, p, assignmentChar="="): """ Get the type of the variable @param styles {list} The (scintilla) styles for the text @param text {list} The tokens to examine @param p {int} Offset into text/styles to start looking @param assignmentChar {str} The assignment character used @returns {tuple} {list} type names, {int} new offset (p), {bool} true if this is a new instance """ log.debug("_getVariableType: text: %r, assign char: %s", text[p:], assignmentChar) typeNames = [] if p >= len(styles): # Nothing left to examine return typeNames, p, False if assignmentChar and styles[p] == self.JS_OPERATOR and \ text[p] == assignmentChar: # Assignment to the variable p += 1 if p < len(styles) and styles[p] == self.JS_OPERATOR and \ text[p] in "+-": # Example: var x = -1; # Skip over + and -, commonly used with strings and integers p += 1 isNew = False isAlias = False if p < len(styles): if styles[p] == self.JS_WORD: # Keyword keyword = text[p] if keyword == "new": typeNames, p = self._getCitdlTypeInfo(styles, text, p+1) p -= 1 # We are already at the next position, step back # When resolving a class, we want to drop the function call # from the citdl and just use the identifier itself. for i, t in enumerate(typeNames): if t.endswith("()"): typeNames[i] = t[:-2] break #if not typeNames: # typeNames = ["object"] isNew = True elif keyword in ("true", "false"): typeNames = ["boolean"] elif keyword == "this": typeNames, p = self._getCitdlTypeInfo(styles, text, p) p -= 1 # We are already at the next position, step back # Don't record null, as it doesn't help us with anything #elif keyword == "null": # typeNames = ["null"] p += 1 elif styles[p] in self.JS_STRINGS: typeNames = ["string"] p += 1 elif styles[p] == self.JS_NUMBER: typeNames = ["int"] p += 1 elif styles[p] == self.JS_IDENTIFIER: typeNames, p = self._getCitdlTypeInfo(styles, text, p) if not isNew: # this refers to an identifier without "new", it's an alias isAlias = True elif styles[p] == self.JS_OPERATOR: if text[p] == "{": # This is actually a newly created object typeNames = ["Object"] p += 1 elif text[p] == "[": while p+1 < len(styles): if text[p] == "]" and styles[p] == self.JS_OPERATOR: break p += 1 typeNames = ["Array"] p += 1 return typeNames, p, isAlias def _unquoteJsString(self, s): """Return the string without quotes around it""" return Utils.unquoteJsString(s) def _getVariableDetail(self, namelist, styles, text, p, assignmentChar="="): # this.myname = "123"; # myclass.prototype.list = function () { # this.myname = new function(x, y) { # var num = mf.field1; # names = { "myname": 1, "yourname": 2 } log.debug("_getVariableDetail: namelist: %r, text:%r", namelist, text[p:]) if len(namelist) > 1 and "prototype" in namelist: # Check for special class prototypes protoName = namelist[-1] if protoName == "prototype": typeNames, p, isAlias = self._getVariableType(styles, text, p, assignmentChar) return (TYPE_PARENT, typeNames, None, p) elif namelist[-2] == "prototype": typeNames = [] if p+1 < len(styles) and styles[p+1] in self.JS_STRINGS: typeNames = [self._unquoteJsString(text[p+1])] if protoName == "__defineGetter__": return (TYPE_GETTER, typeNames, None, p) elif protoName == "__defineSetter__": return (TYPE_SETTER, typeNames, None, p) elif len(namelist) == 1 and p+1 < len(styles) and \ styles[p] == self.JS_IDENTIFIER: keyword = namelist[0] if keyword == "get": # get log() { newnamelist, p = self._getIdentifiersFromPos(styles, text, p) namelist.pop() for name in newnamelist: namelist.append(name) log.debug("Found getter:%r", namelist) return (TYPE_GETTER, [], None, p) elif keyword == "set": # set application(value) { newnamelist, p = self._getIdentifiersFromPos(styles, text, p) namelist.pop() for name in newnamelist: namelist.append(name) log.debug("Found setter:%r", namelist) return (TYPE_SETTER, [], None, p) if p+1 < len(styles) and styles[p+1] == self.JS_OPERATOR and text[p+1] == "{": # This is actually a newly created object return (TYPE_OBJECT, [], None, p+2) if p+2 < len(styles) and styles[p+1] == self.JS_WORD and \ text[p+1] == "function": # Skip over any function name # Example: var f = function my_f(a, b) { } p += 2 while p < len(styles): if text[p] == "(": break p += 1 args, p = self._getArgumentsFromPos(styles, text, p) return (TYPE_FUNCTION, [], args, p) if p+3 < len(styles) and styles[p+1] == self.JS_WORD and \ text[p+1] == "new" and text[p+2] == "function": # Skip over any function name # Example: var f = new function my_f(a, b) { } p += 3 while p < len(styles): if text[p] == "(": break p += 1 args, p = self._getArgumentsFromPos(styles, text, p) return (TYPE_FUNCTION, [], args, p) if "=>" in text[p:]: # Check for ES6 fat arrow function # Example: var f = (arg1, arg2) => { body } # var g = arg_only => expr arrow_index = text.index("=>") maybe_args = text[p + 1:arrow_index] if maybe_args[0] == "(" and maybe_args[-1] == ")": # Maybe have args... args, new_p = self._getArgumentsFromPos(styles, text, p + 1) if args is not None: return (TYPE_FUNCTION, [], args, arrow_index) elif arrow_index == p + 2 and styles[p+1] == self.JS_IDENTIFIER: # single argument return (TYPE_FUNCTION, [], [text[p+1]], arrow_index) typeNames, p, isAlias = self._getVariableType(styles, text, p, assignmentChar) if len(namelist) > 2 and namelist[-2:] == ["prototype", "constructor"]: # Foo.prototype.constructor = bar; don't treat as an alias pass elif isAlias and typeNames != namelist: log.debug("_getVariableDetail: %r is an alias to %r", namelist, typeNames) return (TYPE_ALIAS, typeNames, None, p) return (TYPE_VARIABLE, typeNames, None, p) def _variableHandler(self, lineno, styles, text, p, namelist, allowedAssignmentChars="=", isLocal=False): log.debug("_variableHandler:: namelist:%r, p:%d, isLocal: %r", namelist, p, isLocal) #print "p:", p #print "text:", text[p:] # The while loop is used to handle multiple variable assignments. # Example1: # var x = 1, y = 2, z = 3; # namelist: ['x'] # text: ['=', '1', ',', 'y', '=', '2', ',', 'z', '=', '3', ';'] # # Example2: # var x = y = z = 1; # namelist: ['x'] # text: ['=', 'y', '=', 'z', '=', '1', ';'] # # Example3: # var x, y, z = 1; # namelist: ['x'] # text: [',', 'y', ',', 'z', '=', '1', ';'] # # Example4: # this.ab['xyz']={one:1,"two":2}; # namelist: ['this', 'ab'] # text: ['[', "'xyz'", ']', '=', '{']] # already_looped = False while p < len(styles): self._metadata = {} #log.debug("_variableHandler:: p: %d, text: %r", p, text[p:]) if already_looped: # We've already done one loop, need to get a new namelist # text: [',', 'y', '=', '2', ',', 'z', '=', '3'] #log.debug("_variableHandler:: already_looped:: text:%r, p:%d", text[p:], p) if text[p] == "=" and styles[p] == self.JS_OPERATOR and len(typeNames) > 0: # Assignment to an assignment (aka Example 2) namelist = typeNames elif text[p] != "," or styles[p] != self.JS_OPERATOR: p = self._skipToEndOfVariableAssignment(styles, text, p) if p < len(styles) and text[p] == ";": p += 1 continue else: # Multiple assignment (aka Example 1) namelist, p = self._getIdentifiersFromPos(styles, text, p+1) # The namelist may contain array assignments that we cannot # deal with, check to ensure we have a variable assignment that # we can deal with. # Handled array assignment examples: # this['field1'] = # Unhandled array assignment examples: # this[0] = # myvar[unknownvar] = # new_namelist = [] for name in namelist: if '[' in name: name_parts = name.split('[') for subname in name_parts: if subname.endswith("]"): subname = subname[:-1] # ensure the array reference is a string, otherwise # we cannot do anything with it. if not subname or subname[0] not in "'\"": log.debug("_variableHandler:: Ignoring non-string indexed array assignment: %r", namelist) return subname = self._unquoteJsString(subname) new_namelist.append(subname) else: new_namelist.append(name) namelist = new_namelist log.debug("_variableHandler:: already_looped:: namelist now:%r, p:%d", namelist, p) if len(namelist) < 1: log.debug("_variableHandler:: Invalid namelist! Text: %r", text) return # Whether the namelist includes an array piece whose name/scope # could not be determined (i.e. foo[myname] = 1), as myname is # an unknown. unknown_array_namelist = False if p >= len(styles) or text[p] in ",;": # It's a uninitialized variable? already_known = False if len(namelist) == 1: for varType in ("variables", "classes", "functions", "members"): if namelist[0] in getattr(self.currentScope, varType, {}): # it's a variable we already know about, don't shadow it log.debug("uninitialized variable %r is already known, skipping", namelist) already_known = True break if not already_known: log.debug("Adding uninitialized variable: %r, line: %d", namelist, lineno) self.addVariable(namelist, [], doc=self.comment, isLocal=isLocal) already_looped = True continue if p+1 < len(styles) and text[p] == '[' and \ styles[p] == self.JS_OPERATOR: # It may be an array assignment (Example4 above). unknown_array_namelist = True array_args, p = self._getParenArguments(styles, text, p, '[') if len(array_args) == 3 and array_args[1][0] in "'\"": unknown_array_namelist = False namelist.append(self._unquoteJsString(array_args[1])) log.debug("Namelist includes an array scope, now: %r", namelist) if p >= len(styles): already_looped = True continue typeNames = [] name_prefix = namelist[0] assignChar = text[p] try_getter_setter = False addToClass = False assignToCurrentScope = False if assignChar == ":" or name_prefix == "this": assignToCurrentScope = True if name_prefix == "this" or \ isinstance(self.currentScope, JSClass): addToClass = True if p+1 < len(styles) and len(namelist) == 1 and \ name_prefix in ("get", "set") and styles[p] == self.JS_IDENTIFIER: log.debug("First element in namelist is a getter/setter") try_getter_setter = True if p+1 < len(styles) and (try_getter_setter or (styles[p] == self.JS_OPERATOR and assignChar in allowedAssignmentChars)): if name_prefix == "this": namelist = namelist[1:] if len(namelist) < 1: log.debug("_variableHandler:: No namelist for 'this'! Text: %r", text) return name_prefix = namelist[0] elif name_prefix[0] in "'\"": # String assignment { "myfield" : 123, .... name_prefix = self._unquoteJsString(name_prefix) namelist = [name_prefix] # Treat it like a variable/object assignment assignToCurrentScope = True # Assignment to the scope #print "text[p:]", text[p:] varType, typeNames, args, p = self._getVariableDetail(namelist, styles, text, p, assignmentChar=assignChar) if varType == TYPE_ALIAS: if len(typeNames) < 1 or len(typeNames) == 1 and typeNames[0] in known_javascript_types: # "alias" to a primitive varType = TYPE_VARIABLE varType_mapping = dict([(v, k) for k,v in globals().items() if k.startswith("TYPE_")]) log.debug("_variableHandler:: varType:%r, typeNames:%r, args:%r, p: %d", varType_mapping.get(varType, varType), typeNames, args, p) if varType == TYPE_FUNCTION: if addToClass: log.debug("_variableHandler:: Line %d, class function: %r(%r)", lineno, namelist, args) self.addClassFunction(namelist, args, doc=self.comment) else: log.debug("_variableHandler:: Line %d, function: %r(%r)", lineno, namelist, args) self.addFunction(namelist, args, doc=self.comment, isLocal=(not assignToCurrentScope)) elif varType == TYPE_VARIABLE or varType == TYPE_ALIAS: if varType == TYPE_VARIABLE: varCtor = JSVariable typeString = "type" else: # TYPE_ALIAS varCtor = lambda *args, **kwargs: \ JSAlias(target=typeNames, scope=self.currentScope, *args, **kwargs) typeString = "alias" if assignToCurrentScope: log.debug("_variableHandler:: Line %d, class member variable: %r (%s=%r)", lineno, namelist, typeString, typeNames) self.addClassOrVariableMember(namelist, typeNames, doc=self.comment, isLocal=isLocal, varCtor=varCtor) else: if len(namelist) > 1: log.debug("_variableHandler:: Line %d, scoped assignment: %r, %s=%r scope %r", lineno, namelist, typeString, typeNames, self.currentScope.name) else: log.debug("_variableHandler:: Line %d, local variable assignment: %r, %s=%r scope %r", lineno, namelist, typeString, typeNames, self.currentScope.name) # XXX - Check this, do we need this hack? if typeNames == ["Object"] and text[-1] == "{": # Turn it into a class log.info("_variableHandler:: Turning Object into class: %r", namelist) #self.addVariable(namelist, typeNames) self.addClass(namelist, doc=self.comment, path=self.path, varCtor=varCtor) else: self.addVariable(namelist, typeNames, doc=self.comment, isLocal=isLocal, varCtor=varCtor) # We ignore any defined functions, as we gain no value from them elif varType == TYPE_PARENT: if len(typeNames) > 0: self.addClassParent(namelist, typeNames) else: self.addAnonymousClass(namelist, doc=self.comment) elif varType == TYPE_GETTER: log.debug("_variableHandler:: Found getter:%r", namelist) self.addGetter(namelist, [], doc=self.comment) elif varType == TYPE_SETTER: log.debug("_variableHandler:: Found setter:%r", namelist) self.addSetter(namelist, doc=self.comment) elif varType == TYPE_OBJECT: # var obj = { observer: function() { ... }, count: 10 } if not typeNames: typeNames = ["Object"] if unknown_array_namelist: # Create a dummy object that will then accept any # subsequent scope assignments. This object will *not* # be included in the cix output. log.debug("_variableHandler:: Line: %d, unknown array " "assingment, creating a dummy variable: %r", lineno, namelist) self.currentScope = JSObject(None, self.currentScope, self.lineno, self.depth, "Object", path=self.path) elif assignToCurrentScope: log.debug("_variableHandler:: Line %d, class object variable: %r", lineno, namelist) self.addClassOrVariableMember(namelist, typeNames, doc=self.comment, assignAsCurrentScope=True) else: log.debug("_variableHandler:: Line %d, object variable: %r", lineno, namelist) self.addObjectVariable(namelist, doc=self.comment, isLocal=isLocal) else: log.info("_variableHandler:: Ignoring. Unhandled assignment type: %r", text) return else: log.debug("_variableHandler:: Line %d, calling scoped variable: %r", lineno, namelist) for pos, obj in self.objectArguments: if not isinstance(obj, JSFunction): continue # we have a function, tell it about the caller obj.addCaller(caller=namelist, pos=pos, line=lineno) already_looped = True def createObjectArgument(self, styles, text): log.debug("createObjectArgument") #obj = toScope.addVariable(varName, self.lineno, self.depth, "Object", doc=doc) obj = JSObject(None, None, self.lineno, self.depth, "Object", path=self.path) return obj def _addCodePiece(self, styles, text, pos=0): if pos >= len(styles): return lineno = self.lineno log.debug("*** Line: %d ********************************", lineno) #log.debug("Styles: %r", self.styles) log.debug("Text: %r", self.text[pos:]) log.debug("currentScope: %s %r", self.currentScope.cixname, self.currentScope.name) if self.currentClass: log.debug("currentClass: %r", self.currentClass.name) if self.in_variable_definition: log.debug("in_variable_definition: %r", self.in_variable_definition) #print "%d: %r" % (lineno, " ".join(self.text[pos:])) #log.debug("Comment: %r", self.comment) #log.debug("") firstStyle = styles[pos] if firstStyle == self.JS_WORD: # Keyword keyword = text[pos] if keyword == "function": isLocal = not isinstance(self.currentScope, JSFile) namelist, p = self._getIdentifiersFromPos(styles, text, pos+1) if namelist: args, p = self._getArgumentsFromPos(styles, text, p) log.debug("Line %d, function: %r(%r)", lineno, namelist, args) self.addFunction(namelist, args, doc=self.comment, isLocal=isLocal) else: # We shall add the function, but without a name as it does # not really have one... it's anonymous. args, p = self._getArgumentsFromPos(styles, text, p) self.addAnonymousFunction(args, doc=self.comment) elif keyword == "this": # Member variable of current object p = pos+1 if p < len(styles) and styles[p] == self.JS_OPERATOR and \ text[p] == ".": namelist, p = self._getIdentifiersFromPos(styles, text, p+1) self._variableHandler(lineno, styles, text, p, ["this"] + namelist) elif keyword in ("let", "var", "const"): # Variable of current scope self.in_variable_definition = True namelist, p = self._getIdentifiersFromPos(styles, text, pos+1) # if in the global/file scope the variable is global also, # if the scope is something else, add as a local variable if namelist: isLocal = not isinstance(self.currentScope, JSFile) if p < len(styles): self._variableHandler(lineno, styles, text, p, namelist, isLocal=isLocal) else: log.debug("Adding uninitialized variable: %r, line: %d", namelist, lineno) self.addVariable(namelist, [], doc=self.comment, isLocal=isLocal) elif keyword == "return": p = pos+1 if p < len(styles) and styles[p] == self.JS_OPERATOR and \ text[p] == "{": # Returning a new object self.addReturnObject(doc=self.comment) ## XXX - Fixme to allow variables with sub-elements #log.debug("Ignoring scope due to return of object") #newstate = S_IGNORE_SCOPE else: # Return types are only valid in functions if isinstance(self.currentScope, JSFunction): typeNames, p, isAlias = self._getVariableType(styles, text, pos+1, assignmentChar=None) #varType, typeNames, args, p = self._getVariableDetail([], styles, text, pos, assignmentChar="return") log.debug("Return type: %r", typeNames) self.addFunctionReturnType(typeNames) elif keyword == "if": # if (....) xyz p = self._skipOverParenArguments(styles, text, pos+1) self._addCodePiece(styles, text, p) elif keyword == "else": pos += 1 # Check for: 'else if (....) xyz' if pos < len(styles) and styles[pos] == self.JS_WORD and \ text[pos] == "if": pos = self._skipOverParenArguments(styles, text, pos+1) self._addCodePiece(styles, text, pos) else: log.debug("_addCodePiece: Unhandled keyword:%r", keyword) elif firstStyle == self.JS_IDENTIFIER: isLocal = False if self.in_variable_definition: if self.currentScope != self.cile and \ ((pos > 0 and text[pos-1] == ",") or (self.lastText and self.lastText[-1] == ",")): isLocal = True else: self.in_variable_definition = False # Defining scope for action namelist, p = self._getIdentifiersFromPos(styles, text, pos) self._variableHandler(lineno, styles, text, p, namelist, allowedAssignmentChars=":=", isLocal=isLocal) elif firstStyle == self.JS_OPERATOR: if self.lastText and self.lastText[-1] == "{" and \ text[:2] == ['(', ')'] and isinstance(self.lastScope, JSFunction): # It's a closure log.debug("Found a closure: function: %r", self.lastScope.name) self._convertFunctionToClosureVariable(self.lastScope) else: # We don't do anything here log.debug("Ignoring when starting with an operator") elif firstStyle in self.JS_STRINGS: # Check for object string names, see below: # "element1": [ 1, "one" ], # "field1": "name", #print "String assignment: %r" % (text[pos], ) #print "Text: %r" % (text, ) if pos+1 < len(styles) and \ styles[pos+1] == self.JS_OPERATOR and text[pos+1] == ":": self._variableHandler(lineno, styles, text, pos+1, text[pos:pos+1], allowedAssignmentChars=":", isLocal=False) else: log.debug("Unhandled first style:%d", firstStyle) self._resetState() #if log.level == logging.DEBUG: # print # print '\n'.join(self.cile.outline()) # print def _chooseBestVariable(self, jsvar1, jsvar2): # 1. Choose the one with a jsdoc. if jsvar1.jsdoc and not jsvar2.jsdoc: return jsvar1 if jsvar2.jsdoc and not jsvar1.jsdoc: return jsvar2 # 2. Choose the one with the a citdl. if jsvar1.type and not jsvar2.type: return jsvar1 if jsvar2.type and not jsvar1.type: return jsvar2 # 3. Choose the one with the best citdl. We prefer the one # that is not a standard type, because standard types # can be null or boring :D citdl1 = standardizeJSType(jsvar1.type) citdl2 = standardizeJSType(jsvar2.type) if citdl1 in known_javascript_types and \ not citdl2 in known_javascript_types: return jsvar2 if citdl2 in known_javascript_types and \ not citdl1 in known_javascript_types: return jsvar1 # 4. Default to the first one given. return jsvar1 def _copyObjectToAnother(self, jsobject, jsother): #print #print "Full outline:" #print '\n'.join(self.cile.outline()) #print #print "jsobject:" #print '\n'.join(jsobject.outline()) #print #print "jsother:" #print '\n'.join(jsother.outline()) appliedToGlobalScope = False if jsother == self.cile: appliedToGlobalScope = True for fieldname in ('classes', 'members', 'variables', 'functions', ): d_obj = getattr(jsobject, fieldname, {}) d_oth = getattr(jsother, fieldname, {}) for name, jsobj in d_obj.items(): # Check the parents are not the same. if jsobj.parent == jsother: parent = jsobj.parent log.warn("%s %r has parent %s %r, file: %s#%d", parent.cixname, parent.name, jsother.cixname, jsother.name, self.cile.name, self.lineno) jsobj.setParent(jsother) if appliedToGlobalScope: # Remove the __local__ and private attributes if "__local__" in jsobj.attributes: jsobj.attributes.remove("__local__") if "private" in jsobj.attributes: jsobj.attributes.remove("private") if fieldname == 'functions' and jsobj._class: jsobj._class = jsobj.parent d_oth.update(d_obj) # Ensure the variables are not already known as member variables. d_members = getattr(jsother, "members", {}) d_variables = getattr(jsother, "variables", {}) for name, jsobj in d_variables.items(): if name in d_members: # Decide which one to keep then, remove the variable and then # replace the member with the best choice. del d_variables[name] d_members[name] = self._chooseBestVariable(d_members[name], jsobj) log.info("Dupe found: %s %r has both variable and member %r, " "keeping %r", jsother.cixname, jsother.name, name, d_members[name]) if jsobject.anonymous_functions: jsother.anonymous_functions = jsobject.anonymous_functions jsobject.anonymous_functions = [] def _handleDefineProperty(self, styles, text, p): # text example: # ['(', 'namespace', ',', '"propname"', ',', '{', ')'] namelist, p = self._getIdentifiersFromPos(styles, text, p+1) if namelist and p+3 < len(styles) and styles[p+1] in self.JS_STRINGS: propertyname = self._unquoteJsString(text[p+1]) if namelist == ["this"]: scope = self.currentScope else: scope = self._findOrCreateScope(namelist, ('variables', 'classes', 'functions')) v = JSVariable(propertyname, scope, self.lineno, self.depth) if self.lastScope and isinstance(self.lastScope, JSFunction): v.type = "%s()" % (self.lastScope.name, ) scope.addVariable(propertyname, value=v) def _handleYAHOOExtension(self, styles, text, p): # text example: # ['(', 'Dog', ',', 'Mammal', ',', '{', ')'] #print "YAHOO!!!!!!" #print "len(self.objectArguments): %d" % (len(self.objectArguments), ) if p+5 < len(styles) and text[p] == "(" and len(self.objectArguments) == 1: extendClassNamelist, p = self._getIdentifiersFromPos(styles, text, p+1) log.debug("_handleYAHOOExtension:: extendClassNamelist: %r", extendClassNamelist) parentClassNamelist, p = self._getIdentifiersFromPos(styles, text, p+1) if extendClassNamelist and parentClassNamelist: # Add class parent reference #print "Extending %r, parent %r" % (extendClassNamelist, parentClassNamelist) jsclass = self._addClassPart(".".join(parentClassNamelist), self.ADD_CLASS_PARENT, extendClassNamelist, path=self.path) # Now add all information from objectArguments self._copyObjectToAnother(self.objectArguments[0][1], jsclass) #log.setLevel(logging.WARN) def _handleDojoExtension(self, type, styles, text, p): if p+4 < len(styles) and text[p] == "(" and len(self.objectArguments) == 1: if type=='declare': extendClassNamelist = self._unquoteJsString(text[p+1]).split('.') p+=2 parentClassNamelist, p = self._getIdentifiersFromPos(styles, text, p+1) if len(extendClassNamelist)>1: scope = self._findOrCreateScope(extendClassNamelist[:-1], ('variables', 'classes', 'functions')) else: scope = self.currentScope #TODO: should use the lineno of dojo.declare rather than self.objectArguments[0][1].line below jsclass = JSClass(extendClassNamelist[-1], scope, self.objectArguments[0][1].line, self.depth) scope.classes[jsclass.name] = jsclass if parentClassNamelist: jsclass = self._addClassPart(".".join(parentClassNamelist), self.ADD_CLASS_PARENT, extendClassNamelist, path=self.path) else: args,p = self._getParenArguments(styles,text,p,'[') parentClassNamelists=[''] for arg in args[1:-1]: if arg=='{': #super class is null break if arg==',': parentClassNamelists.append('') continue parentClassNamelists[-1] += arg if len(parentClassNamelists[0]): self._addClassPart(' '.join(parentClassNamelists), self.ADD_CLASS_PARENT, extendClassNamelist, path=self.path) else: #extend extendClassNamelist, p = self._getIdentifiersFromPos(styles, text, p+1) jsclass = self._locateScopeForName(extendClassNamelist, attrlist=("classes", "functions", "variables", )) if not jsclass: return if isinstance(jsclass, JSFunction): jsclass=self._convertFunctionToClass(jsclass) if jsclass: obj=self.objectArguments[0][1] self._copyObjectToAnother(obj, jsclass) for f in jsclass.functions: jsclass.functions[f]._class = jsclass for f in jsclass.anonymous_functions: f._class = jsclass # Change function constructor name to the class name so that it # is correctly recognized by Komodo as the constructor. if jsclass.functions.has_key('constructor'): func=jsclass.functions.pop('constructor') func.name=jsclass.name jsclass.functions[jsclass.name]=func def _removeObjectFromScope(self, jsobject): removename = jsobject.name parent = jsobject.parent if parent: searchScopeNames = ("variables", "functions", "classes",) if not isinstance(parent, JSFile): searchScopeNames += ("members", ) for scopeName in searchScopeNames: scope = getattr(parent, scopeName) if removename in scope and scope[removename] == jsobject: log.debug("Removing %r from scope: %s in %r", removename, scopeName, parent.name) scope.pop(removename) if jsobject in parent.anonymous_functions: parent.anonymous_functions.remove(jsobject) def _handleFunctionApply(self, namelist=None): """Everything in the function is applied to the supplied scope/namelist""" # XXX : TODO # Not everything should be applied. Only the "this." items get # applied! # Examples: # (function() { this.xyz = 1; }).apply(namelist); # // Giving namelist an xyz member. if namelist is None: scope = self.cile else: # Find the scope scope = self._findOrCreateScope(namelist, attrlist=('variables', 'classes', 'functions', )) if self.lastScope and isinstance(self.lastScope, JSFunction): applyFrom = self.lastScope parent = applyFrom.parent if isinstance(parent, JSClass) and \ parent.name == applyFrom.name: # We apply everything from the parent then, except the function # itself, start by copying everything inside the function. self._copyObjectToAnother(applyFrom, scope) # Remove the function now del parent.functions[applyFrom.name] # The class/parent becomes our next target to copy applyFrom = parent # Copy across everything in the applyFrom object self._copyObjectToAnother(applyFrom, scope) # We need to remove the applyFrom object, it's life is done self._removeObjectFromScope(applyFrom) elif self.lastScope and isinstance(self.lastScope, JSVariable): self._copyObjectToAnother(self.lastScope, scope) elif self.objectArguments and isinstance(self.objectArguments[0][1], JSObject): self._copyObjectToAnother(self.objectArguments[0][1], scope) def _handleFunctionWithArguments(self): styles = self.styles if len(styles) == 0: return text = self.text lineno = self.lineno log.debug("*** _handleFunctionWithArguments line: %d ***", lineno) #log.debug("Styles: %r", self.styles) log.debug("Text: %r", self.text) #log.debug("Comment: %r", self.comment) #log.debug("") pos = 0 firstStyle = styles[pos] getsetPos = None try: getsetPos = text.index("__defineGetter__") except ValueError: try: getsetPos = text.index("__defineSetter__") except ValueError: pass if getsetPos is not None and len(styles) > getsetPos+3 and \ styles[getsetPos+2] in self.JS_STRINGS: scopeNames, p = self._getIdentifiersFromPos(styles, text, pos) namelist = [ self._unquoteJsString(text[getsetPos+2]) ] if scopeNames and scopeNames[0] != "this": namelist = scopeNames[:-1] + namelist if text[getsetPos] == "__defineSetter__": self.addSetter(namelist, doc=self.comment) else: # Getter is different, it can have a type. citdl = None for i, scope in self.objectArguments: if isinstance(scope, JSFunction): self.lineno = scope.line citdl = scope.getReturnType() break if citdl: self.addGetter(namelist, [citdl], doc=self.comment) else: self.addGetter(namelist, [], doc=self.comment) elif firstStyle == self.JS_IDENTIFIER: namelist, p = self._getIdentifiersFromPos(styles, text, pos) if not namelist: return #print "namelist: %r" % (namelist, ) if namelist == ["Object", "defineProperty"]: # Defines a property on a given scope. self._handleDefineProperty(styles, text, p) elif namelist == ["XPCOMUtils", "defineLazyGetter"]: # Mozilla way to define a property on a given scope. self._handleDefineProperty(styles, text, p) elif namelist[0] == "YAHOO" and \ namelist[1:] in (["extend"], ["lang", "extend"]): # XXX - Should YAHOO API catalog be enabled then? self._handleYAHOOExtension(styles, text, p) elif namelist[0] == "dojo" and \ namelist[1:] in (["extend"], ["declare"]): self._handleDojoExtension(namelist[1], styles, text, p) elif namelist == ["Ext", "extend"]: # Same as what YAHOO does. self._handleYAHOOExtension(styles, text, p) elif namelist == ["Ext", "apply"] and text[p:p+1] == ["("]: # Similar to the regular function apply (see below) namelist, p = self._getIdentifiersFromPos(styles, text, p+1) if namelist: log.info("Handling Ext.apply on: %r, line: %d", namelist, self.lineno) self._handleFunctionApply(namelist) elif firstStyle == self.JS_OPERATOR: if text[:4] == [")", ".", "apply", "("]: # Special case for function apply namelist, p = self._getIdentifiersFromPos(styles, text, pos+4) if namelist: self._handleFunctionApply(namelist) elif text[3:5] == ["(", ")"]: # Applied to the global namespace self._handleFunctionApply() def _handleAnonFunctionCall(self, lineno): """ The contents of anonymous functions like: (function() {...})(); (function() {...}).call(this); should be exported to the prior scope for proper codeintel support. This is particularly relevant for top-level anonymous functions. This method is called when such a call is made for the anonymous function that ends on line number `lineno`. @param lineno The line number invoking an anonymous function. """ scope = self.currentScope for func in scope.anonymous_functions: if func.lineend == lineno: if func.cixname == "class": break # do not export anonymous classes for attrname in ("classes", "members", "functions", "variables"): d = getattr(func, attrname, {}) if attrname == "members": # Scopes do not have members, so transform them into # variables instead. attrname = "variables" for k, v in d.items(): getattr(scope, attrname, {})[k] = v scope.anonymous_functions.remove(func) self.anonFuncPending = False def _findScopeFromContext(self, styles, text): """Determine from the text (a namelist) what scope the text is referring to. Returns the scope found or None. """ log.debug("_findScopeFromContext: %r" % (text, )) scope = None try: idx = text.index("prototype") except ValueError: pass else: # We have a class prototype, find the class and return with that # as the current scope. If it's a function that is not part of a # class, then convert the function into a class. if idx >= 2 and text[idx-1] == ".": namelist, p = self._getIdentifiersFromPos(styles[:idx-1], text[:idx-1], 0) if namelist: scope = self._locateScopeForName(namelist, attrlist=("classes", )) if not scope: self._locateScopeForName(namelist, attrlist=("functions", )) if isinstance(scope, JSFunction): # Scope is a function, it should be a class, # convert it now. scope = self._convertFunctionToClass(scope) if scope: log.debug("_findScopeFromContext: found %s %r", scope.cixname, scope.name) return scope def _resetState(self, newstate=S_DEFAULT): self.state = newstate self.bracket_depth = 0 self.styles = [] self.lastText = self.text self.text = [] if self.comment: self.last_comment_and_jsdoc = [self.comment, None] self.comment = [] self.argumentPosition = 0 self.argumentTextPosition = 0 self.objectArguments = [] #log.debug("Set state %d, line: %d", self.state, self.lineno, ) def _popPreviousState(self, keep_style_and_text=False): current_styles = self.styles current_text = self.text current_arguments = self.objectArguments previous_state = self.state if len(self.state_stack) >= 1: # Reset to previous state self.state, self.bracket_depth, self.styles, \ self.text, self.lastText, self.comment, \ self.argumentPosition, self.argumentTextPosition, \ self.objectArguments, \ self.in_variable_definition = self.state_stack.pop() else: # Reset them all self._resetState() log.debug("_popPreviousState:: previous: %d, current: %d", previous_state, self.state) if keep_style_and_text: self.styles += current_styles self.text += current_text self.objectArguments = current_arguments def _pushAndSetState(self, newstate=S_DEFAULT): self.state_stack.append((self.state, self.bracket_depth, self.styles, self.text, self.lastText, self.comment, self.argumentPosition, self.argumentTextPosition, self.objectArguments, self.in_variable_definition)) self._resetState(newstate) self.in_variable_definition = False def _endOfScanReached(self): """Ensure any remaining text is included in the cile""" if len(self.styles) > 0: self._addCodePiece(self.styles, self.text, pos=0) def token_next(self, style, text, start_column, start_line, **other_args): """Loops over the styles in the document and stores important info. When enough info is gathered, will perform a call to analyze the code and generate subsequent language structures. These language structures will later be used to generate XML output for the document.""" if style in self.JS_CILE_STYLES: # We keep track of these styles and the text associated with it. # When we gather enough info, these will be sent to the # _addCodePiece() function which will analyze the info. # We want to use real line numbers starting from 1 (not 0) start_line += 1 #print "state: %d, text: %r" % (self.state, self.text, ) #log.debug("state: %d, line: %d, self.text: %r, text: %r", self.state, start_line, self.text, text) if self.state == S_DEFAULT and len(self.styles) > 0 and \ self.last_lineno < start_line: # We have moved down line(s) and we have data, check if we # need to add code from the previous line(s) # XXX: Need to be careful with e4x! if ((style != self.JS_OPERATOR or text[0] not in "({[.,=") and (self.styles[-1] != self.JS_OPERATOR or self.text[-1] not in "({[.,=") and # We need to ignore certains cases, such as: # "new \n function", see bug 82569. (self.styles[-1] != self.JS_WORD or self.text[-1] != "new")): self._addCodePiece(self.styles, self.text, pos=0) self.in_variable_definition = False self.lineno = start_line if style != self.JS_OPERATOR: self.styles.append(style) self.text.append(text) if self.anonFuncPending and text != "call": # There was a '(function() {...}).' construct, but the # method after the trailing '.' was not 'call'. Therefore # that function is nothing special. self.anonFuncPending = False log.debug("...Anonymous function not invoked.") else: if text == "(": # Only the "(", it's like above # Check if this is of the form "(function { .... })" # This is a fix for self invoking functions/closures # http://bugs.activestate.com/show_bug.cgi?id=63297 if not self.text: log.debug("Ignoring initial brace: '(' on line %d", self.lineno) return if self.anonFuncPending and text[0] != "(": # There was a '(function() {...}).call' construct, but for # whatever reason, there is no invoking '('; the anonymous # function is nothing special. self.anonFuncPending = False log.debug("...Anonymous function not invoked.") #log.debug("token_next: line %d, %r, text: %r" % (self.lineno, text, self.text)) next_op_index = 0 while next_op_index < len(text): op = text[next_op_index] next_op_index += 1 self.styles.append(style) self.text.append(op) #if self.state == S_OBJECT_ARGUMENT: # if op not in "{}": # continue if op == "(": if self.bracket_depth == 0: # We can start defining arguments now log.debug("Entering S_IN_ARGS state, line: %d, col: %d", start_line, start_column) newscope = self._findScopeFromContext(self.styles, self.text) self._pushAndSetState(S_IN_ARGS) if newscope and self.currentScope != newscope: log.debug("Adjusting scope to: %r %r", newscope.cixname, newscope.name) # Need to temporarily adjust the scope to deal # with getters, setters and class prototypes. self.currentScope = newscope self.argumentTextPosition = len(self.text) if self.anonFuncPending: self._handleAnonFunctionCall(start_line) self.bracket_depth += 1 elif op == ")": self.bracket_depth -= 1 if self.bracket_depth <= 0: # Pop the state, but keep the style and text of # the arguments last_state = self.state self._popPreviousState(keep_style_and_text=True) if self.state != S_IN_ARGS and last_state == S_IN_ARGS: self._handleFunctionWithArguments() log.debug("Entering state %d, line: %d, col: %d", self.state, start_line, start_column) elif isinstance(self.lastScope, JSFunction) and self.text[-3:] == ['{', '(', ')']: # It's a function argument closure. self.lastScope = self._convertFunctionToClosureVariable(self.lastScope) elif op == "=": try: next_ch = text[next_op_index] except IndexError: continue if next_ch == ">": # ES6, => fat arrow function self.text[-1] = "=>" next_op_index += 1 #elif op == "=": # if text == op: # log.debug("Entering S_IN_ASSIGNMENT state, line: %d, col: %d", start_line, start_column) # self.state = S_IN_ASSIGNMENT elif op == "{": # Increasing depth/scope, could be an argument object if self.state == S_IN_ARGS: # __defineGetter__("num", function() { return this._num }); argTextPos = self.argumentTextPosition if len(self.text) >= 2 and self.text[-2] == ")": # foo( ... ( ... ) { # this really only makes sense as a function expression definition # foo( ... function (...) { # this function may have multiple arguments, so we can't trust # self.argumentTextPosition (which was clobbered) try: argsStart = len(self.text) - list(reversed(self.text)).index("(") if argsStart > 1: functionPos = argsStart - 2 # position of "function" keyword if self.styles[functionPos] == self.JS_IDENTIFIER: # named function functionPos -= 1 if functionPos > -1 and \ self.styles[functionPos] == self.JS_WORD and \ self.text[functionPos] == "function": # this is indeed a function; check arguments for sanity args = self.text[argsStart:-2] if all(x == ',' for x in args[1::2]): # Passing a function as one of the arguments, # need to create a JSFunction scope for this, # as various information may be needed, i.e. # a getter function return type. obj = self.addAnonymousFunction(args=args[::2]) # don't append to self.objectArguments here, we do # it later when we see the closing brace self._pushAndSetState(S_DEFAULT) except ValueError: # no "(" found in self.text pass elif len(self.text) >= 2 and \ ((self.text[-2] == "(" and self.argumentPosition == 0) or (self.text[-2] == "," and self.argumentPosition > 0)): # It's an object argument log.debug("Entering S_OBJECT_ARGUMENT state, line: %d, col: %d", start_line, start_column) #print "Entering S_OBJECT_ARGUMENT state, line: %d, col: %d" % (start_line, start_column) obj = self.createObjectArgument(self.styles, self.text) self.currentScope = obj self._pushAndSetState(S_OBJECT_ARGUMENT) else: self._pushAndSetState(S_IN_ARGS) else: self._addCodePiece(self.styles, self.text, pos=0) self._pushAndSetState(S_DEFAULT) self.incBlock() elif op == "}": # Decreasing depth/scope previous_state = self.state if self.state != S_IN_ARGS: # only add this piece if we're not in an arg state self._addCodePiece(self.styles, self.text, pos=0) self._popPreviousState() if self.state == S_IN_ARGS: self.objectArguments.append((self.argumentPosition, self.currentScope)) log.debug("Leaving S_OBJECT_ARGUMENT state, entering S_IN_ARGS state, line: %d, col: %d", start_line, start_column) #print "Leaving S_OBJECT_ARGUMENT state, entering S_IN_ARGS state, line: %d, col: %d" % (start_line, start_column) self.decBlock() if self.anonFuncPending: # Check to see if the function is immediately # invoked or has a trailing '.', which may indicate # a '.call()'. That case must be handled later. index = next_op_index if index < len(text) and text[index] == ')': index += 1 # skip trailing ')' in '(function() {...})' if index >= len(text) or (text[index] != '(' and \ text[index] != '.'): self.anonFuncPending = False log.debug("...Anonymous function not invoked.") elif op == "," and self.text[0] not in ("let", "var", "const"): # Ignore when it's inside arguments if self.state == S_IN_ARGS: self.argumentPosition += 1 self.argumentTextPosition = len(self.text) else: self._addCodePiece(self.styles, self.text, pos=0) elif op == ";": # Statement is done if self.state != S_IN_ARGS: # only add this piece if we're not in an arg state self._addCodePiece(self.styles, self.text, pos=0) self.in_variable_definition = False # Remember the last code line we looked at self.last_lineno = self.lineno elif style in self.JS_COMMENT_STYLES: self.comment.append(text) def scan_puretext(self, content, updateAllScopeNames=True): """Scan the given pure javascript content""" #XXX Should eventually use lang_javascript.JavaScriptLexer() # because (1) it's word lists might differ and (2) the # codeintel system manages one instance of it. JavaScriptLexer(self.mgr).tokenize_by_style(content, self.token_next) # Ensure we take notice of any text left in the ciler self._endOfScanReached() if updateAllScopeNames: # We've parsed up the JavaScript, fix any variables types self.cile.updateAllScopeNames() def convertToElementTreeFile(self, cixelement, file_lang, module_lang=None): """Store JS information into the cixelement as a file(s) sub element""" self.cile.convertToElementTreeFile(cixelement, file_lang, module_lang) def convertToElementTreeModule(self, cixmodule): """Store JS information into already created cixmodule""" self.cile.convertToElementTreeModule(cixmodule) class Utils(object): @staticmethod def unquoteJsString(s): """Return the string without quotes around it""" if len(s) >= 2 and s[0] in "\"'": return s[1:-1] return s #---- internal support stuff def _isident(char): return "a" <= char <= "z" or "A" <= char <= "Z" or char == "_" def _isdigit(char): return "0" <= char <= "9" def _walk_js_scopes(scope, lpath=None): """Walk the subscopes of the given element. Note that in JavaScript elements with children are considered a scope. Yields (scope, lpath) depth-first. The given top-level element is not yielded. """ if lpath is None: lpath = [] for subscope in scope: if subscope.tag == "variable" and not subscope: continue sublpath = lpath + [subscope.get("name")] yield (subscope, sublpath) for r in _walk_js_scopes(subscope, sublpath): yield r def _walk_js_symbols(elem, _prefix=None): if _prefix: lpath = _prefix + (elem.get("name"), ) else: lpath = (elem.get("name"), ) yield lpath if elem.tag == "variable": pass # don't descend into variables elif not (elem.tag == "scope" and elem.get("ilk") == "function"): for child in elem: for child_lpath in _walk_js_symbols(child, lpath): yield child_lpath #---- registration def register(mgr): """Register language support with the Manager.""" mgr.set_lang_info(lang, silvercity_lexer=JavaScriptLexer(mgr), buf_class=JavaScriptBuffer, langintel_class=JavaScriptLangIntel, import_handler_class=JavaScriptImportHandler, cile_driver_class=JavaScriptCILEDriver, is_cpln_lang=True, import_everything=True)