From 8e10480bf860665dce5ebc7c12983d028b0f83cb Mon Sep 17 00:00:00 2001 From: Vladimir K Urushev Date: Fri, 26 Jan 2018 06:30:16 +0300 Subject: [PATCH 1/3] Implement importmagic --- ThirdPartyNotices-Distribution.txt | 29 + package.json | 30 +- package.nls.json | 5 +- pythonFiles/importMagic.py | 236 +++++++ pythonFiles/importmagic/__init__.py | 11 + pythonFiles/importmagic/importer.py | 365 ++++++++++ pythonFiles/importmagic/index.py | 446 ++++++++++++ pythonFiles/importmagic/six.py | 637 ++++++++++++++++++ pythonFiles/importmagic/symbols.py | 409 +++++++++++ pythonFiles/importmagic/util.py | 73 ++ src/client/extension.ts | 8 + .../importMagicProxyFactory.ts | 40 ++ .../importMagicCompletionProvider.ts | 90 +++ src/client/providers/importMagicProvider.ts | 176 +++++ src/client/providers/importMagicProxy.ts | 360 ++++++++++ src/test/importMagic/base.test.ts | 111 +++ src/test/pythonFiles/importMagic/classes.py | 14 + src/test/pythonFiles/importMagic/classes2.py | 5 + src/test/pythonFiles/importMagic/compl.py | 1 + src/test/pythonFiles/importMagic/methods.py | 2 + src/test/pythonFiles/importMagic/one.py | 7 + 21 files changed, 3053 insertions(+), 2 deletions(-) create mode 100644 pythonFiles/importMagic.py create mode 100755 pythonFiles/importmagic/__init__.py create mode 100755 pythonFiles/importmagic/importer.py create mode 100755 pythonFiles/importmagic/index.py create mode 100755 pythonFiles/importmagic/six.py create mode 100755 pythonFiles/importmagic/symbols.py create mode 100755 pythonFiles/importmagic/util.py create mode 100644 src/client/languageServices/importMagicProxyFactory.ts create mode 100644 src/client/providers/importMagicCompletionProvider.ts create mode 100644 src/client/providers/importMagicProvider.ts create mode 100644 src/client/providers/importMagicProxy.ts create mode 100644 src/test/importMagic/base.test.ts create mode 100644 src/test/pythonFiles/importMagic/classes.py create mode 100644 src/test/pythonFiles/importMagic/classes2.py create mode 100644 src/test/pythonFiles/importMagic/compl.py create mode 100644 src/test/pythonFiles/importMagic/methods.py create mode 100644 src/test/pythonFiles/importMagic/one.py diff --git a/ThirdPartyNotices-Distribution.txt b/ThirdPartyNotices-Distribution.txt index bc1024215287..4f55c8296d82 100644 --- a/ThirdPartyNotices-Distribution.txt +++ b/ThirdPartyNotices-Distribution.txt @@ -45,6 +45,7 @@ Microsoft Python extension for Visual Studio Code incorporates components from t 34. vscode-extension-telemetry (https://github.com/Microsoft/vscode-extension-telemetry) 35. vscode-languageclient (https://github.com/Microsoft/vscode-languageserver-node) 36. vscode-languageserver (https://github.com/Microsoft/vscode-languageserver-node/) +37. importmagic (https://github.com/alecthomas/importmagic/) %% Arch NOTICES AND INFORMATION BEGIN HERE @@ -1511,3 +1512,31 @@ THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ========================================= END OF vscode-languageserver NOTICES AND INFORMATION + +%% importmagic NOTICES AND INFORMATION BEGIN HERE +========================================= +Copyright (c) 2014, 2015, Alec Thomas +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========================================= +END OF importmagic NOTICES AND INFORMATION diff --git a/package.json b/package.json index b9458fd647bb..afb44fb5b538 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,10 @@ "onCommand:python.goToPythonObject", "onCommand:python.setLinter", "onCommand:python.enableLinting", - "onCommand:python.createTerminal" + "onCommand:python.createTerminal", + "onCommand:importMagic.resolveImport", + "onCommand:importMagic.insertImport", + "onCommand:importMagic.findSymbols" ], "main": "./out/client/extension", "contributes": { @@ -94,6 +97,11 @@ "command": "jupyter.runSelectionLine", "key": "ctrl+alt+enter", "when": "editorFocus && !replaceActive && !searchViewletVisible && !findWidgetVisible" + }, + { + "command": "importMagic.findSymbols", + "key": "ctrl+shift+space", + "when": "editorLangId == python" } ], "commands": [ @@ -102,6 +110,21 @@ "title": "%python.command.python.sortImports.title%", "category": "Python Refactor" }, + { + "command": "importMagic.resolveImport", + "title": "%python.command.importMagic.resolveImport.title%", + "category": "Python Refactor" + }, + { + "command": "importMagic.insertImport", + "title": "%python.command.importMagic.insertImport.title%", + "category": "Python Refactor" + }, + { + "command": "importMagic.findSymbols", + "title": "%python.command.importMagic.findSymbols.title%", + "category": "Python Refactor" + }, { "command": "python.startREPL", "title": "%python.command.python.startREPL.title%", @@ -253,6 +276,11 @@ "group": "Refactor", "when": "editorLangId == python" }, + { + "command": "importMagic.resolveImport", + "group": "Refactor", + "when": "editorLangId == python" + }, { "command": "python.execSelectionInTerminal", "group": "Python", diff --git a/package.nls.json b/package.nls.json index dd6cedc42d68..2f3cf00425c7 100644 --- a/package.nls.json +++ b/package.nls.json @@ -49,5 +49,8 @@ "python.snippet.launch.attach.label": "Python: Attach", "python.snippet.launch.attach.description": "Attach the debugger for remote debugging", "python.snippet.launch.scrapy.label": "Python: Scrapy", - "python.snippet.launch.scrapy.description": "Scrapy with Integrated Terminal/Console" + "python.snippet.launch.scrapy.description": "Scrapy with Integrated Terminal/Console", + "python.command.importMagic.resolveImport.title": "Resolve Imports", + "python.command.importMagic.insertImport.title": "Insert Import", + "python.command.importMagic.findSymbols.title": "Find Symbols" } diff --git a/pythonFiles/importMagic.py b/pythonFiles/importMagic.py new file mode 100644 index 000000000000..2206ecaf3828 --- /dev/null +++ b/pythonFiles/importMagic.py @@ -0,0 +1,236 @@ +import io +import os +import re +import sys +import json +import time +import traceback + +try: + import importmagic +except: + json_message = {'error': True, 'message': 'Importmagic not installed', + 'traceback': '', 'type': 'ModuleNotFoundError'} + sys.stderr.write(json.dumps(json_message)) + sys.stderr.write('\n') + sys.stderr.flush() + + +ITEMS_LIMIT = 50 +INDEX_FILE_TTL = 300 + + +class ImportMagicDaemon(object): + def __init__(self): + self._input = io.open(sys.stdin.fileno(), encoding='utf-8') + self._index = None + + def method_build_index(self, **kwargs): + workspace_path = kwargs.get('workspacePath') + extra_paths = kwargs.get('extraPaths', []) + skip_test_folders = kwargs.get('skipTestFolders') + force_rebuild = kwargs.get('forceRebuild') + + if not workspace_path: + raise ValueError('Empty workspacePath') + + paths = list(set(extra_paths + [workspace_path] + sys.path)) + + index_file = os.path.join(workspace_path, '.vscode', 'importmagic') + index_file_ts = None + + try: + index_file_ts = os.path.getmtime(index_file) + except Exception: + pass + + if force_rebuild or not index_file_ts or time.time() - index_file_ts > INDEX_FILE_TTL: + if skip_test_folders: + blacklist_re = importmagic.index.DEFAULT_BLACKLIST_RE + else: + blacklist_re = re.compile(r'^$') + + self._index = importmagic.SymbolIndex(blacklist_re=blacklist_re) + self._index.build_index(paths) + + try: + with open(index_file, 'w') as fd: + self._index.serialize(fd) + except Exception: + pass + else: + # Read old index + with open(index_file, 'r') as fd: + self._index = importmagic.SymbolIndex.deserialize(fd) + + return dict(success=True) + + def method_get_unresolved(self, **kwargs): + source_file = kwargs.get('sourceFile') + + if not source_file: + raise ValueError('Empty sourceFile') + + if not self._index: + raise Exception('First run build_index() method') + + with open(source_file, 'r') as fd: + python_source = fd.read() + scope = importmagic.Scope.from_source(python_source) + + unresolved, unreferenced = \ + scope.find_unresolved_and_unreferenced_symbols() + + return dict(unresolved=list(unresolved), unreferenced=list(unreferenced)) + + def method_import_suggestions(self, **kwargs): + source_file = kwargs.get('sourceFile') + unresolved_name = kwargs.get('unresolvedName') + + if not source_file: + raise ValueError('Empty sourceFile') + + if not unresolved_name: + raise ValueError('Empty unresolvedName') + + if not self._index: + raise Exception('First run build_index() method') + + with open(source_file, 'r') as fd: + python_source = fd.read() + scope = importmagic.Scope.from_source(python_source) + + _unresolved, _unreferenced = \ + scope.find_unresolved_and_unreferenced_symbols() + + # Sometimes unresolved may contain "sys.path". + # Split this cases for find "sys.path", "sys" and "path" + unresolved = set() + for item1 in _unresolved: + for item2 in item1.split('.'): + unresolved.add(item2) + + if unresolved_name not in unresolved: + raise Exception('Import this expression is not necessary') + + candidates = [] + for score, module, variable in self._index.symbol_scores(unresolved_name): + candidates.append(dict(score=score, module=module, + variable=variable)) + if len(candidates) >= ITEMS_LIMIT: + break + + return dict(candidates=candidates) + + def method_insert_import(self, **kwargs): + source_file = kwargs.get('sourceFile') + + module = kwargs.get('module') + variable = kwargs.get('variable') + + if not source_file: + raise ValueError('Empty sourceFile') + + if not module: + raise ValueError('Empty module') + + if not self._index: + raise Exception('First run inibuild_indext() method') + + with open(source_file, 'r') as fd: + python_source = fd.read() + + imports = importmagic.Imports(self._index, python_source) + imports.set_style(multiline='backslash', max_columns=79) # TODO: Get style from VS Code + + if variable is None: + imports.add_import(module) + else: + imports.add_import_from(module, variable) + start, end, text = imports.get_update() + + return dict(fromLine=start, endLine=end, text=text) + + def method_get_symbols(self, **kwargs): + find = kwargs.get('text', '').replace('_', '').lower() + + if not self._index: + raise Exception('First run build_index() method') + + results = {} + + def scan_tree(scope): + for key, subscope in scope._tree.items(): + if type(subscope) is not float: + get_result(key, scope, subscope.score) + scan_tree(subscope) + else: + get_result(key, scope, subscope) + + def get_result(key, scope, key_score): + k = key.replace('_', '').lower() + if not k.startswith(find): + return + + score = -(key_score + (100-scope.depth())*10) + if k not in results or results[k]['_score'] > score: + if scope.path(): + module = scope.path() + variable = key + else: + module = key + variable = None + + results[k] = dict(key=key, module=module, variable=variable, + depth=scope.depth(), _score=score, score=key_score) + + + scan_tree(self._index) + + return dict(items=sorted(results.values(), key=lambda x: x['_score'])[:ITEMS_LIMIT]) + + def _process_request(self, request): + method_name = request.get('method') + + result = None + method = getattr(self, 'method_%s' % method_name) + if not method: + raise ValueError('Invalid method name') + + result = method(**request) + + if not result or type(result) != dict: + raise ValueError('Method must return dict') + + return result + + def _error_response(self, response): + sys.stderr.write(json.dumps(response)) + sys.stderr.write('\n') + sys.stderr.flush() + + def _success_response(self, response): + sys.stdout.write(json.dumps(response)) + sys.stdout.write('\n') + sys.stdout.flush() + + def watch(self): + while True: + try: + request = json.loads(self._input.readline()) + request_id = request.get('requestId') + + if not request_id: + raise ValueError('Empty request id') + + response = self._process_request(request) + json_message = dict(id=request_id, **response) + self._success_response(json_message) + except: + exc_type, exc_value, exc_tb = sys.exc_info() + tb_info = traceback.extract_tb(exc_tb) + json_message = dict(error=True, id=request_id, message=str(exc_value), traceback=str(tb_info), type=str(exc_type)) + self._error_response(json_message) + +if __name__ == '__main__': + ImportMagicDaemon().watch() diff --git a/pythonFiles/importmagic/__init__.py b/pythonFiles/importmagic/__init__.py new file mode 100755 index 000000000000..bce354a99850 --- /dev/null +++ b/pythonFiles/importmagic/__init__.py @@ -0,0 +1,11 @@ +"""Python Import Magic - automagically add, remove and manage imports + +This module just exports the main API of importmagic. +""" + +__author__ = 'Alec Thomas ' +__version__ = '0.1.7' + +from importmagic.importer import Import, Imports, get_update, update_imports +from importmagic.index import SymbolIndex +from importmagic.symbols import Scope diff --git a/pythonFiles/importmagic/importer.py b/pythonFiles/importmagic/importer.py new file mode 100755 index 000000000000..f9e0a6733f47 --- /dev/null +++ b/pythonFiles/importmagic/importer.py @@ -0,0 +1,365 @@ +"""Imports new symbols.""" + +import os +import tokenize +from collections import defaultdict +try: + from ConfigParser import ConfigParser +except: + from configparser import ConfigParser + +from importmagic.six import StringIO + + +class Iterator(object): + def __init__(self, tokens, start=None, end=None): + self._tokens = tokens + self._cursor = start or 0 + self._end = end or len(self._tokens) + + def rewind(self): + self._cursor -= 1 + + def next(self): + if not self: + return None, None + token = self._tokens[self._cursor] + index = self._cursor + self._cursor += 1 + return index, token + + def peek(self): + return self._tokens[self._cursor] if self else None + + def until(self, type): + tokens = [] + while self: + index, token = self.next() + tokens.append((index, token)) + if type == token[0]: + break + return tokens + + def __nonzero__(self): + return self._cursor < self._end + __bool__ = __nonzero__ + + +class Import(object): + def __init__(self, location, name, alias): + self.location = location + self.name = name + self.alias = alias + + def __repr__(self): + return 'Import(location=%r, name=%r, alias=%r)' % \ + (self.location, self.name, self.alias) + + def __hash__(self): + return hash((self.location, self.name, self.alias)) + + def __eq__(self, other): + return self.location == other.location and self.name == other.name and self.alias == other.alias + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return self.location < other.location \ + or self.name < other.name \ + or (self.alias is not None and other.alias is not None and self.alias < other.alias) + + +# See SymbolIndex.LOCATIONS for details. +LOCATION_ORDER = 'FS3L' + +PROJECT_CONFIG_FILE = 'setup.cfg' + + +class Imports(object): + + _style = {'multiline': 'parentheses', 'max_columns': 79} + + def __init__(self, index, source, root_dir=None): + self._imports = set() + self._imports_from = defaultdict(set) + self._imports_begin = self._imports_end = None + self._source = source + self._index = index + self._root_dir = root_dir + self._parse(source) + if root_dir: + style = self.get_style_from_config() + self._style.update(style) + + @classmethod + def set_style(cls, **kwargs): + cls._style.update(kwargs) + + def get_style_from_config(self): + style = {} + cfg_path = os.path.join(self._root_dir, PROJECT_CONFIG_FILE) + config = ConfigParser() + config.read([cfg_path]) + imp_cfg = config['importmagic'] + + if imp_cfg['multiline']: + style['multiline'] = imp_cfg['multiline'] + if imp_cfg['max_columns']: + style['max_columns'] = int(imp_cfg['max_columns']) + return style + + def add_import(self, name, alias=None): + location = LOCATION_ORDER.index(self._index.location_for(name)) + self._imports.add(Import(location, name, alias)) + + def add_import_from(self, module, name, alias=None): + location = LOCATION_ORDER.index(self._index.location_for(module)) + self._imports_from[module].add(Import(location, name, alias)) + + def remove(self, references): + for imp in list(self._imports): + if imp.name in references: + self._imports.remove(imp) + for name, imports in self._imports_from.items(): + for imp in list(imports): + if imp.name in references: + imports.remove(imp) + + def get_update(self): + groups = [] + for expected_location in range(len(LOCATION_ORDER)): + out = StringIO() + for imp in sorted(self._imports): + if expected_location != imp.location: + continue + out.write('import {module}{alias}\n'.format( + module=imp.name, + alias=' as {alias}'.format(alias=imp.alias) if imp.alias else '', + )) + + for module, imports in sorted(self._imports_from.items()): + imports = sorted(imports) + if not imports or expected_location != imports[0].location: + continue + line = 'from {module} import '.format(module=module) + clauses = ['{name}{alias}'.format( + name=i.name, + alias=' as {alias}'.format(alias=i.alias) if i.alias else '' + ) for i in imports] + clauses.reverse() + line_len = len(line) + line_pieces = [] + paren_used = False + while clauses: + clause = clauses.pop() + next_len = line_len + len(clause) + 2 + if next_len > self._style['max_columns']: + imported_items = ', '.join(line_pieces) + if self._style['multiline'] == 'parentheses': + line_tail = ',\n' + if not paren_used: + line += '(' + paren_used = True + line_pieces.append('\n') + else: + # Use a backslash + line_tail = ', \\\n' + out.write(line + imported_items + line_tail) + line = ' ' + line_len = len(line) + len(clause) + 2 + line_pieces = [clause] + else: + line_pieces.append(clause) + line_len = next_len + line += ', '.join(line_pieces) + (')\n' if paren_used else '\n') + if line.strip(): + out.write(line) + + text = out.getvalue() + if text: + groups.append(text) + + start = self._tokens[self._imports_begin][2][0] - 1 + end = self._tokens[min(len(self._tokens) - 1, self._imports_end)][2][0] - 1 + if groups: + text = '\n'.join(groups) + '\n\n' + else: + text = '' + return start, end, text + + def update_source(self): + start, end, text = self.get_update() + lines = self._source.splitlines() + lines[start:end] = text.splitlines() + return '\n'.join(lines) + '\n' + + def _parse(self, source): + reader = StringIO(source) + # parse until EOF or TokenError (allows incomplete modules) + tokens = [] + try: + tokens.extend(tokenize.generate_tokens(reader.readline)) + except tokenize.TokenError: + # TokenError happens always at EOF, for unclosed strings or brackets. + # We don't care about that here, since we still can recover the whole + # source code. + pass + self._tokens = tokens + it = Iterator(self._tokens) + self._imports_begin, self._imports_end = self._find_import_range(it) + it = Iterator(self._tokens, start=self._imports_begin, end=self._imports_end) + self._parse_imports(it) + + def _find_import_range(self, it): + ranges = self._find_import_ranges(it) + start, end = ranges[0][1:] + return start, end + + def _find_import_ranges(self, it): + ranges = [] + indentation = 0 + explicit = False + size = 0 + start = None + potential_end_index = -1 + + while it: + index, token = it.next() + + if token[0] == tokenize.INDENT: + indentation += 1 + continue + elif token[0] == tokenize.DEDENT: + indentation += 1 + continue + + if indentation: + continue + + # Explicitly tell importmagic to manage the following block of imports + if token[1] == '# importmagic: manage': + ranges = [] + start = index + 2 # Start managing imports after directive comment + newline. + explicit = True + continue + elif token[0] in (tokenize.STRING, tokenize.COMMENT): + # If a non-import statement follows, stop the range *before* + # this string or comment, in order to keep it out of the + # updated import block. + if potential_end_index == -1: + potential_end_index = index + continue + elif token[0] in (tokenize.NEWLINE, tokenize.NL): + continue + + if not ranges: + ranges.append((0, index, index)) + + # Accumulate imports + if token[1] in ('import', 'from'): + potential_end_index = -1 + if start is None: + start = index + size += 1 + while it: + token = it.peek() + if token[0] == tokenize.NEWLINE or token[1] == ';': + break + index, _ = it.next() + + # Terminate this import range + elif start is not None and token[1].strip(): + if potential_end_index > -1: + index = potential_end_index + potential_end_index = -1 + ranges.append((size, start, index)) + + start = None + size = 0 + if explicit: + break + + if start is not None: + ranges.append((size, start, index)) + ranges.sort(reverse=True) + return ranges + + def _parse_imports(self, it): + while it: + index, token = it.next() + + if token[1] not in ('import', 'from') and token[1].strip(): + continue + + type = token[1] + if type in ('import', 'from'): + tokens = it.until(tokenize.NEWLINE) + tokens = [ + t[1] for i, t in tokens + if t[0] == tokenize.NAME or t[1] in (',', '.', '...')] + tokens.reverse() + self._parse_import(type, tokens) + + def _parse_import(self, type, tokens): + module = None + if type == 'from': + module = '' + while tokens and tokens[-1] != 'import': + module += tokens.pop() + assert tokens.pop() == 'import' + while tokens: + name = '' + while True: + name += tokens.pop() + next = tokens.pop() if tokens else None + if next == '.': + name += next + else: + break + + alias = None + if next == 'as': + alias = tokens.pop() + if alias == name: + alias = None + next = tokens.pop() if tokens else None + if next == ',': + pass + if type == 'import': + self.add_import(name, alias=alias) + else: + self.add_import_from(module, name, alias=alias) + + def __repr__(self): + return 'Imports(imports=%r, imports_from=%r)' % (self._imports, self._imports_from) + + +def _process_imports(src, index, unresolved, unreferenced, project_root): + imports = Imports(index, src, project_root) + imports.remove(unreferenced) + for symbol in unresolved: + scores = index.symbol_scores(symbol) + if not scores: + continue + _, module, variable = scores[0] + # Direct module import: eg. os.path + if variable is None: + # sys.path sys path -> import sys + # os.path.basename os.path basename -> import os.path + imports.add_import(module) + else: + # basename os.path basename -> from os.path import basename + # path.basename os.path basename -> from os import path + imports.add_import_from(module, variable) + return imports + + +def get_update(src, index, unresolved, unreferenced, project_root=None): + imports = _process_imports(src, index, unresolved, unreferenced, project_root) + return imports.get_update() + + +def update_imports(src, index, unresolved, unreferenced, project_root=None): + imports = _process_imports(src, index, unresolved, unreferenced, project_root) + return imports.update_source() diff --git a/pythonFiles/importmagic/index.py b/pythonFiles/importmagic/index.py new file mode 100755 index 000000000000..f972b15460be --- /dev/null +++ b/pythonFiles/importmagic/index.py @@ -0,0 +1,446 @@ +"""Build an index of top-level symbols from Python modules and packages.""" + +import ast +import json +import logging +import os +import re +import sys +from contextlib import contextmanager +from distutils import sysconfig + +from importmagic.util import get_cache_dir, parse_ast + + +LIB_LOCATIONS = sorted(set(( + (sysconfig.get_python_lib(standard_lib=True), 'S'), + (sysconfig.get_python_lib(plat_specific=True), '3'), + (sysconfig.get_python_lib(standard_lib=True, prefix=sys.prefix), 'S'), + (sysconfig.get_python_lib(plat_specific=True, prefix=sys.prefix), '3'), + (sysconfig.get_python_lib(standard_lib=True, prefix='/usr/local'), 'S'), + (sysconfig.get_python_lib(plat_specific=True, prefix='/usr/local'), '3'), +)), key=lambda l: -len(l[0])) + +# Regex matching modules that we never attempt to index. +DEFAULT_BLACKLIST_RE = re.compile(r'\btest[s]?|test[s]?\b', re.I) +# Modules to treat as built-in. +# +# "os" is here mostly because it imports a whole bunch of aliases from other +# modules. The simplest way of dealing with that is just to import it and use +# vars() on it. +BUILTIN_MODULES = sys.builtin_module_names + ('os', 'typing',) + +_PYTHON_VERSION = 'python{}.{}'.format(sys.version_info.major, sys.version_info.minor) + +LOCATION_BOOSTS = { + '3': 1.2, + 'L': 1.5, +} + + +# TODO: Update scores based on import reference frequency. +# eg. if "sys.path" is referenced more than os.path, prefer it. + + +logger = logging.getLogger(__name__) + + +class JSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, SymbolIndex): + d = o._tree.copy() + d.update(('.' + name, getattr(o, name)) + for name in SymbolIndex._SERIALIZED_ATTRIBUTES) + if o._lib_locations is not None: + d['.lib_locations'] = o._lib_locations + return d + return super(JSONEncoder, self).default(o) + + +class SymbolIndex(object): + PACKAGE_ALIASES = { + # Give 'os.path' a score boost over posixpath and ntpath. + 'os.path': (os.path.__name__, 1.2), + # Same with 'os', due to the heavy aliasing of other packages. + 'os': ('os', 1.2), + } + LOCATIONS = { + 'F': 'Future', + '3': 'Third party', + 'S': 'System', + 'L': 'Local', + } + _PACKAGE_ALIASES = dict((v[0], (k, v[1])) for k, v in PACKAGE_ALIASES.items()) + _SERIALIZED_ATTRIBUTES = {'score': 1.0, 'location': '3'} + + def __init__(self, name=None, parent=None, score=1.0, location='L', + blacklist_re=None, locations=None): + self._name = name + self._tree = {} + self._exports = {} + self._parent = parent + if blacklist_re: + self._blacklist_re = blacklist_re + elif parent: + self._blacklist_re = parent._blacklist_re + else: + self._blacklist_re = DEFAULT_BLACKLIST_RE + self.score = score + self.location = location + if parent is None: + self._lib_locations = locations or LIB_LOCATIONS + else: + self._lib_locations = None + if parent is None: + self._merge_aliases() + with self.enter('__future__', location='F'): + pass + with self.enter('__builtin__', location='S'): + pass + + @property + def lib_locations(self): + if self._parent: + return self._parent.lib_locations + return self._lib_locations + + @classmethod + def deserialize(self, file): + def load(tree, data, parent_location): + for key, value in data.items(): + if isinstance(value, dict): + score = value.pop('.score', 1.0) + location = value.pop('.location', parent_location) + with tree.enter(key, score=score, location=location) as subtree: + load(subtree, value, location) + else: + assert isinstance(value, float), '%s expected to be float was %r' % (key, value) + tree.add(key, value) + + data = json.load(file) + data.pop('.location', None) + data.pop('.score', None) + tree = SymbolIndex(locations=data.pop('.lib_locations', LIB_LOCATIONS)) + load(tree, data, 'L') + return tree + + def index_source(self, filename, source): + try: + st = parse_ast(source, filename) + except Exception as e: + logger.debug('failed to parse %s: %s', filename, e) + return False + visitor = SymbolVisitor(self) + visitor.visit(st) + return True + + def index_file(self, module, filename): + if self._blacklist_re.search(filename): + return + logger.debug('parsing Python module %s for indexing', filename) + with open(filename, 'rb') as fd: + source = fd.read() + with self.enter(module, location=self._determine_location_for(filename)) as subtree: + success = subtree.index_source(filename, source) + if not success: + self._tree.pop(module, None) + + def index_path(self, root): + """Index a path. + + :param root: Either a package directory, a .so or a .py module. + """ + basename = os.path.basename(root) + if os.path.splitext(basename)[0] != '__init__' and basename.startswith('_'): + return + location = self._determine_location_for(root) + if os.path.isfile(root): + self._index_module(root, location) + elif os.path.isdir(root) and os.path.exists(os.path.join(root, '__init__.py')): + self._index_package(root, location) + + def _index_package(self, root, location): + basename = os.path.basename(root) + with self.enter(basename, location=location) as subtree: + for filename in os.listdir(root): + subtree.index_path(os.path.join(root, filename)) + + def _index_module(self, root, location): + basename, ext = os.path.splitext(os.path.basename(root)) + if basename == '__init__': + basename = None + ext = ext.lower() + import_path = '.'.join(filter(None, [self.path(), basename])) + if import_path in BUILTIN_MODULES: + return + if ext == '.py': + self.index_file(basename, root) + elif ext in ('.dll', '.so'): + self.index_builtin(import_path, location=location) + + def index_builtin(self, name, location): + basename = name.rsplit('.', 1)[-1] + if basename.startswith('_'): + return + logger.debug('importing builtin module %s for indexing', name) + try: + module = __import__(name, fromlist=['.']) + except Exception: + logger.debug('failed to index builtin module %s', name) + return + + with self.enter(basename, location=location) as subtree: + for key, value in vars(module).items(): + if not key.startswith('_'): + subtree.add(key, 1.1) + + def build_index(self, paths): + for builtin in BUILTIN_MODULES: + self.index_builtin(builtin, location='S') + for path in paths: + # for the implicit "" entry in sys.path + path = path or '.' + if os.path.isdir(path): + for filename in os.listdir(path): + filename = os.path.join(path, filename) + self.index_path(filename) + + def get_or_create_index(self, paths=None, name=None, refresh=False): + """ + Get index with given name from cache. Create if it doesn't exists. + """ + if not paths: + paths = sys.path + if not name: + name = 'default' + self._name = name + + idx_dir = get_cache_dir() + idx_file = os.path.join(idx_dir, name + '.json') + + if os.path.exists(idx_file) and not refresh: + with open(idx_file) as fd: + self.deserialize(fd) + else: + self.build_index(paths) + with open(idx_file, 'w') as fd: + self.serialize(fd) + + return self + + def symbol_scores(self, symbol): + """Find matches for symbol. + + :param symbol: A . separated symbol. eg. 'os.path.basename' + :returns: A list of tuples of (score, package, reference|None), + ordered by score from highest to lowest. + """ + scores = [] + path = [] + + # sys.path sys path -> import sys + # os.path.basename os.path basename -> import os.path + # basename os.path basename -> from os.path import basename + # path.basename os.path basename -> from os import path + def fixup(module, variable): + prefix = module.split('.') + if variable is not None: + prefix.append(variable) + seeking = symbol.split('.') + new_module = [] + while prefix and seeking[0] != prefix[0]: + new_module.append(prefix.pop(0)) + if new_module: + module, variable = '.'.join(new_module), prefix[0] + else: + variable = None + return module, variable + + def score_walk(scope, scale): + sub_path, score = self._score_key(scope, full_key) + if score > 0.1: + try: + i = sub_path.index(None) + sub_path, from_symbol = sub_path[:i], '.'.join(sub_path[i + 1:]) + except ValueError: + from_symbol = None + package_path = '.'.join(path + sub_path) + package_path, from_symbol = fixup(package_path, from_symbol) + scores.append((score * scale, package_path, from_symbol)) + + for key, subscope in scope._tree.items(): + if type(subscope) is not float: + path.append(key) + score_walk(subscope, subscope.score * scale - 0.1) + path.pop() + + full_key = symbol.split('.') + score_walk(self, 1.0) + scores.sort(reverse=True) + return scores + + def depth(self): + depth = 0 + node = self + while node._parent: + depth += 1 + node = node._parent + return depth + + def path(self): + path = [] + node = self + while node and node._name: + path.append(node._name) + node = node._parent + return '.'.join(reversed(path)) + + def add_explicit_export(self, name, score): + self._exports[name] = score + self.add(name, score) + + def find(self, path): + """Return the node for a path, or None.""" + path = path.split('.') + node = self + while node._parent: + node = node._parent + for name in path: + node = node._tree.get(name, None) + if node is None or type(node) is float: + return None + return node + + def location_for(self, path): + """Return the location code for a path.""" + path = path.split('.') + node = self + while node._parent: + node = node._parent + location = node.location + for name in path: + tree = node._tree.get(name, None) + if tree is None or type(tree) is float: + return location + location = tree.location + return location + + def add(self, name, score): + current_score = self._tree.get(name, 0.0) + if isinstance(current_score, float) and score > current_score: + self._tree[name] = score + + @contextmanager + def enter(self, name, location='L', score=1.0): + if name is None: + tree = self + else: + tree = self._tree.get(name) + if not isinstance(tree, SymbolIndex): + tree = self._tree[name] = SymbolIndex(name, self, score=score, location=location) + if tree.path() in SymbolIndex._PACKAGE_ALIASES: + alias_path, _ = SymbolIndex._PACKAGE_ALIASES[tree.path()] + alias = self.find(alias_path) + alias._tree = tree._tree + yield tree + if tree._exports: + # Delete unexported variables + for key in set(tree._tree) - set(tree._exports): + del tree._tree[key] + + def serialize(self, fd=None): + if fd is None: + return json.dumps(self, cls=JSONEncoder) + return json.dump(self, fd, cls=JSONEncoder) + + def boost(self): + return LOCATION_BOOSTS.get(self.location, 1.0) + + def __repr__(self): + return '<%s:%r %r>' % (self.location, self.score, self._tree) + + def _merge_aliases(self): + def create(node, alias, score): + if not alias: + return + name = alias.pop(0) + with node.enter(name, location='S', score=1.0 if alias else score) as index: + create(index, alias, score) + + # Sort the aliases to always create 'os' before 'os.path' + for alias in sorted(SymbolIndex._PACKAGE_ALIASES): + package, score = SymbolIndex._PACKAGE_ALIASES[alias] + create(self, package.split('.'), score) + + def _score_key(self, scope, key): + if not key: + return [], 0.0 + key_score = value = scope._tree.get(key[0], None) + if value is None: + return [], 0.0 + if type(value) is float: + return [None, key[0]], key_score * scope.boost() + else: + path, score = self._score_key(value, key[1:]) + return [key[0]] + path, (score + value.score) * scope.boost() + + def _determine_location_for(self, path): + parts = path.split(os.path.sep) + # Heuristic classifier + if 'site-packages' in parts: + return '3' + elif _PYTHON_VERSION in parts: + return 'S' + # Use table from sysconfig.get_python_lib() + for dir, location in self.lib_locations: + if path.startswith(dir): + return location + return 'L' + + +class SymbolVisitor(ast.NodeVisitor): + def __init__(self, tree): + self._tree = tree + + def visit_ImportFrom(self, node): + for name in node.names: + if name.name == '*' or name.name.startswith('_'): + continue + self._tree.add(name.name, 0.25) + + def visit_Import(self, node): + for name in node.names: + if name.name.startswith('_'): + continue + self._tree.add(name.name, 0.25) + + def visit_ClassDef(self, node): + if not node.name.startswith('_'): + self._tree.add(node.name, 1.1) + + def visit_FunctionDef(self, node): + if not node.name.startswith('_'): + self._tree.add(node.name, 1.1) + + def visit_Assign(self, node): + # TODO: Handle __all__ + is_name = lambda n: isinstance(n, ast.Name) + for name in filter(is_name, node.targets): + if name.id == '__all__' and isinstance(node.value, ast.List): + for subnode in node.value.elts: + if isinstance(subnode, ast.Str): + self._tree.add_explicit_export(subnode.s, 1.2) + elif not name.id.startswith('_'): + self._tree.add(name.id, 1.1) + + def visit_If(self, node): + # NOTE: In lieu of actually parsing if/else blocks at the top-level, + # we'll just ignore them. + pass + + +if __name__ == '__main__': + # print ast.dump(ast.parse(open('pyautoimp.py').read(), 'pyautoimp.py')) + tree = SymbolIndex() + tree.build_index(sys.path) + tree.serialize(sys.stdout) diff --git a/pythonFiles/importmagic/six.py b/pythonFiles/importmagic/six.py new file mode 100755 index 000000000000..58cb95b3c794 --- /dev/null +++ b/pythonFiles/importmagic/six.py @@ -0,0 +1,637 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2014 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.5.2" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + # This is a bit ugly, but it avoids running this again. + delattr(obj.__class__, self.name) + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + # Hack around the Django autoreloader. The reloader tries to get + # __file__ or __name__ of every module in sys.modules. This doesn't work + # well if this MovedModule is for an module that is unavailable on this + # machine (like winreg on Unix systems). Thus, we pretend __file__ and + # __name__ don't exist if the module hasn't been loaded yet. We give + # __path__ the same treatment for Google AppEngine. See issues #51, #53 + # and #56. + if (attr in ("__file__", "__name__", "__path__") and + self.mod not in sys.modules): + raise AttributeError + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + + +class _MovedItems(_LazyModule): + """Lazy loading of moved objects""" + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "xmlrpclib", "xmlrpc.server"), + MovedModule("winreg", "_winreg"), +] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + sys.modules[__name__ + ".moves." + attr.name] = attr +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +sys.modules[__name__ + ".moves.urllib_parse"] = sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse") + + +class Module_six_moves_urllib_error(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +sys.modules[__name__ + ".moves.urllib_error"] = sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +sys.modules[__name__ + ".moves.urllib_request"] = sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +sys.modules[__name__ + ".moves.urllib_response"] = sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +sys.modules[__name__ + ".moves.urllib_robotparser"] = sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + parse = sys.modules[__name__ + ".moves.urllib_parse"] + error = sys.modules[__name__ + ".moves.urllib_error"] + request = sys.modules[__name__ + ".moves.urllib_request"] + response = sys.modules[__name__ + ".moves.urllib_response"] + robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"] + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + + +sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" + + _iterkeys = "keys" + _itervalues = "values" + _iteritems = "items" + _iterlists = "lists" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + _iterkeys = "iterkeys" + _itervalues = "itervalues" + _iteritems = "iteritems" + _iterlists = "iterlists" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +def iterkeys(d, **kw): + """Return an iterator over the keys of a dictionary.""" + return iter(getattr(d, _iterkeys)(**kw)) + +def itervalues(d, **kw): + """Return an iterator over the values of a dictionary.""" + return iter(getattr(d, _itervalues)(**kw)) + +def iteritems(d, **kw): + """Return an iterator over the (key, value) pairs of a dictionary.""" + return iter(getattr(d, _iteritems)(**kw)) + +def iterlists(d, **kw): + """Return an iterator over the (key, [values]) pairs of a dictionary.""" + return iter(getattr(d, _iterlists)(**kw)) + + +if PY3: + def b(s): + return s.encode("latin-1") + def u(s): + return s + unichr = chr + if sys.version_info[1] <= 1: + def int2byte(i): + return bytes((i,)) + else: + # This is about 2x faster than the implementation above on 3.2+ + int2byte = operator.methodcaller("to_bytes", 1, "big") + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO +else: + def b(s): + return s + # Workaround for standalone backslash + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + def byte2int(bs): + return ord(bs[0]) + def indexbytes(buf, i): + return ord(buf[i]) + def iterbytes(buf): + return (ord(byte) for byte in buf) + import StringIO + StringIO = BytesIO = StringIO.StringIO +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + +_add_doc(reraise, """Reraise an exception.""") + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + return meta("NewBase", bases, {}) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper diff --git a/pythonFiles/importmagic/symbols.py b/pythonFiles/importmagic/symbols.py new file mode 100755 index 000000000000..9bd51d0314a3 --- /dev/null +++ b/pythonFiles/importmagic/symbols.py @@ -0,0 +1,409 @@ +"""Parse Python source and extract unresolved symbols.""" + +import ast +import sys +from contextlib import contextmanager +from itertools import chain + +from importmagic.six import string_types +from importmagic.util import parse_ast + + +try: + import builtins as __builtin__ +except: + import __builtin__ + + +class _InvalidSymbol(Exception): + pass + + +class Scope(object): + GLOBALS = ['__name__', '__file__', '__loader__', '__package__', '__path__'] + PYTHON3_BUILTINS = ['PermissionError'] + ALL_BUILTINS = set(dir(__builtin__)) | set(GLOBALS) | set(PYTHON3_BUILTINS) + + def __init__(self, parent=None, define_builtins=True, is_class=False): + self._parent = parent + self._definitions = set() + self._references = set() + self._children = [] + self._cursors = [self] + self._cursor = self + self._define_builtins = define_builtins + self._is_class = is_class + if define_builtins: + self._define_builtin_symbols() + self._add_symbol = [] + self._symbol = [] + + @contextmanager + def start_symbol(self): + self._add_symbol.append(self._add_symbol[-1] if self._add_symbol else self.reference) + try: + yield self + finally: + self.flush_symbol() + + @contextmanager + def start_definition(self): + self._add_symbol.append(self.define) + try: + yield self + finally: + self.flush_symbol() + + @contextmanager + def start_reference(self): + self._add_symbol.append(self.reference) + try: + yield self + finally: + self.flush_symbol() + + def extend_symbol(self, segment, extend_only=False): + if extend_only and not self._symbol: + return + self._symbol.append(segment) + + def end_symbol(self): + if self._symbol: + add = self._add_symbol[-1] if self._add_symbol else self.reference + add('.'.join(self._symbol)) + self._symbol = [] + + def flush_symbol(self): + self.end_symbol() + if self._add_symbol: + self._add_symbol.pop() + + @classmethod + def from_source(cls, src, trace=False, define_builtins=True): + scope = Scope(define_builtins=define_builtins) + visitor = UnknownSymbolVisitor(scope, trace=trace) + if isinstance(src, string_types): + src = parse_ast(src) + visitor.visit(src) + scope.flush_symbol() + return scope + + def _define_builtin_symbols(self): + self._cursor._definitions.update(Scope.ALL_BUILTINS) + + def define(self, name): + if '.' in name: + self.reference(name) + else: + self._cursor._definitions.add(name) + + def reference(self, name): + self._cursor._references.add(name) + + @contextmanager + def enter(self, is_class=False): + child = Scope(self._cursor, is_class=is_class, define_builtins=self._define_builtins) + self._cursor._children.append(child) + self._cursors.append(child) + self._cursor = child + try: + yield child + finally: + child.end_symbol() + self._cursors.pop() + self._cursor = self._cursors[-1] + + def find_unresolved_and_unreferenced_symbols(self): + """Find any unresolved symbols, and unreferenced symbols from this scope. + + :returns: ({unresolved}, {unreferenced}) + """ + unresolved = set() + unreferenced = self._definitions.copy() + self._collect_unresolved_and_unreferenced(set(), set(), unresolved, unreferenced, + frozenset(self._definitions), start=True) + return unresolved, unreferenced - Scope.ALL_BUILTINS + + def _collect_unresolved_and_unreferenced(self, definitions, definitions_excluding_top, + unresolved, unreferenced, top, start=False): + scope_definitions = definitions | self._definitions + scope_definitions_excluding_top = definitions_excluding_top | (set() if start else self._definitions) + + # When we're in a class, don't export definitions to descendant scopes + if not self._is_class: + definitions = scope_definitions + definitions_excluding_top = scope_definitions_excluding_top + + for reference in self._references: + symbols = set(_symbol_series(reference)) + # Symbol has no definition anywhere in ancestor scopes. + if symbols.isdisjoint(scope_definitions): + unresolved.add(reference) + # Symbol is referenced only in the top level scope. + elif not symbols.isdisjoint(top) and symbols.isdisjoint(scope_definitions_excluding_top): + unreferenced -= symbols + + # Recurse + for child in self._children: + child._collect_unresolved_and_unreferenced( + definitions, definitions_excluding_top, unresolved, unreferenced, top, + ) + + def __repr__(self): + return 'Scope(definitions=%r, references=%r, children=%r)' \ + % (self._definitions - Scope.ALL_BUILTINS, self._references, self._children) + + +def _symbol_series(s): + tokens = s.split('.') + return ['.'.join(tokens[:n + 1]) for n in range(len(tokens))] + + +class UnknownSymbolVisitor(ast.NodeVisitor): + def __init__(self, scope=None, trace=False): + super(UnknownSymbolVisitor, self).__init__() + self._scope = scope or Scope() + self._trace = trace + + def visit(self, node): + if node is None: + return + elif isinstance(node, list): + for subnode in node: + self.visit(subnode) + return + if self._trace: + print(node, vars(node)) + method = getattr(self, 'visit_%s' % node.__class__.__name__, None) + if method is not None: + try: + method(node) + except Exception: + # print >> sys.stderr, node, vars(node) + raise + else: + self.generic_visit(node) + self._scope.end_symbol() + + def visit_Raise(self, node): + if hasattr(node, 'type'): # Python 2: raise A[, B[, C]] + with self._scope.start_reference(): + self.visit(node.type) + with self._scope.start_reference(): + self.visit(node.inst) + with self._scope.start_reference(): + self.visit(node.tback) + else: # Python 3: raise A[ from B] + with self._scope.start_reference(): + self.visit(node.exc) + with self._scope.start_reference(): + self.visit(node.cause) + + def visit_TryExcept(self, node): + for sub in node.body: + with self._scope.start_reference(): + self.visit(sub) + self.visit(node.handlers) + for n in node.orelse: + with self._scope.start_reference(): + self.visit(n) + + def visit_ExceptHandler(self, node): + with self._scope.start_reference(): + self.visit(node.type) + with self._scope.start_definition(): + if isinstance(node.name, str): + # Python 3 + self._scope.extend_symbol(node.name) + else: + self.visit(node.name) + for n in node.body: + with self._scope.start_reference(): + self.visit(n) + + def visit_Return(self, node): + with self._scope.start_reference(): + self.visit(node.value) + + def visit_If(self, node): + with self._scope.start_reference(): + self.visit(node.test) + for child in node.body: + with self._scope.start_reference(): + self.visit(child) + for child in node.orelse: + with self._scope.start_reference(): + self.visit(child) + + def visit_While(self, node): + return self.visit_If(node) + + def visit_FunctionDef(self, node): + self._scope.define(node.name) + self.visit_Lambda(node) + + def visit_Lambda(self, node): + for decorator in getattr(node, 'decorator_list', []): + with self._scope.start_reference() as scope: + self.visit(decorator) + with self._scope.enter() as scope: + with scope.start_definition(): + args = node.args + for arg in [args.kwarg, args.vararg]: + if arg: + # arg is either an "arg" object (Python 3.4+) or a str + scope.define(arg.arg if hasattr(arg, 'arg') else arg) + # kwonlyargs was added in Python 3 + for arg in args.args + getattr(args, 'kwonlyargs', []): + scope.define(arg.id if hasattr(arg, 'id') else arg.arg) + + # Python 3 arguments annotation + if hasattr(arg, 'annotation') and arg.annotation: + self.visit(arg.annotation) + + for default in args.defaults: + self.visit(default) + + # Python 3 return annotation + if hasattr(node, 'returns'): + self.visit(node.returns) + body = [node.body] if isinstance(node, ast.Lambda) else node.body + with scope.start_reference(): + for statement in body: + self.visit(statement) + + def visit_ListComp(self, node): + return self.visit_GeneratorExp(node) + + def visit_Print(self, node): + for value in node.values: + with self._scope.start_reference(): + self.visit(value) + if node.dest: + with self._scope.start_reference(): + self.visit(node.dest) + + def visit_GeneratorExp(self, node): + with self._scope.start_reference(): + self.visit(node.elt) + for elt in node.generators: + self.visit(elt) + + def visit_comprehension(self, node): + with self._scope.start_definition(): + self.visit(node.target) + with self._scope.start_reference(): + self.visit(node.iter) + for elt in node.ifs: + with self._scope.start_reference(): + self.visit(elt) + + def visit_Assign(self, node): + for target in node.targets: + with self._scope.start_definition(): + self.visit(target) + with self._scope.start_reference(): + self.visit(node.value) + + def visit_ClassDef(self, node): + for decorator in getattr(node, 'decorator_list', []): + with self._scope.start_reference(): + self.visit(decorator) + self._scope.define(node.name) + for base in node.bases: + with self._scope.start_reference(): + self.visit(base) + with self._scope.enter(is_class=True): + for body in node.body: + with self._scope.start_reference(): + self.visit(body) + + def visit_ImportFrom(self, node): + for name in node.names: + if name.name == '*': + # TODO: Do something? + continue + symbol = name.asname or name.name.split('.')[0] + self._scope.define(symbol) + # Explicitly add a reference for __future__ imports so they don't + # get pruned. + if node.module == '__future__': + self._scope.reference(symbol) + self.generic_visit(node) + + def visit_Import(self, node): + for name in node.names: + self._scope.define(name.asname or name.name) + self.generic_visit(node) + + def visit_With(self, node): + if hasattr(node, 'items'): + for item in node.items: + self._visit_withitem(item) + else: + self._visit_withitem(node) + with self._scope.start_reference(): + self.visit(node.body) + + def _visit_withitem(self, node): + if node.optional_vars: + with self._scope.start_definition(): + self.visit(node.optional_vars) + with self._scope.start_reference(): + self.visit(node.context_expr) + + def visit_For(self, node): + with self._scope.start_definition(): + self.visit(node.target) + with self._scope.start_reference(): + self.visit(node.iter) + with self._scope.start_reference(): + self.visit(node.body) + with self._scope.start_reference(): + self.visit(node.orelse) + + def visit_Attribute(self, node, chain=False): + if isinstance(node.value, ast.Name): + self._scope.extend_symbol(node.value.id) + self._scope.extend_symbol(node.attr) + if not chain: + self._scope.end_symbol() + elif isinstance(node.value, ast.Attribute): + self.visit_Attribute(node.value, chain=True) + self._scope.extend_symbol(node.attr, extend_only=True) + else: + self._scope.end_symbol() + self.visit(node.value) + self._scope.end_symbol() + + def visit_Subscript(self, node): + self._scope.end_symbol() + with self._scope.start_reference(): + self.visit(node.value) + self.visit(node.slice) + + def visit_Call(self, node): + with self._scope.start_reference(): + self.visit(node.func) + # Python 3.5 AST removed starargs and kwargs + additional = [] + if getattr(node, 'starargs', None): + additional.append(node.starargs) + if getattr(node, 'kwargs', None): + additional.append(node.kwargs) + for arg in chain(node.args, node.keywords, additional): + with self._scope.start_reference(): + self.visit(arg) + + def visit_Name(self, node): + self._scope.extend_symbol(node.id) + self._scope.end_symbol() + + +if __name__ == '__main__': + with open(sys.argv[1]) as fd: + scope = Scope.from_source(fd.read()) + unresolved, unreferenced = scope.find_unresolved_and_unreferenced_symbols() + from pprint import pprint + pprint(unresolved) + pprint(unreferenced) diff --git a/pythonFiles/importmagic/util.py b/pythonFiles/importmagic/util.py new file mode 100755 index 000000000000..9007580b2b90 --- /dev/null +++ b/pythonFiles/importmagic/util.py @@ -0,0 +1,73 @@ +import ast +import os +import re +import sys +from sys import platform +from ast import AST, iter_fields + +from importmagic.six import text_type + + +CODING_COOKIE_RE = re.compile('(^\s*#.*)coding[:=]', re.M) + + +def parse_ast(source, filename=None): + """Parse source into a Python AST, taking care of encoding.""" + if isinstance(source, text_type) and sys.version_info[0] == 2: + # ast.parse() on Python 2 doesn't like encoding declarations + # in Unicode strings + source = CODING_COOKIE_RE.sub(r'\1', source, 1) + return ast.parse(source, filename or '') + + +def dump(node, annotate_fields=True, include_attributes=False, indent=' '): + """Return a formatted dump of the tree in *node*. + + This is mainly useful for debugging purposes. The returned string will + show the names and the values for fields. This makes the code impossible + to evaluate, so if evaluation is wanted *annotate_fields* must be set to + False. Attributes such as line numbers and column offsets are not dumped + by default. If this is wanted, *include_attributes* can be set to True. + """ + def _format(node, level=0): + if isinstance(node, AST): + fields = [(a, _format(b, level + 1)) for a, b in iter_fields(node)] + if include_attributes and node._attributes: + fields.extend([(a, _format(getattr(node, a), level + 1)) + for a in node._attributes]) + return ''.join([ + node.__class__.__name__, + '(\n' + indent + indent * level if fields else '(', + (',\n' + indent + indent * level).join(('%s=%s' % field for field in fields) + if annotate_fields else + (b for a, b in fields)), + ')']) + elif isinstance(node, list): + lines = ['['] + lines.extend((indent * (level + 2) + _format(x, level + 2) + ',' + for x in node)) + if len(lines) > 1: + lines.append(indent * (level + 1) + ']') + else: + lines[-1] += ']' + return '\n'.join(lines) + return repr(node) + + if not isinstance(node, AST): + raise TypeError('expected AST, got %r' % node.__class__.__name__) + return _format(node) + + +def get_cache_dir(): + if platform.startswith('linux'): + cache_dir = '~/.cache/importmagic/' + elif platform == 'darwin': + cache_dir = '~/Library/Caches/importmagic' + elif platform in ('win32', 'cygwin'): + cache_dir = '%APPDATA%\importmagic' + + cache_dir = os.path.expanduser(cache_dir) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + return cache_dir diff --git a/src/client/extension.ts b/src/client/extension.ts index 9da8774ad0cc..3fd05626a590 100644 --- a/src/client/extension.ts +++ b/src/client/extension.ts @@ -60,6 +60,9 @@ import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants'; import * as tests from './unittests/main'; import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry'; import { WorkspaceSymbols } from './workspaceSymbols/main'; +import { ImportMagicCompletionItemProvider } from './providers/importMagicCompletionProvider'; +import { ImportMagicProxyFactory } from './languageServices/importMagicProxyFactory'; +import { ImportMagicProvider } from './providers/importMagicProvider'; const PYTHON: vscode.DocumentFilter = { language: 'python' }; const activationDeferred = createDeferred(); @@ -121,6 +124,11 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(new ReplProvider(serviceContainer)); context.subscriptions.push(new TerminalProvider(serviceContainer)); + // Initialize importMagic + const importMagicFactory = new ImportMagicProxyFactory(context.asAbsolutePath('.'), serviceContainer); + context.subscriptions.push(vscode.languages.registerCompletionItemProvider(PYTHON, new ImportMagicCompletionItemProvider(importMagicFactory), '.')); + context.subscriptions.push(new ImportMagicProvider(importMagicFactory)); + // Enable indentAction // tslint:disable-next-line:no-non-null-assertion vscode.languages.setLanguageConfiguration(PYTHON.language!, { diff --git a/src/client/languageServices/importMagicProxyFactory.ts b/src/client/languageServices/importMagicProxyFactory.ts new file mode 100644 index 000000000000..5a8471f33a7c --- /dev/null +++ b/src/client/languageServices/importMagicProxyFactory.ts @@ -0,0 +1,40 @@ +import { Disposable, Uri, workspace } from 'vscode'; +import { IServiceContainer } from '../ioc/types'; +import { ImportMagicProxy } from '../providers/importMagicProxy'; + +export class ImportMagicProxyFactory implements Disposable { + private disposables: Disposable[]; + private proxyHandlers: Map; + + constructor(private extensionRootPath: string, private serviceContainer: IServiceContainer) { + this.disposables = []; + this.proxyHandlers = new Map(); + } + + public dispose() { + this.disposables.forEach(disposable => disposable.dispose()); + this.disposables = []; + } + + public getImportMagicProxy(resource: Uri): ImportMagicProxy { + const workspaceFolder = workspace.getWorkspaceFolder(resource); + let workspacePath = workspaceFolder ? workspaceFolder.uri.fsPath : undefined; + if (!workspacePath) { + if (Array.isArray(workspace.workspaceFolders) && workspace.workspaceFolders.length > 0) { + workspacePath = workspace.workspaceFolders[0].uri.fsPath; + } else { + workspacePath = __dirname; + } + } + + let importMagic = this.proxyHandlers.get(workspacePath); + + if (!importMagic) { + importMagic = new ImportMagicProxy(this.extensionRootPath, this.serviceContainer, workspacePath); + this.disposables.push(importMagic); + this.proxyHandlers.set(workspacePath, importMagic); + } + + return importMagic; + } +} diff --git a/src/client/providers/importMagicCompletionProvider.ts b/src/client/providers/importMagicCompletionProvider.ts new file mode 100644 index 000000000000..30f179499a78 --- /dev/null +++ b/src/client/providers/importMagicCompletionProvider.ts @@ -0,0 +1,90 @@ +import * as vscode from 'vscode'; +import { CompletionContext, commands, Disposable } from 'vscode'; +import { isTestExecution } from '../common/configSettings'; +import { ImportMagicProxyFactory } from '../languageServices/importMagicProxyFactory'; +import { IResultSymbols, MethodType, ICommandSymbols, ISuggestionSymbol } from './importMagicProxy'; + +export class ImportMagicCompletionItemProvider implements vscode.CompletionItemProvider { + private disposables: Disposable[] = []; + private enable: boolean = false; + + constructor(private importMagicFactory: ImportMagicProxyFactory) { + this.disposables.push(commands.registerCommand('importMagic.findSymbols', this.toggleFindSymbols.bind(this))); + } + + public dispose() { + this.disposables.forEach(disposable => disposable.dispose()); + } + + public async provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, + token: vscode.CancellationToken, context: CompletionContext): Promise { + if (!this.enable) { + return Promise.resolve([]); + } + + const result = await this.getCompletionResult(document, position, token); + this.enable = false; + + if (result === undefined) { + return Promise.resolve([]); + } + + return result; + } + + private toggleFindSymbols() { + this.enable = true; + if (!isTestExecution()) { + commands.executeCommand('editor.action.triggerSuggest'); + } + } + + private async getCompletionResult(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken) + : Promise { + const range = document.getWordRangeAtPosition(position); + const text = document.getText(range); + if (!text) { + return undefined; + } + + const importMagic = this.importMagicFactory.getImportMagicProxy(document.uri); + const cmd: ICommandSymbols = { + method: MethodType.Symbols, + text + }; + const result = await importMagic.sendCommand(cmd); + return result.items.map(item => this.toVsCodeCompletion(document, position, item)); + } + + private toVsCodeCompletion(document: vscode.TextDocument, position: vscode.Position, item: ISuggestionSymbol): vscode.CompletionItem { + const completionItem = new vscode.CompletionItem(item.key); + switch (item.score) { + case 1.1: + completionItem.kind = vscode.CompletionItemKind.Class; + break; + case 0.25: + completionItem.kind = vscode.CompletionItemKind.Reference; + break; + case 1.2: + completionItem.kind = vscode.CompletionItemKind.Function; + break; + case 1: + completionItem.kind = vscode.CompletionItemKind.Module; + break; + default: + completionItem.kind = vscode.CompletionItemKind.Text; + } + + completionItem.command = { + command: 'importMagic.insertImport', + title: 'Import Magic', + arguments: [item.module, item.variable] + }; + + if (item.variable) { + completionItem.detail = `from ${item.module}`; + } + + return completionItem; + } +} diff --git a/src/client/providers/importMagicProvider.ts b/src/client/providers/importMagicProvider.ts new file mode 100644 index 000000000000..dcb47794290f --- /dev/null +++ b/src/client/providers/importMagicProvider.ts @@ -0,0 +1,176 @@ +import * as fs from 'fs-extra'; +import {commands, Disposable, Position, QuickPickItem, QuickPickOptions, Range, TextDocument, window, workspace} from 'vscode'; +import { getTempFileWithDocumentContents } from '../common/editor'; +import { IResultSuggestions, MethodType, ICommandSuggestions, ICommandImport, IResultImport, ISuggestionItem } from './importMagicProxy'; +import { ImportMagicProxyFactory } from './../languageServices/importMagicProxyFactory'; + +export interface ImportPathQuickPickItem extends QuickPickItem { + module: string; + variable?: string; +} + +export class ImportMagicProvider implements Disposable { + private disposables: Disposable[] = []; + + constructor(private importMagicFactory: ImportMagicProxyFactory, noReg: boolean = false) { + if (!noReg) { + this.disposables.push(commands.registerCommand('importMagic.resolveImport', this.resolveImport.bind(this))); + this.disposables.push(commands.registerCommand('importMagic.insertImport', this.insertImport.bind(this))); + this.disposables.push(workspace.onDidSaveTextDocument(this.onSave.bind(this))); + } + } + + public dispose() { + this.disposables.forEach(disposable => disposable.dispose()); + } + + public async getImportSuggestions(sourceFile: string, unresolvedName: string): Promise { + const activeEditor = window.activeTextEditor; + if (!activeEditor) { + return Promise.resolve([]); + } + + const cmd: ICommandSuggestions = { + method: MethodType.Suggestions, + sourceFile, + unresolvedName + }; + + try{ + const importMagic = this.importMagicFactory.getImportMagicProxy(activeEditor.document.uri); + + const result = await importMagic.sendCommand(cmd); + const candidates = result.candidates; + + candidates.sort((a, b) => a.score > b.score ? -1 : 1); + const suggestions: ImportPathQuickPickItem[] = candidates.map(item => this.suggestionToQuickPickItem(item)); + return Promise.resolve(suggestions); + } catch (e) { + window.showErrorMessage(e.message); + return Promise.resolve([]); + } + } + + private async onSave(document: TextDocument) { + if (document.fileName.endsWith('.py')) { + const importMagic = this.importMagicFactory.getImportMagicProxy(document.uri); + await importMagic.rebuildIndex(); + } + } + + private async resolveImport() { + const activeEditor = window.activeTextEditor; + if (!activeEditor) { + return Promise.resolve(); + } + + const document = activeEditor.document; + + if (!activeEditor || document.languageId !== 'python') { + window.showErrorMessage('Please open a Python source file to show import suggestions.'); + return Promise.resolve(); + } + + // Get current selected name + const position: Position = activeEditor.selection.start; + const range = document.getWordRangeAtPosition(position); + if (!range || range.isEmpty) { + window.showErrorMessage('Empty resolve expression'); + return Promise.resolve(); + } + const unresolvedName : string = document.getText(range); + + if (!unresolvedName) { + window.showErrorMessage('Empty resolve expression'); + return Promise.resolve(); + } + + const tmpFileCreated = document.isDirty; + const filePath = tmpFileCreated ? await getTempFileWithDocumentContents(document) : document.fileName; + + try { + const quickPickOptions: QuickPickOptions = { + matchOnDetail: true, + matchOnDescription: true, + placeHolder: `Import statement for ${unresolvedName}` + }; + + const suggestions = this.getImportSuggestions(filePath, unresolvedName); + const selection = await window.showQuickPick(suggestions, quickPickOptions); + + if (selection !== undefined) { + commands.executeCommand('importMagic.insertImport', selection.module, selection.variable); + } + } finally { + if (tmpFileCreated) { + fs.unlinkSync(filePath); + } + } + } + + private suggestionToQuickPickItem(suggestion: ISuggestionItem): ImportPathQuickPickItem { + const module = suggestion.module; + const variable = suggestion.variable; + const path = variable ? `from ${module} import ${variable}` : `import ${module}`; + + return { + label: path, + description: '', // suggestion.score.toPrecision(3), + module: module, + variable: variable ? variable : undefined + }; + } + + private async insertImport(module: string, variable?: string) { + const activeEditor = window.activeTextEditor; + if (!activeEditor) { + return Promise.resolve(); + } + + const document = activeEditor.document; + + if (!activeEditor || document.languageId !== 'python') { + window.showErrorMessage('Please open a Python source file to show import suggestions.'); + return Promise.resolve(); + } + + const tmpFileCreated = document.isDirty; + const filePath = tmpFileCreated ? await getTempFileWithDocumentContents(document) : document.fileName; + + try { + const cmd: ICommandImport = { + method: MethodType.Import, + sourceFile: filePath, + module, + variable + }; + + const importMagic = this.importMagicFactory.getImportMagicProxy(document.uri); + await this.updateSource(await importMagic.sendCommand(cmd)); + } catch (e) { + window.showErrorMessage(e.message); + } finally { + if (tmpFileCreated) { + fs.unlinkSync(filePath); + } + } + } + + private async updateSource(update: IResultImport) { + const activeEditor = window.activeTextEditor; + if (!activeEditor) { + return undefined; + } + + // select from fromLine to endLine and paste text text + const start: Position = new Position(update.fromLine, 0); + const end: Position = new Position(update.endLine, 0); + + let range: Range = new Range(start, end); + range = activeEditor.document.validateRange(range); + + return activeEditor.edit(builder => { + builder.replace(range, update.text); + }); + } +} diff --git a/src/client/providers/importMagicProxy.ts b/src/client/providers/importMagicProxy.ts new file mode 100644 index 000000000000..d913743e48d5 --- /dev/null +++ b/src/client/providers/importMagicProxy.ts @@ -0,0 +1,360 @@ +import { ICommandResult } from './importMagicProxy'; +import * as logger from './../common/logger'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { ChildProcess } from 'child_process'; +import { createDeferred, Deferred } from '../common/helpers'; +import { IPythonExecutionFactory } from '../common/process/types'; +import { IServiceContainer } from './../ioc/types'; +import { PythonSettings, isTestExecution } from '../common/configSettings'; +import { debounce } from '../common/decorators'; + +export enum MethodType { + BuildIndex = 'build_index', + Suggestions = 'import_suggestions', + Import = 'insert_import', + Symbols = 'get_symbols' +} + +export interface ICommandResult { + requestId?: number; +} + +interface IResultBuild extends ICommandResult { + success: boolean; +} + +export interface ISuggestionItem { + score: number; + module: string; + variable?: string; +} + +export interface IResultSuggestions extends ICommandResult { + candidates: ISuggestionItem[]; +} + +export interface ISuggestionSymbol { + key: string; + score: number; + module: string; + variable?: string; + depth: number; +} + +export interface IResultSymbols extends ICommandResult { + items: ISuggestionSymbol[]; +} + +export interface IResultImport extends ICommandResult { + fromLine: number; + endLine: number; + text: string; +} + +interface IResultError extends ICommandResult { + error: boolean; + message: string; + traceback: string; + type: string; +} + +export interface ICommand { + method: MethodType; + deferred?: Deferred; + commandId?: number; + text?: string; +} + +interface ICommandBuild extends ICommand { + workspacePath: string; + extraPaths: string[]; + skipTestFolders: boolean; + forceRebuild: boolean; +} + +export interface ICommandSuggestions extends ICommand { + sourceFile: string; + unresolvedName: string; +} + +export interface ICommandSymbols extends ICommand { + text: string; +} + +export interface ICommandImport extends ICommand { + sourceFile: string; + module: string; + variable?: string; +} + +export class ImportMagicProxy { + private pythonSettings: PythonSettings; + private workspacePath: string; + + private proc: ChildProcess | null; + private previousData = ''; + + private languageServerStarted: Deferred; + private commands = new Map>(); + private commandId: number = 0; + private restartAttempts: number = 0; + + constructor(private extensionRootDir: string, private serviceContainer: IServiceContainer, workspacePath: string) { + this.workspacePath = workspacePath; + this.pythonSettings = PythonSettings.getInstance(vscode.Uri.file(workspacePath)); + this.pythonSettings.on('change', () => this.onChangeSettings()); + + this.restartServer(); + } + + private static getProperty(o: object, name: string): T { + return o[name]; + } + + public dispose() { + this.killProcess(); + } + + public async sendCommand(cmd: ICommand): Promise { + await this.languageServerStarted.promise; + return this.sendRequest(cmd); + } + + @debounce(10000) + public async rebuildIndex() { + await this.buildIndex(true); + } + + private onChangeSettings() { + this.restartAttempts = 0; + this.restartServer(); + } + + private restartServer() { + this.languageServerStarted = createDeferred(); + this.killProcess(); + this.clearPendingRequests(); + this.spawnProcess(); + } + + private async spawnProcess() { + this.restartAttempts += 1; + const cwd: string = path.join(this.extensionRootDir, 'pythonFiles'); + const pythonProcess = await this.serviceContainer.get(IPythonExecutionFactory).create(vscode.Uri.file(this.workspacePath)); + + const args = ['importMagic.py']; + const result = pythonProcess.execObservable(args, { cwd }); + this.proc = result.proc; + + result.proc.on('end', (end) => { + logger.error('spawnProcess.end', `End - ${end}`); + }); + result.proc.on('error', error => { + logger.error('Error', `${error}`); + }); + result.out.subscribe(output => { + const data = output.out; + const dataStr = this.previousData = `${this.previousData}${data}`; + + if (dataStr.endsWith('\n')) { + this.previousData = ''; + this.onData(dataStr); + } + }); + + await this.buildIndex(); + this.languageServerStarted.resolve(); + } + + private async buildIndex(forceRebuild: boolean = false) { + const skipTestFolders = !isTestExecution(); + + const cmd: ICommandBuild = { + method: MethodType.BuildIndex, + workspacePath: this.workspacePath, + extraPaths: this.getExtraPaths(), + skipTestFolders, + forceRebuild + }; + + await this.sendRequest(cmd); + } + + private killProcess() { + try { + if (this.proc) { + this.proc.kill(); + } + // tslint:disable-next-line:no-empty + } catch { } + this.proc = null; + } + + private getExtraPaths() { + // Add support for paths relative to workspace. + return this.pythonSettings.autoComplete.extraPaths.map(extraPath => { + if (path.isAbsolute(extraPath)) { + return extraPath; + } + if (typeof this.workspacePath !== 'string') { + return ''; + } + return path.join(this.workspacePath, extraPath); + }); + } + + private sendRequest(cmd: ICommand): Promise { + this.commandId += 1; + + const extendedPayload = { + requestId: this.commandId, + ...cmd + }; + + const executionCmd = cmd; + executionCmd.deferred = createDeferred(); + executionCmd.commandId = this.commandId; + + try { + if (!this.proc) { + throw new Error('ImportMagic process is die'); + } + + this.proc.stdin.write(`${JSON.stringify(extendedPayload)}\n`); + this.commands.set(this.commandId, executionCmd); + }catch (ex) { + logger.error('ImportMagicProxy', `Error "${ex.message}"`); + if (this.restartAttempts < 10) { + this.restartServer(); + return Promise.reject('ImportMagic process will be restarted. Try run command again'); + } else { + return Promise.reject('ImportMagic can not start the process'); + } + } + return executionCmd.deferred.promise; + } + + private clearPendingRequests() { + this.commands.forEach(item => { + if (item.deferred !== undefined) { + item.deferred.reject(); + } + }); + this.commands.clear(); + } + + private onData(dataStr: string) { + try{ + const response = JSON.parse(dataStr); + const responseId = ImportMagicProxy.getProperty(response, 'id'); + + if (!responseId) { + logger.error('ImportMagicProxy', 'Response not contain id'); + return; + } + + const cmd = this.commands.get(responseId); + if (!cmd) { + return; + } + this.commands.delete(responseId); + + const isError: boolean = response.error ? true : false; + const handler = isError ? this.onError : this.getCommandHandler(cmd.method); + if (!handler) { + return; + } + + const result = handler.call(this, cmd, response); + + if (cmd && cmd.deferred) { + if (isError) { + cmd.deferred.reject(result); + } else { + cmd.deferred.resolve(result); + } + } + // tslint:disable-next-line:no-empty + } catch { } + } + + private getCommandHandler(command: MethodType): undefined | ((command: ICommand, response: object) => ICommandResult) { + switch (command) { + case MethodType.BuildIndex: + return this.onBuild; + case MethodType.Suggestions: + return this.onSuggestions; + case MethodType.Import: + return this.onImport; + case MethodType.Symbols: + return this.onSymbols; + default: + return; + } + } + + private onError(command: ICommand, response: object): IResultError { + return { + requestId: command.commandId, + error: true, + message: ImportMagicProxy.getProperty(response, 'message'), + traceback: ImportMagicProxy.getProperty(response, 'traceback'), + type: ImportMagicProxy.getProperty(response, 'type') + }; + } + + private onBuild(command: ICommand, response: object): IResultBuild { + return { + requestId: command.commandId, + success: ImportMagicProxy.getProperty(response, 'success') + }; + } + + private onSuggestions(command: ICommand, response: object): IResultSuggestions { + let candidates = ImportMagicProxy.getProperty(response, 'candidates'); + + candidates = Array.isArray(candidates) ? candidates : []; + + const suggestionItems: ISuggestionItem[] = candidates.map(item => { + return { + score: (ImportMagicProxy.getProperty(item, 'score') || 0), + module: ImportMagicProxy.getProperty(item, 'module'), + variable: ImportMagicProxy.getProperty(item, 'variable') || undefined + }; + }); + + return { + requestId: command.commandId, + candidates: suggestionItems + }; + } + + private onImport(command: ICommand, response: object): IResultImport { + return { + requestId: command.commandId, + fromLine: ImportMagicProxy.getProperty(response, 'fromLine'), + endLine: ImportMagicProxy.getProperty(response, 'endLine'), + text: ImportMagicProxy.getProperty(response, 'text') || '' + }; + } + + private onSymbols(command: ICommand, response: object): IResultSymbols { + let items = ImportMagicProxy.getProperty(response, 'items'); + items = Array.isArray(items) ? items : []; + + const suggestionItems: ISuggestionSymbol[] = items.map(item => { + return { + key: ImportMagicProxy.getProperty(item, 'key'), + module: ImportMagicProxy.getProperty(item, 'module'), + variable: ImportMagicProxy.getProperty(item, 'variable') || undefined, + depth: ImportMagicProxy.getProperty(item, 'depth'), + score: ImportMagicProxy.getProperty(item, 'score') || 0 + }; + }); + return { + requestId: command.commandId, + items: suggestionItems + }; + } +} diff --git a/src/test/importMagic/base.test.ts b/src/test/importMagic/base.test.ts new file mode 100644 index 000000000000..35d1d2294143 --- /dev/null +++ b/src/test/importMagic/base.test.ts @@ -0,0 +1,111 @@ +import * as assert from 'assert'; +import * as path from 'path'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; +import { UnitTestIocContainer } from '../unittests/serviceRegistry'; +import * as vscode from 'vscode'; +import { Uri, workspace, window, commands, WorkspaceConfiguration } from 'vscode'; +import { ImportMagicProxyFactory } from './../../client/languageServices/importMagicProxyFactory'; +import { ImportMagicProvider } from './../../client/providers/importMagicProvider'; + +const importMagicPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'importMagic'); +const extensionRootDir = path.join(__dirname, '..', '..', '..'); + +const fileOne = path.join(importMagicPath, 'one.py'); +const fileCompl = path.join(importMagicPath, 'compl.py'); + +// tslint:disable-next-line:max-func-body-length +suite('ImportMagic', () => { + let ioc: UnitTestIocContainer; + let im: ImportMagicProvider; + let importMagicProxyFactory: ImportMagicProxyFactory; + let settings: WorkspaceConfiguration; + let oldExtraPaths: string[] | undefined; + + suiteSetup(async () => { + await initialize(); + initializeDI(); + + importMagicProxyFactory = new ImportMagicProxyFactory(extensionRootDir, ioc.serviceContainer); + + settings = workspace.getConfiguration('python', Uri.file(importMagicPath)); + oldExtraPaths = settings.get('autoComplete.extraPaths'); + await settings.update('autoComplete.extraPaths', [importMagicPath]); + }); + setup(async () => { + await initializeTest(); + initializeDI(); + + im = new ImportMagicProvider(importMagicProxyFactory, true); + }); + + suiteTeardown(async () => { + closeActiveWindows(); + importMagicProxyFactory.dispose(); + await settings.update('autoComplete.extraPaths', oldExtraPaths); + }); + + teardown(async () => { + await closeActiveWindows(); + ioc.dispose(); + im.dispose(); + }); + + function initializeDI() { + ioc = new UnitTestIocContainer(); + ioc.registerCommonTypes(); + ioc.registerVariableTypes(); + ioc.registerProcessTypes(); + } + + test('Import candidates from project folder', async () => { + const textDocument = await workspace.openTextDocument(fileOne); + await window.showTextDocument(textDocument); + const suggestions = await im.getImportSuggestions(fileOne, 'FirstClass'); + assert.equal(suggestions.length > 0, true); + assert.equal(suggestions[0].variable, 'FirstClass'); + assert.equal(suggestions[0].label, 'from classes import FirstClass'); + }); + + test('Import candidates from system module', async () => { + const textDocument = await workspace.openTextDocument(fileOne); + await window.showTextDocument(textDocument); + const suggestions = await im.getImportSuggestions(fileOne, 'sys'); + assert.equal(suggestions.length > 0, true); + assert.equal(suggestions[0].variable, undefined); + assert.equal(suggestions[0].module, 'sys'); + assert.equal(suggestions[0].label, 'import sys'); + }); + + test('Import candidates from system modules', async () => { + const textDocument = await workspace.openTextDocument(fileOne); + await window.showTextDocument(textDocument); + const suggestions = await im.getImportSuggestions(fileOne, 'path'); + assert.equal(suggestions.length > 0, true); + assert.equal(suggestions.filter(value => value.label === 'from os import path').length, 1); + assert.equal(suggestions.filter(value => value.label === 'from sys import path').length, 1); + }); + + test('Insert import', async () => { + const textDocument = await workspace.openTextDocument(fileOne); + const originalContent = textDocument.getText(); + + await window.showTextDocument(textDocument); + const suggestions = await im.getImportSuggestions(fileOne, 'FirstClass'); + const selection = suggestions[0]; + await commands.executeCommand('importMagic.insertImport', selection.module, selection.variable); + assert.notEqual(originalContent, textDocument.getText(), 'Contents have not changed'); + }); + + test('Completion provider get symbols', async () => { + const textDocument = await workspace.openTextDocument(fileCompl); + + await window.showTextDocument(textDocument); + const position = new vscode.Position(0, 34); + + await commands.executeCommand('importMagic.findSymbols'); + const list = await commands.executeCommand('vscode.executeCompletionItemProvider', textDocument.uri, position); + + assert.equal(list!.items.length > 0, true); + assert.equal(list!.items.filter(item => item.label === 'AnotherClass2' && item.detail === 'from classes2').length, 1); + }); +}); diff --git a/src/test/pythonFiles/importMagic/classes.py b/src/test/pythonFiles/importMagic/classes.py new file mode 100644 index 000000000000..8c919dac03ee --- /dev/null +++ b/src/test/pythonFiles/importMagic/classes.py @@ -0,0 +1,14 @@ +class FirstClass(object): + def method(self): + pass + + @staticmethod + def static_method(): + pass + + @classmethod + def class_method(cls, **kwargs): + pass + +def method1(): + pass diff --git a/src/test/pythonFiles/importMagic/classes2.py b/src/test/pythonFiles/importMagic/classes2.py new file mode 100644 index 000000000000..7b7c6985aab2 --- /dev/null +++ b/src/test/pythonFiles/importMagic/classes2.py @@ -0,0 +1,5 @@ +class Class2(object): + pass + +class AnotherClass2(object): + pass diff --git a/src/test/pythonFiles/importMagic/compl.py b/src/test/pythonFiles/importMagic/compl.py new file mode 100644 index 000000000000..96d8162e6a3d --- /dev/null +++ b/src/test/pythonFiles/importMagic/compl.py @@ -0,0 +1 @@ +class SecondClass(FirstClass, Anot diff --git a/src/test/pythonFiles/importMagic/methods.py b/src/test/pythonFiles/importMagic/methods.py new file mode 100644 index 000000000000..6243af23699f --- /dev/null +++ b/src/test/pythonFiles/importMagic/methods.py @@ -0,0 +1,2 @@ +def method2(): + pass diff --git a/src/test/pythonFiles/importMagic/one.py b/src/test/pythonFiles/importMagic/one.py new file mode 100644 index 000000000000..2d7469acd214 --- /dev/null +++ b/src/test/pythonFiles/importMagic/one.py @@ -0,0 +1,7 @@ +import os +from classes2 import Class2 + +class SecondClass(FirstClass, Class2): + def aaa(self): + a = sys.path() + From cce3245be262e33a23a79255955c2a08e7bdbb7a Mon Sep 17 00:00:00 2001 From: Vladimir K Urushev Date: Fri, 26 Jan 2018 07:51:38 +0300 Subject: [PATCH 2/3] Fix indents --- package.json | 6 +++--- package.nls.json | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index afb44fb5b538..bd754275d41f 100644 --- a/package.json +++ b/package.json @@ -78,9 +78,9 @@ "onCommand:python.startREPL", "onCommand:python.goToPythonObject", "onCommand:python.setLinter", - "onCommand:python.enableLinting", - "onCommand:python.createTerminal", - "onCommand:importMagic.resolveImport", + "onCommand:python.enableLinting", + "onCommand:python.createTerminal", + "onCommand:importMagic.resolveImport", "onCommand:importMagic.insertImport", "onCommand:importMagic.findSymbols" ], diff --git a/package.nls.json b/package.nls.json index 2f3cf00425c7..c16388e32465 100644 --- a/package.nls.json +++ b/package.nls.json @@ -49,8 +49,8 @@ "python.snippet.launch.attach.label": "Python: Attach", "python.snippet.launch.attach.description": "Attach the debugger for remote debugging", "python.snippet.launch.scrapy.label": "Python: Scrapy", - "python.snippet.launch.scrapy.description": "Scrapy with Integrated Terminal/Console", - "python.command.importMagic.resolveImport.title": "Resolve Imports", - "python.command.importMagic.insertImport.title": "Insert Import", - "python.command.importMagic.findSymbols.title": "Find Symbols" + "python.snippet.launch.scrapy.description": "Scrapy with Integrated Terminal/Console", + "python.command.importMagic.resolveImport.title": "Resolve Imports", + "python.command.importMagic.insertImport.title": "Insert Import", + "python.command.importMagic.findSymbols.title": "Find Symbols" } From 45e40b82c663bdbdf9a963312894721b23f9a60f Mon Sep 17 00:00:00 2001 From: Vladimir K Urushev Date: Mon, 5 Feb 2018 23:09:11 +0300 Subject: [PATCH 3/3] Fix async functions --- .../importMagicCompletionProvider.ts | 4 ++-- src/client/providers/importMagicProvider.ts | 19 +++++++++---------- src/test/pythonFiles/importMagic/methods.py | 2 -- 3 files changed, 11 insertions(+), 14 deletions(-) delete mode 100644 src/test/pythonFiles/importMagic/methods.py diff --git a/src/client/providers/importMagicCompletionProvider.ts b/src/client/providers/importMagicCompletionProvider.ts index 30f179499a78..8b6ce830d2cb 100644 --- a/src/client/providers/importMagicCompletionProvider.ts +++ b/src/client/providers/importMagicCompletionProvider.ts @@ -19,14 +19,14 @@ export class ImportMagicCompletionItemProvider implements vscode.CompletionItemP public async provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken, context: CompletionContext): Promise { if (!this.enable) { - return Promise.resolve([]); + return []; } const result = await this.getCompletionResult(document, position, token); this.enable = false; if (result === undefined) { - return Promise.resolve([]); + return []; } return result; diff --git a/src/client/providers/importMagicProvider.ts b/src/client/providers/importMagicProvider.ts index dcb47794290f..92055c720a49 100644 --- a/src/client/providers/importMagicProvider.ts +++ b/src/client/providers/importMagicProvider.ts @@ -27,7 +27,7 @@ export class ImportMagicProvider implements Disposable { public async getImportSuggestions(sourceFile: string, unresolvedName: string): Promise { const activeEditor = window.activeTextEditor; if (!activeEditor) { - return Promise.resolve([]); + return []; } const cmd: ICommandSuggestions = { @@ -43,11 +43,10 @@ export class ImportMagicProvider implements Disposable { const candidates = result.candidates; candidates.sort((a, b) => a.score > b.score ? -1 : 1); - const suggestions: ImportPathQuickPickItem[] = candidates.map(item => this.suggestionToQuickPickItem(item)); - return Promise.resolve(suggestions); + return candidates.map(item => this.suggestionToQuickPickItem(item)); } catch (e) { window.showErrorMessage(e.message); - return Promise.resolve([]); + return []; } } @@ -61,14 +60,14 @@ export class ImportMagicProvider implements Disposable { private async resolveImport() { const activeEditor = window.activeTextEditor; if (!activeEditor) { - return Promise.resolve(); + return undefined; } const document = activeEditor.document; if (!activeEditor || document.languageId !== 'python') { window.showErrorMessage('Please open a Python source file to show import suggestions.'); - return Promise.resolve(); + return undefined; } // Get current selected name @@ -76,13 +75,13 @@ export class ImportMagicProvider implements Disposable { const range = document.getWordRangeAtPosition(position); if (!range || range.isEmpty) { window.showErrorMessage('Empty resolve expression'); - return Promise.resolve(); + return undefined; } const unresolvedName : string = document.getText(range); if (!unresolvedName) { window.showErrorMessage('Empty resolve expression'); - return Promise.resolve(); + return undefined; } const tmpFileCreated = document.isDirty; @@ -124,14 +123,14 @@ export class ImportMagicProvider implements Disposable { private async insertImport(module: string, variable?: string) { const activeEditor = window.activeTextEditor; if (!activeEditor) { - return Promise.resolve(); + return undefined; } const document = activeEditor.document; if (!activeEditor || document.languageId !== 'python') { window.showErrorMessage('Please open a Python source file to show import suggestions.'); - return Promise.resolve(); + return undefined; } const tmpFileCreated = document.isDirty; diff --git a/src/test/pythonFiles/importMagic/methods.py b/src/test/pythonFiles/importMagic/methods.py deleted file mode 100644 index 6243af23699f..000000000000 --- a/src/test/pythonFiles/importMagic/methods.py +++ /dev/null @@ -1,2 +0,0 @@ -def method2(): - pass