From 0f073eed0a3a61b41e10fdc2a04c440fe06c0457 Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Sat, 18 Mar 2017 13:53:31 -0700
Subject: [PATCH 1/6] test_stub now returns a list of errors

Error defined as NamedTuple(name, error_type, message).
stubtest no longer errors when there is no stubfile
---
 scripts/stubtest.py | 47 +++++++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 1c9efadc11d31..d890bfc29f1f9 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -4,12 +4,11 @@
 at runtime.
 """
 
-import importlib
 import json
 import subprocess
 import sys
 from typing import Dict, Any
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 
 from mypy import build
 from mypy.build import default_data_dir, default_lib_path, find_modules_recursive
@@ -28,19 +27,13 @@
 }
 
 
-class Errors:
-    def __init__(self, id):
-        self.id = id
-        self.num_errors = 0
-
-    def fail(self, msg):
-        print('{}: {}'.format(self.id, msg))
-        self.num_errors += 1
+Error = namedtuple('Error', ('name', 'error_type', 'message'))
 
 
 def test_stub(id: str) -> None:
-    result = build_stubs(id)
-    verify_stubs(result.files, prefix=id)
+    result, errors = build_stubs(id)
+    errors += list(verify_stubs(result.files, prefix=id))
+    return errors
 
 
 def verify_stubs(files: Dict[str, MypyFile], prefix: str) -> None:
@@ -52,12 +45,11 @@ def verify_stubs(files: Dict[str, MypyFile], prefix: str) -> None:
             # There's some issue with processing this module; skip for now
             continue
         dumped = dump_module(id)
-        verify_stub(id, node.names, dumped)
+        yield from verify_stub(id, node.names, dumped)
 
 
 # symbols is typeshed, dumped is runtime
-def verify_stub(id, symbols, dumped):
-    errors = Errors(id)
+def verify_stub(name, symbols, dumped):
     symbols = defaultdict(lambda: None, symbols)
     dumped = defaultdict(lambda: None, dumped)
 
@@ -70,22 +62,25 @@ def verify_stub(id, symbols, dumped):
 
     for name, (typeshed, runtime) in all_symbols.items():
         if runtime is None:
-            errors.fail('"{}" defined in stub but not at runtime'.format(name))
+            yield Error(name, 'not_in_runtime',
+                '"{}" defined in stub but not at runtime'.format(name))
         elif typeshed is None:
-            errors.fail('"{}" defined at runtime but not in stub'.format(name))
+            yield Error(name, 'not_in_stub',
+                '"{}" defined at runtime but not in stub'.format(name))
         else:
-            verify_node(name, typeshed, runtime, errors)
+            verify_node(name, typeshed, runtime)
 
 
-def verify_node(name, node, dump, errors):
+def verify_node(name, node, dump):
     if isinstance(node.node, TypeInfo):
         if not isinstance(dump, dict) or dump['type'] != 'class':
-            errors.fail('"{}" is a class in stub but not at runtime'.format(name))
+            yield Error(name, 'class_not_in_stub',
+                '"{}" is a class in stub but not at runtime'.format(name))
             return
         all_attrs = {x[0] for x in dump['attributes']}
         for attr, attr_node in node.node.names.items():
             if isinstance(attr_node.node, FuncItem) and attr not in all_attrs:
-                errors.fail(
+                yield Error(name, 'method_not_in_stub',
                     ('"{}.{}" defined as a method in stub but not defined '
                      'at runtime in class object').format(
                         name, attr))
@@ -103,6 +98,7 @@ def dump_module(id: str) -> Dict[str, Any]:
 
 
 def build_stubs(id):
+    errors = []
     data_dir = default_data_dir(None)
     options = Options()
     options.python_version = (3, 6)
@@ -111,8 +107,8 @@ def build_stubs(id):
                                 custom_typeshed_dir=None)
     sources = find_modules_recursive(id, lib_path)
     if not sources:
-        sys.exit('Error: Cannot find module {}'.format(repr(id)))
-    msg = []
+        errors.append(Error(repr(id), 'no_typeshed',
+            'could not find typeshed {}'.format(repr(id))))
     try:
         res = build.build(sources=sources,
                           options=options)
@@ -123,8 +119,9 @@ def build_stubs(id):
         for m in msg:
             print(m)
         sys.exit(1)
-    return res
+    return res, errors
 
 
 if __name__ == '__main__':
-    test_stub(sys.argv[1])
+    for error in test_stub(sys.argv[1]):
+        print(error.message)

From c4aadc6312a1110e908aba8d13f1331b06d70628 Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Sat, 18 Mar 2017 14:34:01 -0700
Subject: [PATCH 2/6] Make stubtest runnable from any directory

Instead of calling the dumpmodule script, do a relative import and
call the function directly.
---
 scripts/stubtest.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index d890bfc29f1f9..1fbe4739ffd36 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -4,8 +4,7 @@
 at runtime.
 """
 
-import json
-import subprocess
+import importlib
 import sys
 from typing import Dict, Any
 from collections import defaultdict, namedtuple
@@ -16,6 +15,8 @@
 from mypy.nodes import MypyFile, TypeInfo, FuncItem
 from mypy.options import Options
 
+import dumpmodule
+
 
 skipped = {
     '_importlib_modulespec',
@@ -88,13 +89,8 @@ def verify_node(name, node, dump):
 
 
 def dump_module(id: str) -> Dict[str, Any]:
-    try:
-        o = subprocess.check_output(
-            ['python', 'scripts/dumpmodule.py', id])
-    except subprocess.CalledProcessError:
-        print('Failure to dump module contents of "{}"'.format(id))
-        sys.exit(1)
-    return json.loads(o.decode('ascii'))
+    m = importlib.import_module(id)
+    return dumpmodule.module_to_json(m)
 
 
 def build_stubs(id):

From 52908bebdfd31eb10fb0fd069dddff71c63c7183 Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Sun, 19 Mar 2017 10:39:41 -0700
Subject: [PATCH 3/6] record line number in Error object, blacklist
 unittest.mock

unittest.mock problem related to https://bugs.python.org/issue25532
---
 scripts/dumpmodule.py | 36 ++++++++++-------
 scripts/stubtest.py   | 94 ++++++++++++++++++++++++++++++-------------
 2 files changed, 88 insertions(+), 42 deletions(-)

diff --git a/scripts/dumpmodule.py b/scripts/dumpmodule.py
index fb705f05fe631..0a2ca5dd9f6c0 100644
--- a/scripts/dumpmodule.py
+++ b/scripts/dumpmodule.py
@@ -32,16 +32,26 @@ def module_to_json(m):
     result = {}
     for name, value in m.__dict__.items():
         # Filter out some useless attributes.
+
         if name in ('__file__',
                     '__doc__',
                     '__name__',
                     '__builtins__',
                     '__package__'):
             continue
+
         if name == '__all__':
-            result[name] = sorted(value)
+            result[name] = {'values': sorted(value)}
         else:
             result[name] = dump_value(value)
+
+        try:
+            _, line = inspect.getsourcelines(getattr(m, name))
+        except (TypeError, OSError):
+            line = None
+
+        result[name]['line'] = line
+
     return result
 
 
@@ -53,25 +63,25 @@ def dump_value(value, depth=0):
     if inspect.isfunction(value):
         return dump_function(value)
     if callable(value):
-        return 'callable'  # TODO more information
+        return {'type': 'callable'}  # TODO more information
     if isinstance(value, types.ModuleType):
-        return 'module'  # TODO module name
+        return {'type': 'module'}  # TODO module name
     if inspect.isdatadescriptor(value):
-        return 'datadescriptor'
+        return {'type': 'datadescriptor'}
 
     if inspect.ismemberdescriptor(value):
-        return 'memberdescriptor'
+        return {'type': 'memberdescriptor'}
     return dump_simple(value)
 
 
 def dump_simple(value):
     if type(value) in (int, bool, float, str, bytes, Text, long, list, set, dict, tuple):
-        return type(value).__name__
+        return {'type': type(value).__name__}
     if value is None:
-        return 'None'
+        return {'type': 'None'}
     if value is inspect.Parameter.empty:
-        return None
-    return 'unknown'
+        return {'type': None}  # 'None' and None: Ruh-Roh
+    return {'type': 'unknown'}
 
 
 def dump_class(value, depth):
@@ -133,14 +143,12 @@ def dump_function(value):
         sig = inspect.signature(value)
     except ValueError:
         # The signature call sometimes fails for some reason.
-        return 'invalid_signature'
+        return {'type': 'invalid_signature'}
     params = list(sig.parameters.items())
     return {
         'type': 'function',
-        'args': [(name,
-             param_kind(p),
-             dump_simple(p.default))
-            for name, p in params],
+        'args': [(name, param_kind(p), dump_simple(p.default))
+                 for name, p in params],
     }
 
 
diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 1fbe4739ffd36..9b7cdecbf47a6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -6,7 +6,7 @@
 
 import importlib
 import sys
-from typing import Dict, Any
+from typing import Dict, Any, List
 from collections import defaultdict, namedtuple
 
 from mypy import build
@@ -17,7 +17,7 @@
 
 import dumpmodule
 
-
+# TODO: why are these skipped
 skipped = {
     '_importlib_modulespec',
     '_subprocess',
@@ -25,13 +25,27 @@
     'distutils.command.bdist_packager',
     'msvcrt',
     'wsgiref.types',
+    'unittest.mock',  # mock.call infinite loops on inspect.getsourcelines
+                      # https://bugs.python.org/issue25532
 }
 
+not_in_runtime_msg = ('"{name}" defined at line {line} in stub '
+                      'but is not defined at runtime').format
+
+not_in_stub_msg = ('"{obj_type} {name}" defined at line {line} at runtime '
+                   'but is not defined in stub').format
+
+no_typeshed_msg = 'could not find typeshed {}'.format
+
+class_not_in_stub_msg = '"{}" is a class in stub but not at runtime'.format
 
-Error = namedtuple('Error', ('name', 'error_type', 'message'))
+method_not_in_stub_msg = ('"{}.{}" defined as a method in stub but not defined '
+                          'at runtime in class object').format
 
+Error = namedtuple('Error', ('module', 'name', 'error_type', 'line', 'message'))
 
-def test_stub(id: str) -> None:
+
+def test_stub(id: str) -> List[Error]:
     result, errors = build_stubs(id)
     errors += list(verify_stubs(result.files, prefix=id))
     return errors
@@ -50,7 +64,14 @@ def verify_stubs(files: Dict[str, MypyFile], prefix: str) -> None:
 
 
 # symbols is typeshed, dumped is runtime
-def verify_stub(name, symbols, dumped):
+def verify_stub(module: str, symbols: dict, dumped: dict):
+    """Generate mismatches between typshed and runtime types
+
+    It accepts the following parameters
+    module: name of module to check
+    symbols: dictionary of typeshed types
+    dumped: dictionary of runtime types
+    """
     symbols = defaultdict(lambda: None, symbols)
     dumped = defaultdict(lambda: None, dumped)
 
@@ -63,28 +84,31 @@ def verify_stub(name, symbols, dumped):
 
     for name, (typeshed, runtime) in all_symbols.items():
         if runtime is None:
-            yield Error(name, 'not_in_runtime',
-                '"{}" defined in stub but not at runtime'.format(name))
+            line = getattr(typeshed.node, 'line', None)
+            msg = not_in_runtime_msg(name=name, line=line)
+            yield Error(module, name, 'not_in_runtime', line, msg)
         elif typeshed is None:
-            yield Error(name, 'not_in_stub',
-                '"{}" defined at runtime but not in stub'.format(name))
+            msg = not_in_stub_msg(
+                obj_type=runtime['type'],
+                name=name,
+                line=runtime['line'])
+            yield Error(module, name, 'not_in_stub', runtime['line'], msg)
         else:
-            verify_node(name, typeshed, runtime)
+            for obj, error_type, line, msg in verify_node(name, typeshed, runtime):
+                yield Error(module, obj, error_type, line, msg)
 
 
 def verify_node(name, node, dump):
     if isinstance(node.node, TypeInfo):
         if not isinstance(dump, dict) or dump['type'] != 'class':
-            yield Error(name, 'class_not_in_stub',
-                '"{}" is a class in stub but not at runtime'.format(name))
+            yield (name, 'class_not_in_stub', node.node.line,
+                   class_not_in_stub_msg(name))
             return
         all_attrs = {x[0] for x in dump['attributes']}
         for attr, attr_node in node.node.names.items():
             if isinstance(attr_node.node, FuncItem) and attr not in all_attrs:
-                yield Error(name, 'method_not_in_stub',
-                    ('"{}.{}" defined as a method in stub but not defined '
-                     'at runtime in class object').format(
-                        name, attr))
+                yield (name, 'method_not_in_stub', node.node.line,
+                       method_not_in_stub_msg(name, attr))
     # TODO other kinds of nodes
 
 
@@ -93,7 +117,7 @@ def dump_module(id: str) -> Dict[str, Any]:
     return dumpmodule.module_to_json(m)
 
 
-def build_stubs(id):
+def build_stubs(mod):
     errors = []
     data_dir = default_data_dir(None)
     options = Options()
@@ -101,23 +125,37 @@ def build_stubs(id):
     lib_path = default_lib_path(data_dir,
                                 options.python_version,
                                 custom_typeshed_dir=None)
-    sources = find_modules_recursive(id, lib_path)
+    sources = find_modules_recursive(mod, lib_path)
     if not sources:
-        errors.append(Error(repr(id), 'no_typeshed',
-            'could not find typeshed {}'.format(repr(id))))
+        errors += [Error(repr(mod), repr(mod), 'no_typeshed', None,
+                        no_typeshed_msg(repr(mod)))]
     try:
         res = build.build(sources=sources,
                           options=options)
-        msg = res.errors
-    except CompileError as e:
-        msg = e.messages
-    if msg:
-        for m in msg:
-            print(m)
+        messages = res.errors
+    except CompileError as err:
+        messages = err.messages
+
+    if messages:
+        for msg in messages:
+            print(msg)
         sys.exit(1)
     return res, errors
 
 
 if __name__ == '__main__':
-    for error in test_stub(sys.argv[1]):
-        print(error.message)
+    if len(sys.argv) == 1:
+        print('must provide at least one module to test, or --all_stdlib')
+        sys.exit(1)
+    elif sys.argv[1] == '--all_stdlib':
+        version = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
+        from stdlib_list import stdlib_list
+        modules = set(stdlib_list(version))
+        modules.remove('unittest')
+        modules.remove('unittest.mock')
+    else:
+        modules = sys.argv[1:]
+
+    for module in modules:
+        for error in test_stub(module):
+            print(error.module, ':', error.message)

From 73eb38bb1fabc5639eddb1a7ac5e23a845d1f048 Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Sun, 19 Mar 2017 15:48:13 -0700
Subject: [PATCH 4/6] record types in Error object, make verify_node recursive

also changed dump['attrs'] to a dict
---
 scripts/dumpmodule.py |   7 +--
 scripts/stubtest.py   | 108 ++++++++++++++++++++++++++----------------
 2 files changed, 70 insertions(+), 45 deletions(-)

diff --git a/scripts/dumpmodule.py b/scripts/dumpmodule.py
index 0a2ca5dd9f6c0..d4e1b46d6a179 100644
--- a/scripts/dumpmodule.py
+++ b/scripts/dumpmodule.py
@@ -102,8 +102,9 @@ def dump_class(value, depth):
 ]
 
 
+# Change to return a dict
 def dump_attrs(d, depth):
-    result = []
+    result = {}
     seen = set()
     try:
         mro = d.mro()
@@ -113,11 +114,11 @@ def dump_attrs(d, depth):
         v = vars(base)
         for name, value in v.items():
             if name not in seen:
-                result.append([name, dump_value(value, depth + 1)])
+                result[name] = dump_value(value, depth + 1)
                 seen.add(name)
     for m in special_methods:
         if hasattr(d, m) and m not in seen:
-            result.append([m, dump_value(getattr(d, m), depth + 1)])
+            result[m] = dump_value(getattr(d, m), depth + 1)
     return result
 
 
diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 9b7cdecbf47a6..bb08b10f1a286 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -12,7 +12,7 @@
 from mypy import build
 from mypy.build import default_data_dir, default_lib_path, find_modules_recursive
 from mypy.errors import CompileError
-from mypy.nodes import MypyFile, TypeInfo, FuncItem
+from mypy.nodes import MypyFile, TypeInfo, FuncItem, Var, OverloadedFuncDef, TypeVarExpr, Decorator
 from mypy.options import Options
 
 import dumpmodule
@@ -29,20 +29,23 @@
                       # https://bugs.python.org/issue25532
 }
 
-not_in_runtime_msg = ('"{name}" defined at line {line} in stub '
-                      'but is not defined at runtime').format
-
-not_in_stub_msg = ('"{obj_type} {name}" defined at line {line} at runtime '
-                   'but is not defined in stub').format
-
-no_typeshed_msg = 'could not find typeshed {}'.format
-
-class_not_in_stub_msg = '"{}" is a class in stub but not at runtime'.format
-
-method_not_in_stub_msg = ('"{}.{}" defined as a method in stub but not defined '
-                          'at runtime in class object').format
+messages = {
+    'not_in_runtime': ('<{error.stub_type}> "{error.name}" defined at line '
+                       ' {error.line} in stub but is not defined at runtime'),
+    'not_in_stub': ('<{error.module_type}> "{error.name}" defined at line'
+                    ' {error.line} at runtime but is not defined in stub'),
+    'no_typeshed': 'could not find typeshed {error.name}',
+    'inconsistent': ('"{error.name}" is <{error.stub_type}> in stub but'
+                     ' <{error.module_type}> at runtime'),
+}
 
-Error = namedtuple('Error', ('module', 'name', 'error_type', 'line', 'message'))
+Error = namedtuple('Error', (
+    'module',
+    'name',
+    'error_type',
+    'line',
+    'stub_type',
+    'module_type'))
 
 
 def test_stub(id: str) -> List[Error]:
@@ -85,31 +88,49 @@ def verify_stub(module: str, symbols: dict, dumped: dict):
     for name, (typeshed, runtime) in all_symbols.items():
         if runtime is None:
             line = getattr(typeshed.node, 'line', None)
-            msg = not_in_runtime_msg(name=name, line=line)
-            yield Error(module, name, 'not_in_runtime', line, msg)
+            shed_type = type(typeshed.node).__name__
+            yield Error(module, name, 'not_in_runtime', line, shed_type, None)
         elif typeshed is None:
-            msg = not_in_stub_msg(
-                obj_type=runtime['type'],
-                name=name,
-                line=runtime['line'])
-            yield Error(module, name, 'not_in_stub', runtime['line'], msg)
+            yield Error(module, name, 'not_in_stub', runtime['line'], None, runtime['type'])
         else:
-            for obj, error_type, line, msg in verify_node(name, typeshed, runtime):
-                yield Error(module, obj, error_type, line, msg)
+            for err in verify_node(name, typeshed, runtime):
+                yield Error(module, *err)
 
 
 def verify_node(name, node, dump):
+    module_type = dump.get('type', None)
+    shed_type = type(node.node).__name__
     if isinstance(node.node, TypeInfo):
-        if not isinstance(dump, dict) or dump['type'] != 'class':
-            yield (name, 'class_not_in_stub', node.node.line,
-                   class_not_in_stub_msg(name))
-            return
-        all_attrs = {x[0] for x in dump['attributes']}
-        for attr, attr_node in node.node.names.items():
-            if isinstance(attr_node.node, FuncItem) and attr not in all_attrs:
-                yield (name, 'method_not_in_stub', node.node.line,
-                       method_not_in_stub_msg(name, attr))
-    # TODO other kinds of nodes
+        if dump['type'] != 'class':
+            yield name, 'inconsistent', node.node.line, shed_type, module_type
+        else:
+            for attr, attr_node in node.node.names.items():
+                subname = '{}.{}'.format(name, attr)
+                subdump = dump['attributes'].get(attr, {})
+                for err in verify_node(subname, attr_node, subdump):
+                    yield err
+
+    elif isinstance(node.node, FuncItem):
+        if 'type' not in dump or dump['type'] not in ('function', 'callable'):
+            yield name, 'inconsistent', node.node.line, shed_type, module_type
+        # TODO check arguments and return value
+    elif isinstance(node.node, Var):
+        pass
+        # Need to check if types are inconsistent.
+        #if 'type' not in dump or dump['type'] != node.node.type:
+        #    import ipdb; ipdb.set_trace()
+        #    yield name, 'inconsistent', node.node.line, shed_type, module_type
+    elif isinstance(node.node, MypyFile):
+        pass # TODO: what checking can we do here?
+    elif isinstance(node.node, OverloadedFuncDef):
+        # Should check types of the union of the overloaded types.
+        pass
+    elif isinstance(node.node, TypeVarExpr):
+        pass # TODO: what even is this?
+    elif isinstance(node.node, Decorator):
+        pass # What can we check here?
+    else:
+        raise TypeError('unkonwn node type {}'.format(node.node))
 
 
 def dump_module(id: str) -> Dict[str, Any]:
@@ -127,8 +148,7 @@ def build_stubs(mod):
                                 custom_typeshed_dir=None)
     sources = find_modules_recursive(mod, lib_path)
     if not sources:
-        errors += [Error(repr(mod), repr(mod), 'no_typeshed', None,
-                        no_typeshed_msg(repr(mod)))]
+        errors += [Error(repr(mod), repr(mod), 'no_typeshed', None, None, None)]
     try:
         res = build.build(sources=sources,
                           options=options)
@@ -143,19 +163,23 @@ def build_stubs(mod):
     return res, errors
 
 
-if __name__ == '__main__':
-    if len(sys.argv) == 1:
+def main(args):
+    if len(args) == 1:
         print('must provide at least one module to test, or --all_stdlib')
         sys.exit(1)
-    elif sys.argv[1] == '--all_stdlib':
+    elif args[1] == '--all_stdlib':
         version = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
         from stdlib_list import stdlib_list
         modules = set(stdlib_list(version))
-        modules.remove('unittest')
-        modules.remove('unittest.mock')
     else:
-        modules = sys.argv[1:]
+        modules = args[1:]
 
     for module in modules:
         for error in test_stub(module):
-            print(error.module, ':', error.message)
+            yield error
+
+
+if __name__ == '__main__':
+
+    for err in main(sys.argv):
+        print(messages[err.error_type].format(error=err))

From 97fb64c98fbada1157ee1c4bbf843b17a99d0951 Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Sun, 19 Mar 2017 21:40:14 -0700
Subject: [PATCH 5/6] Remove '--all_stdlib' flag from stubtest.

doing this automatically depends on module stdlib_list, which is not
suitable for master branch.
---
 scripts/stubtest.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index bb08b10f1a286..48504c829669f 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -165,12 +165,8 @@ def build_stubs(mod):
 
 def main(args):
     if len(args) == 1:
-        print('must provide at least one module to test, or --all_stdlib')
+        print('must provide at least one module to test')
         sys.exit(1)
-    elif args[1] == '--all_stdlib':
-        version = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
-        from stdlib_list import stdlib_list
-        modules = set(stdlib_list(version))
     else:
         modules = args[1:]
 

From ae75b4b51659958b67ddd1db1a78fd29f50285bb Mon Sep 17 00:00:00 2001
From: Jared Garst <cultofjared@gmail.com>
Date: Tue, 21 Mar 2017 20:48:14 -0700
Subject: [PATCH 6/6] Refactored verify_node to single dispatch recursion

---
 scripts/dumpmodule.py |   3 +-
 scripts/stubtest.py   | 218 +++++++++++++++++++++++-------------------
 2 files changed, 123 insertions(+), 98 deletions(-)

diff --git a/scripts/dumpmodule.py b/scripts/dumpmodule.py
index d4e1b46d6a179..5d31241f4a0df 100644
--- a/scripts/dumpmodule.py
+++ b/scripts/dumpmodule.py
@@ -12,6 +12,7 @@
 import sys
 import types
 from typing import Text
+from collections import defaultdict
 
 
 if sys.version_info >= (3, 0):
@@ -41,7 +42,7 @@ def module_to_json(m):
             continue
 
         if name == '__all__':
-            result[name] = {'values': sorted(value)}
+            result[name] = {'type': 'list', 'values': sorted(value)}
         else:
             result[name] = dump_value(value)
 
diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 48504c829669f..40c804f47efcc 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -12,31 +12,40 @@
 from mypy import build
 from mypy.build import default_data_dir, default_lib_path, find_modules_recursive
 from mypy.errors import CompileError
-from mypy.nodes import MypyFile, TypeInfo, FuncItem, Var, OverloadedFuncDef, TypeVarExpr, Decorator
+from mypy import nodes
 from mypy.options import Options
 
 import dumpmodule
 
-# TODO: why are these skipped
-skipped = {
+if sys.version_info < (3, 4):
+    from singledispatch import singledispatch
+else:
+    from functools import singledispatch
+
+# TODO: email.contentmanager has a symbol table with a None node.
+#       This seems like it should not be.
+
+skip = {
     '_importlib_modulespec',
     '_subprocess',
     'distutils.command.bdist_msi',
     'distutils.command.bdist_packager',
     'msvcrt',
     'wsgiref.types',
+    'mypy_extensions',
     'unittest.mock',  # mock.call infinite loops on inspect.getsourcelines
                       # https://bugs.python.org/issue25532
+                      # TODO: can we filter only call?
 }
 
 messages = {
-    'not_in_runtime': ('<{error.stub_type}> "{error.name}" defined at line '
+    'not_in_runtime': ('{error.stub_type} "{error.name}" defined at line '
                        ' {error.line} in stub but is not defined at runtime'),
-    'not_in_stub': ('<{error.module_type}> "{error.name}" defined at line'
+    'not_in_stub': ('{error.module_type} "{error.name}" defined at line'
                     ' {error.line} at runtime but is not defined in stub'),
-    'no_typeshed': 'could not find typeshed {error.name}',
-    'inconsistent': ('"{error.name}" is <{error.stub_type}> in stub but'
-                     ' <{error.module_type}> at runtime'),
+    'no_stubs': 'could not find typeshed {error.name}',
+    'inconsistent': ('"{error.name}" is {error.stub_type} in stub but'
+                     ' {error.module_type} at runtime'),
 }
 
 Error = namedtuple('Error', (
@@ -48,98 +57,115 @@
     'module_type'))
 
 
-def test_stub(id: str) -> List[Error]:
-    result, errors = build_stubs(id)
-    errors += list(verify_stubs(result.files, prefix=id))
-    return errors
-
-
-def verify_stubs(files: Dict[str, MypyFile], prefix: str) -> None:
-    for id, node in files.items():
-        if not (id == prefix or id.startswith(prefix + '.')):
-            # Not one of the target modules
-            continue
-        if id in skipped:
-            # There's some issue with processing this module; skip for now
-            continue
-        dumped = dump_module(id)
-        yield from verify_stub(id, node.names, dumped)
-
-
-# symbols is typeshed, dumped is runtime
-def verify_stub(module: str, symbols: dict, dumped: dict):
-    """Generate mismatches between typshed and runtime types
-
-    It accepts the following parameters
-    module: name of module to check
-    symbols: dictionary of typeshed types
-    dumped: dictionary of runtime types
-    """
-    symbols = defaultdict(lambda: None, symbols)
-    dumped = defaultdict(lambda: None, dumped)
-
-    all_symbols = {
-        name: (symbols[name], dumped[name])
-        for name in (set(symbols) | set(dumped))
-        if not name.startswith('_')  # private attributes
-        and (symbols[name] is None or symbols[name].module_public)
+def test_stub(name: str):
+    stubs = {
+        mod: stub for mod, stub in build_stubs(name).items()
+        if (mod == name or mod.startswith(name + '.')) and mod not in skip
     }
 
-    for name, (typeshed, runtime) in all_symbols.items():
-        if runtime is None:
-            line = getattr(typeshed.node, 'line', None)
-            shed_type = type(typeshed.node).__name__
-            yield Error(module, name, 'not_in_runtime', line, shed_type, None)
-        elif typeshed is None:
-            yield Error(module, name, 'not_in_stub', runtime['line'], None, runtime['type'])
-        else:
-            for err in verify_node(name, typeshed, runtime):
-                yield Error(module, *err)
-
-
-def verify_node(name, node, dump):
-    module_type = dump.get('type', None)
-    shed_type = type(node.node).__name__
-    if isinstance(node.node, TypeInfo):
-        if dump['type'] != 'class':
-            yield name, 'inconsistent', node.node.line, shed_type, module_type
-        else:
-            for attr, attr_node in node.node.names.items():
-                subname = '{}.{}'.format(name, attr)
-                subdump = dump['attributes'].get(attr, {})
-                for err in verify_node(subname, attr_node, subdump):
-                    yield err
-
-    elif isinstance(node.node, FuncItem):
-        if 'type' not in dump or dump['type'] not in ('function', 'callable'):
-            yield name, 'inconsistent', node.node.line, shed_type, module_type
-        # TODO check arguments and return value
-    elif isinstance(node.node, Var):
-        pass
-        # Need to check if types are inconsistent.
-        #if 'type' not in dump or dump['type'] != node.node.type:
-        #    import ipdb; ipdb.set_trace()
-        #    yield name, 'inconsistent', node.node.line, shed_type, module_type
-    elif isinstance(node.node, MypyFile):
-        pass # TODO: what checking can we do here?
-    elif isinstance(node.node, OverloadedFuncDef):
-        # Should check types of the union of the overloaded types.
-        pass
-    elif isinstance(node.node, TypeVarExpr):
-        pass # TODO: what even is this?
-    elif isinstance(node.node, Decorator):
-        pass # What can we check here?
+    for mod, stub in stubs.items():
+        instance = dump_module(mod)
+
+        for identifiers, *error in verify(stub, instance):
+            yield Error(mod, '.'.join(identifiers), *error)
+
+
+@singledispatch
+def verify(node, module_node):
+    raise TypeError('unknown mypy node ' + str(node))
+
+
+
+@verify.register(nodes.MypyFile)
+def verify_mypyfile(stub, instance):
+    if instance is None:
+        yield [], 'not_in_runtime', stub.line, type(stub), None
+    elif instance['type'] != 'file':
+        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+    else:
+        stub_children = defaultdict(lambda: None, stub.names)
+        instance_children = defaultdict(lambda: None, instance['names'])
+
+        # TODO: I would rather not filter public children here.
+        #       For example, what if the checkersurfaces an inconsistency
+        #       in the typing of a private child
+        public_nodes = {
+            name: (stub_children[name], instance_children[name])
+            for name in set(stub_children) | set(instance_children)
+            if not name.startswith('_')
+            and (stub_children[name] is None or stub_children[name].module_public)
+        }
+
+        for node, (stub_child, instance_child) in public_nodes.items():
+            stub_child = getattr(stub_child, 'node', None)
+            for identifiers, *error in verify(stub_child, instance_child):
+                yield ([node] + identifiers, *error)
+
+@verify.register(nodes.TypeInfo)
+def verify_typeinfo(stub, instance):
+    if not instance:
+        yield [], 'not_in_runtime', stub.line, type(stub), None
+    elif instance['type'] != 'class':
+        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+    else:
+        for attr, attr_node in stub.names.items():
+            subdump = instance['attributes'].get(attr, None)
+            for identifiers, *error in verify(attr_node.node, subdump):
+                yield ([attr] + identifiers, *error)
+
+
+@verify.register(nodes.FuncItem)
+def verify_funcitem(stub, instance):
+    if not instance:
+        yield [], 'not_in_runtime', stub.line, type(stub), None
+    elif 'type' not in instance or instance['type'] not in ('function', 'callable'):
+        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+    # TODO check arguments and return value
+
+
+@verify.register(type(None))
+def verify_none(stub, instance):
+    if instance is None:
+        yield [], 'not_in_stub', None, None, None
     else:
-        raise TypeError('unkonwn node type {}'.format(node.node))
+        yield [], 'not_in_stub', instance['line'], None, instance['type']
+
+
+@verify.register(nodes.Var)
+def verify_var(node, module_node):
+    if False:
+        yield None
+    # Need to check if types are inconsistent.
+    #if 'type' not in dump or dump['type'] != node.node.type:
+    #    import ipdb; ipdb.set_trace()
+    #    yield name, 'inconsistent', node.node.line, shed_type, module_type
+
+
+@verify.register(nodes.OverloadedFuncDef)
+def verify_overloadedfuncdef(node, module_node):
+    # Should check types of the union of the overloaded types.
+    if False:
+        yield None
+
+
+@verify.register(nodes.TypeVarExpr)
+def verify_typevarexpr(node, module_node):
+    if False:
+        yield None
+
+
+@verify.register(nodes.Decorator)
+def verify_decorator(node, module_noode):
+    if False:
+        yield None
 
 
-def dump_module(id: str) -> Dict[str, Any]:
-    m = importlib.import_module(id)
-    return dumpmodule.module_to_json(m)
+def dump_module(name: str) -> Dict[str, Any]:
+    mod = importlib.import_module(name)
+    return {'type': 'file', 'names': dumpmodule.module_to_json(mod)}
 
 
 def build_stubs(mod):
-    errors = []
     data_dir = default_data_dir(None)
     options = Options()
     options.python_version = (3, 6)
@@ -147,20 +173,18 @@ def build_stubs(mod):
                                 options.python_version,
                                 custom_typeshed_dir=None)
     sources = find_modules_recursive(mod, lib_path)
-    if not sources:
-        errors += [Error(repr(mod), repr(mod), 'no_typeshed', None, None, None)]
     try:
         res = build.build(sources=sources,
                           options=options)
         messages = res.errors
-    except CompileError as err:
-        messages = err.messages
+    except CompileError as error:
+        messages = error.messages
 
     if messages:
         for msg in messages:
             print(msg)
         sys.exit(1)
-    return res, errors
+    return res.files
 
 
 def main(args):