[WIP] Tests without builtins fixtures

mypy/build.py

@@ -42,6 +42,7 @@
 from mypy.stats import dump_type_stats
 from mypy.types import Type
 from mypy.version import __version__
+from mypy.defaults import allow_fixtures
 
 
 # We need to know the location of this file to load data, but
@@ -143,8 +144,9 @@ def build(sources: List[BuildSource],
         # debug).  This is a test-only feature, so assume our files are laid out
         # as in the source tree.
         root_dir = dirname(dirname(__file__))
-        lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub'))
-    else:
+        if allow_fixtures():
+            lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub'))
+    if not options.use_builtins_fixtures or not allow_fixtures():
         for source in sources:
             if source.path:
                 # Include directory of the program file in the module search path.

mypy/defaults.py

@@ -1,4 +1,10 @@
+import os
 PYTHON2_VERSION = (2, 7)
 PYTHON3_VERSION = (3, 6)
 CACHE_DIR = '.mypy_cache'
 CONFIG_FILE = 'mypy.ini'
+# Set this to False (here in the file, not dynamically!) to disable builtins fixtures in tests
+
+
+def allow_fixtures() -> bool:
+    return not os.environ.get('NO_BUILTINS_FIXTURES', False)

mypy/test/data.py

@@ -12,6 +12,8 @@
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
+from mypy.defaults import allow_fixtures
+
 
 def parse_test_cases(
         path: str,
@@ -63,7 +65,8 @@ def parse_test_cases(
                     assert arg is not None
                     file_entry = (join(base_path, arg), '\n'.join(p[i].data))
                     if p[i].id == 'file':
-                        files.append(file_entry)
+                        if arg != 'builtins.py' or allow_fixtures():
+                            files.append(file_entry)
                     elif p[i].id == 'outfile':
                         output_files.append(file_entry)
                 elif p[i].id in ('builtins', 'builtins_py2'):
@@ -77,7 +80,8 @@ def parse_test_cases(
                         # Python 2
                         fnam = '__builtin__.pyi'
                     with open(mpath) as f:
-                        files.append((join(base_path, fnam), f.read()))
+                        if allow_fixtures():
+                            files.append((join(base_path, fnam), f.read()))
                 elif p[i].id == 'stale':
                     arg = p[i].arg
                     if arg is None:

mypy/test/helpers.py

@@ -2,16 +2,51 @@
 import re
 import os
 
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Iterable
+from itertools import groupby
 
 from mypy import defaults
 from mypy.myunit import AssertionFailure
 from mypy.test.data import DataDrivenTestCase
+from mypy.defaults import allow_fixtures
 
 
 # AssertStringArraysEqual displays special line alignment helper messages if
 # the first different line has at least this many characters,
 MIN_LINE_LENGTH_FOR_ALIGNMENT = 5
+# Maximum lines of actual error output to print (to prevent hitting log size limit)
+MAX_ACTUAL_LINES = 25
+
+
+# should match 'tmp/foo.pyi', 'main.py', etc.
+MODULE_NAME_PATTERN = r'[\w/.]'
+# we really want to be precise because accidentally matching the pattern
+# may cause failed tests to look like they pass
+# should match 'tmp/foo.pyi:15:30', 'tmp/foo.pyi:15', 'main.py:1:2', etc.
+ERROR_PATTERN = re.compile(r'^{}+:(\d+):(\d+:)? error: '.format(MODULE_NAME_PATTERN))
+
+
+class ErrorStr(str):
+    '''
+    Same as str, but compares equal iff it starts with the same file and line no prefix
+    '''
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, str):
+            return False
+        m_self = re.search(ERROR_PATTERN, self)
+        m_other = re.search(ERROR_PATTERN, other)
+        if not m_self or not m_other:  # we can't parse it, just use normal str comparison
+            return super().__eq__(other)
+        else:  # parsed successfully, compare line no (ignore filename and column no)
+            return m_self.group(1) == m_other.group(1)
+
+
+def skip_same_line_errors(iterable: Iterable) -> List:
+    '''
+    Converts all strings into ErrorStr objects, skips consecutive
+    duplicates, and returns the list (preserving original order)
+    '''
+    return [k for k, g in groupby(map(ErrorStr, iterable))]
 
 
 def assert_string_arrays_equal(expected: List[str], actual: List[str],
@@ -23,6 +58,10 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str],
 
     actual = clean_up(actual)
 
+    if not allow_fixtures():
+        actual = skip_same_line_errors(actual)
+        expected = skip_same_line_errors(expected)
+
     if actual != expected:
         num_skip_start = num_skipped_prefix_lines(expected, actual)
         num_skip_end = num_skipped_suffix_lines(expected, actual)
@@ -60,6 +99,9 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str],
             sys.stderr.write('  ...\n')
 
         for j in range(num_skip_start, len(actual) - num_skip_end):
+            if j - num_skip_start > MAX_ACTUAL_LINES:
+                sys.stderr.write('... (additional lines suppressed)')
+                break
             if j >= len(expected) or expected[j] != actual[j]:
                 sys.stderr.write('  {:<45} (diff)'.format(actual[j]))
             else:

mypy/test/testmerge.py

@@ -33,6 +33,7 @@
 TYPEINFO = ' TYPEINFO'
 TYPES = 'TYPES'
 AST = 'AST'
+MODULES_TO_DUMP = ('__main__', 'target')
 
 
 class ASTMergeSuite(DataSuite):
@@ -133,8 +134,8 @@ def dump(self,
     def dump_asts(self, modules: Dict[str, MypyFile]) -> List[str]:
         a = []
         for m in sorted(modules):
-            if m == 'builtins':
-                # We don't support incremental checking of changes to builtins.
+            if m not in MODULES_TO_DUMP:
+                # We don't support incremental checking of changes to builtins or stdlib.
                 continue
             s = modules[m].accept(self.str_conv)
             a.extend(s.splitlines())
@@ -143,8 +144,8 @@ def dump_asts(self, modules: Dict[str, MypyFile]) -> List[str]:
     def dump_symbol_tables(self, modules: Dict[str, MypyFile]) -> List[str]:
         a = []
         for id in sorted(modules):
-            if id == 'builtins':
-                # We don't support incremental checking of changes to builtins.
+            if id not in MODULES_TO_DUMP:
+                # We don't support incremental checking of changes to builtins or stdlib.
                 continue
             a.extend(self.dump_symbol_table(id, modules[id].names))
         return a
@@ -169,7 +170,7 @@ def format_symbol_table_node(self, node: SymbolTableNode) -> str:
     def dump_typeinfos(self, modules: Dict[str, MypyFile]) -> List[str]:
         a = []
         for id in sorted(modules):
-            if id == 'builtins':
+            if id not in MODULES_TO_DUMP:
                 continue
             a.extend(self.dump_typeinfos_recursive(modules[id].names))
         return a
@@ -192,7 +193,7 @@ def dump_types(self, graph: Dict[str, State]) -> List[str]:
         # To make the results repeatable, we try to generate unique and
         # deterministic sort keys.
         for module_id in sorted(graph):
-            if module_id == 'builtins':
+            if module_id not in MODULES_TO_DUMP:
                 continue
             type_map = graph[module_id].type_checker.type_map
             if type_map:

runtests.py

@@ -30,6 +30,7 @@ def get_versions():  # type: () -> typing.List[str]
 from mypy import util
 from mypy.test.config import test_data_prefix
 from mypy.test.testpythoneval import python_eval_files, python_34_eval_files
+from mypy.defaults import allow_fixtures
 
 import itertools
 import os
@@ -208,14 +209,18 @@ def add_imports(driver: Driver) -> None:
             driver.add_python_string('import %s' % mod, 'import %s' % mod)
 
 
-PYTEST_FILES = [os.path.join('mypy', 'test', '{}.py'.format(name)) for name in [
+PYTESTS = [
     'testcheck',
     'testextensions',
     'testdeps',
     'testdiff',
-    'testfinegrained',
     'testmerge',
-]]
+]
+
+if allow_fixtures():
+    PYTESTS += ['testfinegrained']
+
+PYTEST_FILES = [os.path.join('mypy', 'test', '{}.py'.format(name)) for name in PYTESTS]
 
 
 def add_pytest(driver: Driver) -> None:

test-data/unit/check-functions.test

@@ -88,6 +88,7 @@ def r(x) -> None: ...
 r = l # E: Incompatible types in assignment (expression has type Callable[[Any, Any], None], variable has type Callable[[Any], None])
 
 [case testSubtypingFunctionsImplicitNames]
+from typing import Any
 
 def f(a, b): pass
 def g(c: Any, d: Any) -> Any: pass
@@ -1567,6 +1568,7 @@ class A(Generic[t]):
 
 
 [case testRedefineFunction]
+from typing import Any
 def f(x) -> Any: pass
 def g(x, y): pass
 def h(x): pass

test-data/unit/check-isinstance.test

@@ -434,13 +434,13 @@ def foo() -> Union[int, str]: pass
 
 x = foo()
 x = 1
-x = x + 1
+x + 1
 x = foo()
-x = x + 1                # E: Unsupported operand types for + (likely involving Union)
+x + 1                # E: Unsupported operand types for + (likely involving Union)
 if isinstance(x, str):
-   x = x + 1             # E: Unsupported operand types for + ("str" and "int")
+   x + 1             # E: Unsupported operand types for + ("str" and "int")
    x = 1
-   x = x + 1
+   x + 1
 
 [builtins fixtures/isinstancelist.pyi]
 
@@ -463,11 +463,11 @@ from typing import Union
 x = None # type: Union[int, str]
 
 if isinstance(x, str):
-    x = x + 1   # E: Unsupported operand types for + ("str" and "int")
-    x = x + 'a'
+    x + 1   # E: Unsupported operand types for + ("str" and "int")
+    x + 'a'
 else:
-    x = x + 'a' # E: Unsupported operand types for + ("int" and "str")
-    x = x + 1
+    x + 'a' # E: Unsupported operand types for + ("int" and "str")
+    x + 1
 [builtins fixtures/isinstancelist.pyi]
 
 [case testIsInstanceIndexing]

test-data/unit/check-modules.test

@@ -670,45 +670,45 @@ def f(x: str) -> None: pass
 [case testTypeCheckNamedModule]
 # cmd: mypy -m m.a
 [file m/__init__.py]
-None + 1
+object() + 1
 [file m/a.py]
 [out]
-tmp/m/__init__.py:1: error: Unsupported left operand type for + (None)
+tmp/m/__init__.py:1: error: Unsupported left operand type for + ("object")
 
 [case testTypeCheckNamedModule2]
 # cmd: mypy -m m.a
 [file m/__init__.py]
 [file m/a.py]
-None + 1
+object() + 1
 [out]
-tmp/m/a.py:1: error: Unsupported left operand type for + (None)
+tmp/m/a.py:1: error: Unsupported left operand type for + ("object")
 
 [case testTypeCheckNamedModule3]
 # cmd: mypy -m m
 [file m/__init__.py]
-None + 1
+object() + 1
 [file m/a.py]
 [out]
-tmp/m/__init__.py:1: error: Unsupported left operand type for + (None)
+tmp/m/__init__.py:1: error: Unsupported left operand type for + ("object")
 
 [case testTypeCheckNamedModule4]
 # cmd: mypy -m m
 [file m/__init__.py]
 [file m/a.py]
-None + 1  # Not analyzed.
+object() + 1  # Not analyzed.
 [out]
 
 [case testTypeCheckNamedModule5]
 # cmd: mypy -m m
-None + ''  # Not analyzed.
+object() + 1  # Not analyzed.
 [file m.py]
-None + 1
+object() + 1
 [out]
-tmp/m.py:1: error: Unsupported left operand type for + (None)
+tmp/m.py:1: error: Unsupported left operand type for + ("object")
 
 [case testTypeCheckNamedModuleWithImportCycle]
 # cmd: mypy -m m.a
-None + 1  # Does not generate error, as this file won't be analyzed.
+object() + 1  # Does not generate error, as this file won't be analyzed.
 [file m/__init__.py]
 import m.a
 [file m/a.py]

test-data/unit/check-newsyntax.test

@@ -35,7 +35,9 @@ d['ab'] = 'ab'  # E: Invalid index type "str" for Dict[int, str]; expected type
 
 [case testNewSyntaxWithRevealType]
 # flags: --python-version 3.6
-from typing import Dict
+from typing import Dict, TypeVar
+
+T = TypeVar('T')
 
 def tst_local(dct: Dict[int, T]) -> Dict[T, int]:
     ret: Dict[T, int] = {}

test-data/unit/check-unreachable-code.test

@@ -84,9 +84,9 @@ main:6: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports"
 import typing
 MYPY = 0
 if MYPY:
-    None + 1 # E: Unsupported left operand type for + (None)
+    object() + 1 # E: Unsupported left operand type for + ("object")
 else:
-    None + ''
+    object() + 1
 [builtins fixtures/bool.pyi]
 
 [case testTypeCheckingConditional]