From f389250299fc7f6840951e5b318d927008a91b74 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 6 Oct 2017 12:21:14 -0700
Subject: [PATCH 01/42] Add dmypy: experimental daemon mode.

---
 mypy/dmypy.py | 394 ++++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/dmypy |  20 +++
 2 files changed, 414 insertions(+)
 create mode 100644 mypy/dmypy.py
 create mode 100755 scripts/dmypy

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
new file mode 100644
index 0000000000000..ee1281d91d259
--- /dev/null
+++ b/mypy/dmypy.py
@@ -0,0 +1,394 @@
+"""Client for mypy daemon mode.
+
+Highly experimental!  Only supports UNIX-like systems.
+
+This manages a daemon process which keeps useful state in memory
+rather than having to read it back from disk on each run.
+"""
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import time
+
+from typing import Any, Callable, Dict, Mapping, Sequence, Tuple, TypeVar
+from mypy_extensions import NoReturn
+
+import mypy.api
+
+# Argument parser.  Subparsers are tied to action functions by the
+# @action(subparse) decorator.
+
+parser = argparse.ArgumentParser(description="Client for mymy daemon mode")
+parser.set_defaults(action=None)
+subparsers = parser.add_subparsers()
+
+start_parser = subparsers.add_parser('start', help="Start daemon")
+start_parser.add_argument('flags', metavar='FLAG', nargs='*', type=str,
+                          help="Regular mypy flags (precede with --)")
+
+status_parser = subparsers.add_parser('status', help="Show daemon status")
+
+stop_parser = subparsers.add_parser('stop', help="Stop daemon (asks it politely to go away)")
+
+kill_parser = subparsers.add_parser('kill', help="Kill daemon (kills the process)")
+
+restart_parser = subparsers.add_parser('restart',
+    help="Restart daemon (stop or kill followed by start)")
+restart_parser.add_argument('flags', metavar='FLAG', nargs='*', type=str,
+                            help="Regular mypy flags (precede with --)")
+
+check_parser = subparsers.add_parser('check', help="Check some files (requires running daemon)")
+check_parser.add_argument('files', metavar='FILE', nargs='+', help="File (or directory) to check")
+
+recheck_parser = subparsers.add_parser('recheck',
+    help="Check the same files as the most previous  check run (requires running daemon)")
+
+hang_parser =  subparsers.add_parser('hang', help="Hang for 100 seconds")
+
+help_parser = subparsers.add_parser('help')
+
+
+def main() -> None:
+    """The code is top-down."""
+    args = parser.parse_args()
+    if not args.action:
+        parser.print_usage()
+    else:
+        args.action(args)
+
+
+ActionFunction = Callable[[argparse.Namespace], None]
+
+def action(subparser: argparse.ArgumentParser) -> Callable[[ActionFunction], None]:
+    """Decorator to tie an action function to a subparser."""
+    def register(func: ActionFunction) -> None:
+        subparser.set_defaults(action=func)
+    return register
+
+
+# Action functions (run in client from command line).
+
+@action(start_parser)
+def do_start(args: argparse.Namespace) -> None:
+    """Start daemon (it must not already be running).
+
+    This is where mypy flags are set.  Setting flags is a bit awkward;
+    you have to use e.g.:
+
+      dmypy start -- --strict
+
+    since we don't want to duplicate mypy's huge list of flags.
+    """
+    try:
+        pid, sockname = get_status()
+    except SystemExit as err:
+        daemonize(server, args.flags)
+        print("Daemon started")
+    else:
+        sys.exit("Daemon is still alive")
+
+
+@action(status_parser)
+def do_status(args: argparse.Namespace) -> None:
+    """Print daemon status.
+
+    This verifies that it is responsive to requests.
+    """
+    pid, sockname = get_status()
+    print("pid: %d" % pid)
+    print("sockname: %s" % sockname)
+    try:
+        status = request('status')
+    except Exception as err:
+        print("Daemon is stuck; consider %s kill" % sys.argv[0])
+    else:
+        print("Status response:", status)
+
+
+@action(stop_parser)
+def do_stop(args: argparse.Namespace) -> None:
+    """Stop daemon politely (via a request)."""
+    try:
+        status = request('stop')
+    except Exception as err:
+        sys.exit("Daemon is stuck; consider %s kill" % sys.argv[0])
+    else:
+        if status:
+            print("Stop response:", status)
+        else:
+            print("Daemon stopped")
+
+
+@action(kill_parser)
+def do_kill(args: argparse.Namespace) -> None:
+    """Kill daemon rudely (by killing the process)."""
+    pid, sockname = get_status()
+    try:
+        os.kill(pid, signal.SIGKILL)
+    except os.error as err:
+        sys.exit(str(err))
+    else:
+        print("Daemon killed")
+
+
+@action(restart_parser)
+def do_restart(args: argparse.Namespace) -> None:
+    """Restart daemon.
+
+    We first try to stop it politely if it's running.  This also sets
+    mypy flags (and has the same issues as start).
+    """
+    try:
+        status = request('stop')
+    except SystemExit:
+        pass
+    else:
+        if status:
+            sys.exit("Status: %s" % str(status))
+        else:
+            print("Daemon stopped")
+    daemonize(server, args.flags)
+    print("Daemon started")
+
+
+@action(check_parser)
+def do_check(args: argparse.Namespace) -> None:
+    """Ask the daemon to check a list of files."""
+    response = request('check', files=args.files)
+    check_output(response)
+
+
+@action(recheck_parser)
+def do_recheck(args: argparse.Namespace) -> None:
+    """Ask the daemon to check the same list of files it checked most recently.
+
+    This doesn't work across daemon restarts.
+    """
+    response = request('recheck')
+    check_output(response)
+
+
+def check_output(response: Dict[str, Any]) -> None:
+    """Print the output from a check or recheck command."""
+    try:
+        out, err, status = response['out'], response['err'], response['status']
+    except KeyError:
+        sys.exit("Response: %s" % str(response))
+    sys.stdout.write(out)
+    sys.stderr.write(err)
+    sys.exit(status)
+
+
+@action(hang_parser)
+def do_hang(args: argparse.Namespace) -> None:
+    """Hang for 100 seconds, as a debug hack."""
+    request('hang')
+
+
+@action(help_parser)
+def do_help(args: argparse.Namespace) -> None:
+    """Print full help (same as dmypy --help)."""
+    parser.print_help()
+
+
+# Client-side infrastructure.
+
+STATUS_FILE = 'dmypy.json'
+
+def request(command: str, **kwds: object) -> Dict[str, Any]:
+    """Send a request to the daemon."""
+    args = dict(kwds)
+    if command:
+        args.update(command=command)
+    data = json.dumps(args)
+    pid, sockname = get_status()
+    sock = socket.socket(socket.AF_UNIX)
+    sock.connect(sockname)
+    sock.sendall(data.encode('utf8'))
+    sock.shutdown(socket.SHUT_WR)
+    bdata = receive(sock)
+    sock.close()
+    response = json.loads(bdata)  # JSON accepts bytes too
+    return response
+
+
+def get_status() -> Tuple[int, str]:
+    """Read status file and check if the process is alive.
+
+    Return (pid, sockname) on success.
+
+    Raise SystemExit(<message>) if something's wrong.
+    """
+    if not os.path.isfile(STATUS_FILE):
+        raise SystemExit("No status file found")
+    with open(STATUS_FILE) as f:
+        data = json.load(f)
+    if not isinstance(data, dict) or 'pid' not in data:
+        raise SystemExit("Malformed status file")
+    pid = data['pid']
+    sockname = data['sockname']
+    try:
+        os.kill(pid, 0)
+    except OSError as err:
+        print("%s: %s" % (err.__class__.__name__, err))
+        raise SystemExit("Daemon has died")
+    return pid, sockname
+
+
+DEBUG = False
+
+def daemonize(func: Callable[[Sequence[str]], NoReturn], flags: Sequence[str]) -> None:
+    """Arrange to call func() in a grandchild of the current process."""
+    pid = os.fork()
+    if pid:
+        return
+    # Child
+    try:
+        if not DEBUG:
+            os.setsid()  # Detach controlling terminal
+            devnull = os.open('/dev/null', os.O_RDWR)
+            os.dup2(devnull, 0)
+            os.dup2(devnull, 1)
+            os.dup2(devnull, 2)
+            os.close(devnull)
+        pid = os.fork()
+        if pid:
+            os._exit(0)
+        # Grandchild
+        func(flags)
+    finally:
+        # Make sure we never get back into the caller.
+        os._exit(1)
+
+
+# Server code.
+
+mypy_flags = None
+
+def server(flags: Sequence[str]) -> NoReturn:
+    global mypy_flags
+    mypy_flags = list(flags)
+    sock = create_listening_socket()
+    with open(STATUS_FILE, 'w') as f:
+        json.dump({'pid': os.getpid(), 'sockname': sock.getsockname()}, f)
+        f.write('\n')  # I like my JSON with trailing newline
+    while True:
+        conn, addr = sock.accept()
+        bdata = receive(conn)
+        data = json.loads(bdata)
+        resp = None  # type: Dict[str, Any]
+        if 'command' not in data:
+            resp = {'error': "No command found in request"}
+        else:
+            command = data['command']
+            if not isinstance(command, str):
+                resp = {'error': "Command is not a string"}
+            else:
+                command = data.pop('command')
+            resp = run_command(command, data)
+        conn.sendall(json.dumps(resp).encode('utf8'))
+        conn.close()
+        if command == 'stop':
+            break
+    sock.close()
+    sys.exit(0)
+
+
+SOCKET_NAME = 'dmypy.sock'  # In current directory.
+
+def create_listening_socket() -> socket.socket:
+    """Create the socket and set it up for listening."""
+    sockname = os.path.abspath(SOCKET_NAME)
+    if os.path.exists(sockname):
+        os.unlink(sockname)
+    sock = socket.socket(socket.AF_UNIX)
+    sock.bind(sockname)
+    sock.listen(1)
+    return sock
+
+
+CommandFunction = Callable[..., Dict[str, Any]]
+
+command_registry = {}  # type: Dict[str, CommandFunction]
+
+def run_command(command: str, data: Mapping[str, object]) -> Dict[str, object]:
+    """Run a specific command from the registry."""
+    key = 'cmd_' + command
+    if key in command_registry:
+        return command_registry[key](**data)
+    else:
+        return {'error': "Unrecognized command '%s'" % command}
+
+
+F = TypeVar('F', bound=CommandFunction)
+
+def command(func: F) -> F:
+    """Decorator to register a command function."""
+    assert func.__name__.startswith('cmd_')
+    command_registry[func.__name__] = func
+    return func
+
+
+# Command functions (run in the server via RPC).
+
+@command
+def cmd_status() -> Dict[str, object]:
+    """Return daemon status."""
+    return {'status': "I'm alive!"}
+
+
+@command
+def cmd_stop() -> Dict[str, object]:
+    """Stop daemon."""
+    return {}
+
+
+last_args = None
+
+@command
+def cmd_check(files: Sequence[str]) -> Dict[str, object]:
+    """Check a list of files."""
+    global last_args
+    last_args = ['--incremental'] + mypy_flags + ['--'] + list(files)
+    stdout, stderr, status = mypy.api.run(last_args)
+    return {'out': stdout, 'err': stderr, 'status': status}
+
+
+@command
+def cmd_recheck() -> Dict[str, object]:
+    """Check the same list of files we checked most recently."""
+    global last_args
+    if not last_args:
+        return {'error': "Command 'recheck' is only valid after a 'check' command"}
+    stdout, stderr, status = mypy.api.run(last_args)
+    return {'out': stdout, 'err': stderr, 'status': status}
+
+
+@command
+def cmd_hang() -> Dict[str, object]:
+    """Hang for 100 seconds, as a debug hack."""
+    time.sleep(100)
+    return {}
+
+
+# Network utilities.
+
+def receive(sock: socket.socket) -> bytes:
+    """Receive data from a socket until EOF."""
+    bdata = bytearray()
+    while True:
+        more = sock.recv(100000)
+        if not more:
+            break
+        bdata.extend(more)
+    return bdata
+
+
+# Run main().
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/dmypy b/scripts/dmypy
new file mode 100755
index 0000000000000..b760d5a7cc1e1
--- /dev/null
+++ b/scripts/dmypy
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Mypy daemon client.
+
+This is just a wrapper script. Look at mypy/dmypy.py for the actual
+implementation.
+"""
+
+import os
+import os.path
+import sys
+
+file_dir = os.path.dirname(__file__)
+parent_dir = os.path.join(file_dir, os.pardir)
+if os.path.exists(os.path.join(parent_dir, '.git')):
+    # We are running from a git clone.
+    sys.path.insert(0, parent_dir)
+
+import mypy.dmypy
+
+mypy.dmypy.main()

From 0108b64d277b5afa288c79cfeb797bb4c37232d4 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 6 Oct 2017 18:44:57 -0700
Subject: [PATCH 02/42] Turn serving code into a class

---
 mypy/dmypy.py | 191 +++++++++++++++++++++++++-------------------------
 1 file changed, 94 insertions(+), 97 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index ee1281d91d259..80031fad2ca23 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -86,8 +86,7 @@ def do_start(args: argparse.Namespace) -> None:
     try:
         pid, sockname = get_status()
     except SystemExit as err:
-        daemonize(server, args.flags)
-        print("Daemon started")
+        daemonize(Server(args.flags).serve)
     else:
         sys.exit("Daemon is still alive")
 
@@ -151,8 +150,7 @@ def do_restart(args: argparse.Namespace) -> None:
             sys.exit("Status: %s" % str(status))
         else:
             print("Daemon stopped")
-    daemonize(server, args.flags)
-    print("Daemon started")
+    daemonize(Server(args.flags).serve)
 
 
 @action(check_parser)
@@ -210,9 +208,8 @@ def request(command: str, **kwds: object) -> Dict[str, Any]:
     sock.connect(sockname)
     sock.sendall(data.encode('utf8'))
     sock.shutdown(socket.SHUT_WR)
-    bdata = receive(sock)
+    response = receive(sock)
     sock.close()
-    response = json.loads(bdata)  # JSON accepts bytes too
     return response
 
 
@@ -226,25 +223,28 @@ def get_status() -> Tuple[int, str]:
     if not os.path.isfile(STATUS_FILE):
         raise SystemExit("No status file found")
     with open(STATUS_FILE) as f:
-        data = json.load(f)
+        try:
+            data = json.load(f)
+        except Exception as err:
+            raise SystemExit("Malformed status file")
     if not isinstance(data, dict) or 'pid' not in data:
-        raise SystemExit("Malformed status file")
+        raise SystemExit("Invalid status file")
     pid = data['pid']
     sockname = data['sockname']
     try:
         os.kill(pid, 0)
     except OSError as err:
-        print("%s: %s" % (err.__class__.__name__, err))
         raise SystemExit("Daemon has died")
     return pid, sockname
 
 
 DEBUG = False
 
-def daemonize(func: Callable[[Sequence[str]], NoReturn], flags: Sequence[str]) -> None:
+def daemonize(func: Callable[[], NoReturn]) -> None:
     """Arrange to call func() in a grandchild of the current process."""
     pid = os.fork()
     if pid:
+        print("Daemon started")
         return
     # Child
     try:
@@ -255,11 +255,11 @@ def daemonize(func: Callable[[Sequence[str]], NoReturn], flags: Sequence[str]) -
             os.dup2(devnull, 1)
             os.dup2(devnull, 2)
             os.close(devnull)
-        pid = os.fork()
-        if pid:
-            os._exit(0)
+            pid = os.fork()
+            if pid:
+                os._exit(0)
         # Grandchild
-        func(flags)
+        func()
     finally:
         # Make sure we never get back into the caller.
         os._exit(1)
@@ -267,63 +267,12 @@ def daemonize(func: Callable[[Sequence[str]], NoReturn], flags: Sequence[str]) -
 
 # Server code.
 
-mypy_flags = None
-
-def server(flags: Sequence[str]) -> NoReturn:
-    global mypy_flags
-    mypy_flags = list(flags)
-    sock = create_listening_socket()
-    with open(STATUS_FILE, 'w') as f:
-        json.dump({'pid': os.getpid(), 'sockname': sock.getsockname()}, f)
-        f.write('\n')  # I like my JSON with trailing newline
-    while True:
-        conn, addr = sock.accept()
-        bdata = receive(conn)
-        data = json.loads(bdata)
-        resp = None  # type: Dict[str, Any]
-        if 'command' not in data:
-            resp = {'error': "No command found in request"}
-        else:
-            command = data['command']
-            if not isinstance(command, str):
-                resp = {'error': "Command is not a string"}
-            else:
-                command = data.pop('command')
-            resp = run_command(command, data)
-        conn.sendall(json.dumps(resp).encode('utf8'))
-        conn.close()
-        if command == 'stop':
-            break
-    sock.close()
-    sys.exit(0)
-
-
 SOCKET_NAME = 'dmypy.sock'  # In current directory.
 
-def create_listening_socket() -> socket.socket:
-    """Create the socket and set it up for listening."""
-    sockname = os.path.abspath(SOCKET_NAME)
-    if os.path.exists(sockname):
-        os.unlink(sockname)
-    sock = socket.socket(socket.AF_UNIX)
-    sock.bind(sockname)
-    sock.listen(1)
-    return sock
-
-
 CommandFunction = Callable[..., Dict[str, Any]]
 
 command_registry = {}  # type: Dict[str, CommandFunction]
 
-def run_command(command: str, data: Mapping[str, object]) -> Dict[str, object]:
-    """Run a specific command from the registry."""
-    key = 'cmd_' + command
-    if key in command_registry:
-        return command_registry[key](**data)
-    else:
-        return {'error': "Unrecognized command '%s'" % command}
-
-
 F = TypeVar('F', bound=CommandFunction)
 
 def command(func: F) -> F:
@@ -333,51 +282,99 @@ def command(func: F) -> F:
     return func
 
 
-# Command functions (run in the server via RPC).
+class Server:
 
-@command
-def cmd_status() -> Dict[str, object]:
-    """Return daemon status."""
-    return {'status': "I'm alive!"}
+    mypy_flags = None
 
+    def __init__(self, flags: Sequence[str]) -> None:
+        """Initialize the server with the desired mypy flags."""
+        self.mypy_flags = list(flags)
 
-@command
-def cmd_stop() -> Dict[str, object]:
-    """Stop daemon."""
-    return {}
+    def serve(self) -> NoReturn:
+        """Serve a single request, synchronously (no thread or fork)."""
+        sock = self.create_listening_socket()
+        with open(STATUS_FILE, 'w') as f:
+            json.dump({'pid': os.getpid(), 'sockname': sock.getsockname()}, f)
+            f.write('\n')  # I like my JSON with trailing newline
+        while True:
+            conn, addr = sock.accept()
+            data = receive(conn)
+            resp = None  # type: Dict[str, Any]
+            if 'command' not in data:
+                resp = {'error': "No command found in request"}
+            else:
+                command = data['command']
+                if not isinstance(command, str):
+                    resp = {'error': "Command is not a string"}
+                else:
+                    command = data.pop('command')
+                resp = self.run_command(command, data)
+            conn.sendall(json.dumps(resp).encode('utf8'))
+            conn.close()
+            if command == 'stop':
+                sock.close()
+                sys.exit(0)
+
+    def create_listening_socket(self) -> socket.socket:
+        """Create the socket and set it up for listening."""
+        self.sockname = os.path.abspath(SOCKET_NAME)
+        if os.path.exists(self.sockname):
+            os.unlink(self.sockname)
+        sock = socket.socket(socket.AF_UNIX)
+        sock.bind(self.sockname)
+        sock.listen(1)
+        return sock
+
+    def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, object]:
+        """Run a specific command from the registry."""
+        key = 'cmd_' + command
+        if key in command_registry:
+            return command_registry[key](self, **data)
+        else:
+            return {'error': "Unrecognized command '%s'" % command}
 
+    # Command functions (run in the server via RPC).
 
-last_args = None
+    @command
+    def cmd_status(self) -> Dict[str, object]:
+        """Return daemon status."""
+        return {'status': "I'm alive!"}
 
-@command
-def cmd_check(files: Sequence[str]) -> Dict[str, object]:
-    """Check a list of files."""
-    global last_args
-    last_args = ['--incremental'] + mypy_flags + ['--'] + list(files)
-    stdout, stderr, status = mypy.api.run(last_args)
-    return {'out': stdout, 'err': stderr, 'status': status}
 
+    @command
+    def cmd_stop(self) -> Dict[str, object]:
+        """Stop daemon."""
+        os.unlink(self.sockname)
+        os.unlink(STATUS_FILE)
+        return {}
 
-@command
-def cmd_recheck() -> Dict[str, object]:
-    """Check the same list of files we checked most recently."""
-    global last_args
-    if not last_args:
-        return {'error': "Command 'recheck' is only valid after a 'check' command"}
-    stdout, stderr, status = mypy.api.run(last_args)
-    return {'out': stdout, 'err': stderr, 'status': status}
+    last_args = None
 
+    @command
+    def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
+        """Check a list of files."""
+        self.last_args = ['--incremental'] + self.mypy_flags + ['--'] + list(files)
+        stdout, stderr, status = mypy.api.run(self.last_args)
+        return {'out': stdout, 'err': stderr, 'status': status}
 
-@command
-def cmd_hang() -> Dict[str, object]:
-    """Hang for 100 seconds, as a debug hack."""
-    time.sleep(100)
-    return {}
+    @command
+    def cmd_recheck(self) -> Dict[str, object]:
+        """Check the same list of files we checked most recently."""
+        if not self.last_args:
+            return {'error': "Command 'recheck' is only valid after a 'check' command"}
+        stdout, stderr, status = mypy.api.run(self.last_args)
+        return {'out': stdout, 'err': stderr, 'status': status}
+
+    @command
+    def cmd_hang(self) -> Dict[str, object]:
+        """Hang for 100 seconds, as a debug hack."""
+        time.sleep(100)
+        return {}
 
 
 # Network utilities.
 
-def receive(sock: socket.socket) -> bytes:
+def receive(sock: socket.socket) -> Any:
     """Receive data from a socket until EOF."""
     bdata = bytearray()
     while True:
@@ -385,7 +382,7 @@ def receive(sock: socket.socket) -> bytes:
         if not more:
             break
         bdata.extend(more)
-    return bdata
+    return json.loads(bdata)
 
 
 # Run main().

From 97d6c6dbca7a8585e92b074dd138652fb8fb97ea Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 6 Oct 2017 20:44:37 -0700
Subject: [PATCH 03/42] Tweaks to daemonize()

---
 mypy/dmypy.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 80031fad2ca23..f278a9f714739 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -238,10 +238,13 @@ def get_status() -> Tuple[int, str]:
     return pid, sockname
 
 
-DEBUG = False
+DEBUG = False  # If True, daemon is not a true daemon process, so we'll see its output.
 
 def daemonize(func: Callable[[], NoReturn]) -> None:
     """Arrange to call func() in a grandchild of the current process."""
+    # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python
+    sys.stdout.flush()
+    sys.stderr.flush()
     pid = os.fork()
     if pid:
         print("Daemon started")

From bfde4ac9815899fbcac005254df3b9d2c2cad159 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 8 Oct 2017 14:17:00 -0700
Subject: [PATCH 04/42] Different approach to debugging. Different approach to
 running mypy.

---
 mypy/dmypy.py | 80 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 56 insertions(+), 24 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index f278a9f714739..f6b26a4c24337 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -17,7 +17,9 @@
 from typing import Any, Callable, Dict, Mapping, Sequence, Tuple, TypeVar
 from mypy_extensions import NoReturn
 
-import mypy.api
+import mypy.build
+import mypy.errors
+import mypy.main
 
 # Argument parser.  Subparsers are tied to action functions by the
 # @action(subparse) decorator.
@@ -47,7 +49,11 @@
 recheck_parser = subparsers.add_parser('recheck',
     help="Check the same files as the most previous  check run (requires running daemon)")
 
-hang_parser =  subparsers.add_parser('hang', help="Hang for 100 seconds")
+hang_parser = subparsers.add_parser('hang', help="Hang for 100 seconds")
+
+daemon_parser = subparsers.add_parser('daemon', help="Run daemon in foreground")
+daemon_parser.add_argument('flags', metavar='FLAG', nargs='*', type=str,
+                           help="Regular mypy flags (precede with --)")
 
 help_parser = subparsers.add_parser('help')
 
@@ -187,6 +193,12 @@ def do_hang(args: argparse.Namespace) -> None:
     request('hang')
 
 
+@action(daemon_parser)
+def do_daemon(args: argparse.Namespace) -> None:
+    """Serve requests in the foreground."""
+    Server(args.flags).serve()
+
+
 @action(help_parser)
 def do_help(args: argparse.Namespace) -> None:
     """Print full help (same as dmypy --help)."""
@@ -208,9 +220,14 @@ def request(command: str, **kwds: object) -> Dict[str, Any]:
     sock.connect(sockname)
     sock.sendall(data.encode('utf8'))
     sock.shutdown(socket.SHUT_WR)
-    response = receive(sock)
-    sock.close()
-    return response
+    try:
+        response = receive(sock)
+    except OSError as err:
+        return {'error': str(err)}
+    else:
+        return response
+    finally:
+        sock.close()
 
 
 def get_status() -> Tuple[int, str]:
@@ -238,8 +255,6 @@ def get_status() -> Tuple[int, str]:
     return pid, sockname
 
 
-DEBUG = False  # If True, daemon is not a true daemon process, so we'll see its output.
-
 def daemonize(func: Callable[[], NoReturn]) -> None:
     """Arrange to call func() in a grandchild of the current process."""
     # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python
@@ -251,16 +266,16 @@ def daemonize(func: Callable[[], NoReturn]) -> None:
         return
     # Child
     try:
-        if not DEBUG:
-            os.setsid()  # Detach controlling terminal
-            devnull = os.open('/dev/null', os.O_RDWR)
-            os.dup2(devnull, 0)
-            os.dup2(devnull, 1)
-            os.dup2(devnull, 2)
-            os.close(devnull)
-            pid = os.fork()
-            if pid:
-                os._exit(0)
+        os.setsid()  # Detach controlling terminal
+        os.umask(0o27)
+        devnull = os.open('/dev/null', os.O_RDWR)
+        os.dup2(devnull, 0)
+        os.dup2(devnull, 1)
+        os.dup2(devnull, 2)
+        os.close(devnull)
+        pid = os.fork()
+        if pid:
+            os._exit(0)
         # Grandchild
         func()
     finally:
@@ -287,11 +302,15 @@ def command(func: F) -> F:
 
 class Server:
 
-    mypy_flags = None
+    # NOTE: the instance is constructed in the parent process but
+    # serve() is called in the grandchild (by daemonize()).
 
     def __init__(self, flags: Sequence[str]) -> None:
         """Initialize the server with the desired mypy flags."""
-        self.mypy_flags = list(flags)
+        sources, options = mypy.main.process_options(flags + ['-i'], False)
+        if sources:
+            sys.exit("dmypy: start/restart does not accept sources")
+        self.options = options
 
     def serve(self) -> NoReturn:
         """Serve a single request, synchronously (no thread or fork)."""
@@ -356,17 +375,28 @@ def cmd_stop(self) -> Dict[str, object]:
     @command
     def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
         """Check a list of files."""
-        self.last_args = ['--incremental'] + self.mypy_flags + ['--'] + list(files)
-        stdout, stderr, status = mypy.api.run(self.last_args)
-        return {'out': stdout, 'err': stderr, 'status': status}
+        self.last_args = list(files)
+        return self.check()
 
     @command
     def cmd_recheck(self) -> Dict[str, object]:
         """Check the same list of files we checked most recently."""
         if not self.last_args:
             return {'error': "Command 'recheck' is only valid after a 'check' command"}
-        stdout, stderr, status = mypy.api.run(self.last_args)
-        return {'out': stdout, 'err': stderr, 'status': status}
+        return self.check()
+
+    def check(self) -> Dict[str, object]:
+        sources = [mypy.build.BuildSource(arg, None, None) for arg in self.last_args]
+        try:
+            res = mypy.main.type_check_only(sources, None, self.options)
+            msgs = res.errors
+        except mypy.errors.CompileError as err:
+            msgs = err.messages
+        if msgs:
+            msgs.append("")
+            return {'out': "\n".join(msgs), 'err': "", 'status': 1}
+        else:
+            return {'out': "", 'err': "", 'status': 0}
 
     @command
     def cmd_hang(self) -> Dict[str, object]:
@@ -385,6 +415,8 @@ def receive(sock: socket.socket) -> Any:
         if not more:
             break
         bdata.extend(more)
+    if not bdata:
+        raise OSError("No data received")
     return json.loads(bdata)
 
 

From 968871606b561a4f775d1bb1964c09a053c601b4 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 8 Oct 2017 16:35:41 -0700
Subject: [PATCH 05/42] Improve processing of file args

---
 mypy/dmypy.py | 18 +++++++++++++++---
 mypy/main.py  | 43 ++++++++++++++++++++++++-------------------
 2 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index f6b26a4c24337..958b6087bc164 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -7,6 +7,7 @@
 """
 
 import argparse
+import io
 import json
 import os
 import signal
@@ -14,7 +15,7 @@
 import sys
 import time
 
-from typing import Any, Callable, Dict, Mapping, Sequence, Tuple, TypeVar
+from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple, TypeVar
 from mypy_extensions import NoReturn
 
 import mypy.build
@@ -305,7 +306,7 @@ class Server:
     # NOTE: the instance is constructed in the parent process but
     # serve() is called in the grandchild (by daemonize()).
 
-    def __init__(self, flags: Sequence[str]) -> None:
+    def __init__(self, flags: List[str]) -> None:
         """Initialize the server with the desired mypy flags."""
         sources, options = mypy.main.process_options(flags + ['-i'], False)
         if sources:
@@ -386,7 +387,18 @@ def cmd_recheck(self) -> Dict[str, object]:
         return self.check()
 
     def check(self) -> Dict[str, object]:
-        sources = [mypy.build.BuildSource(arg, None, None) for arg in self.last_args]
+        # Capture stdout/stderr and catch SystemExit while processing the source list.
+        save_stdout = sys.stdout
+        save_stderr = sys.stderr
+        try:
+            sys.stdout = stdout = io.StringIO()
+            sys.stderr = stderr = io.StringIO()
+            sources = mypy.main.create_source_list(self.last_args, self.options)
+        except SystemExit as err:
+            return {'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': err.code}
+        finally:
+            sys.stdout = save_stdout
+            sys.stderr = save_stderr
         try:
             res = mypy.main.type_check_only(sources, None, self.options)
             msgs = res.errors
diff --git a/mypy/main.py b/mypy/main.py
index 60bf8a22e0bda..10a012107bc71 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -505,28 +505,33 @@ def add_invertible_flag(flag: str,
         targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
         return targets, options
     else:
-        targets = []
-        for f in special_opts.files:
-            if f.endswith(PY_EXTENSIONS):
-                try:
-                    targets.append(BuildSource(f, crawl_up(f)[1], None))
-                except InvalidPackageName as e:
-                    fail(str(e))
-            elif os.path.isdir(f):
-                try:
-                    sub_targets = expand_dir(f)
-                except InvalidPackageName as e:
-                    fail(str(e))
-                if not sub_targets:
-                    fail("There are no .py[i] files in directory '{}'"
-                         .format(f))
-                targets.extend(sub_targets)
-            else:
-                mod = os.path.basename(f) if options.scripts_are_modules else None
-                targets.append(BuildSource(f, mod, None))
+        targets = create_source_list(special_opts.files, options)
         return targets, options
 
 
+def create_source_list(files: List[str], options: Options) -> List[BuildSource]:
+    targets = []
+    for f in files:
+        if f.endswith(PY_EXTENSIONS):
+            try:
+                targets.append(BuildSource(f, crawl_up(f)[1], None))
+            except InvalidPackageName as e:
+                fail(str(e))
+        elif os.path.isdir(f):
+            try:
+                sub_targets = expand_dir(f)
+            except InvalidPackageName as e:
+                fail(str(e))
+            if not sub_targets:
+                fail("There are no .py[i] files in directory '{}'"
+                     .format(f))
+            targets.extend(sub_targets)
+        else:
+            mod = os.path.basename(f) if options.scripts_are_modules else None
+            targets.append(BuildSource(f, mod, None))
+    return targets
+
+
 def keyfunc(name: str) -> Tuple[int, str]:
     """Determines sort order for directory listing.
 

From b4ede7d99a3f6888794a01c473fa18c8da7eabfa Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 8 Oct 2017 16:46:27 -0700
Subject: [PATCH 06/42] Simplify command dispatch

---
 mypy/dmypy.py | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 958b6087bc164..65c8f9bae5f6d 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -288,19 +288,6 @@ def daemonize(func: Callable[[], NoReturn]) -> None:
 
 SOCKET_NAME = 'dmypy.sock'  # In current directory.
 
-CommandFunction = Callable[..., Dict[str, Any]]
-
-command_registry = {}  # type: Dict[str, CommandFunction]
-
-F = TypeVar('F', bound=CommandFunction)
-
-def command(func: F) -> F:
-    """Decorator to register a command function."""
-    assert func.__name__.startswith('cmd_')
-    command_registry[func.__name__] = func
-    return func
-
-
 class Server:
 
     # NOTE: the instance is constructed in the parent process but
@@ -351,20 +338,19 @@ def create_listening_socket(self) -> socket.socket:
     def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, object]:
         """Run a specific command from the registry."""
         key = 'cmd_' + command
-        if key in command_registry:
-            return command_registry[key](self, **data)
-        else:
+        method = getattr(self.__class__, key, None)
+        if method is None:
             return {'error': "Unrecognized command '%s'" % command}
+        else:
+            return method(self, **data)
 
     # Command functions (run in the server via RPC).
 
-    @command
     def cmd_status(self) -> Dict[str, object]:
         """Return daemon status."""
         return {'status': "I'm alive!"}
 
 
-    @command
     def cmd_stop(self) -> Dict[str, object]:
         """Stop daemon."""
         os.unlink(self.sockname)
@@ -373,13 +359,11 @@ def cmd_stop(self) -> Dict[str, object]:
 
     last_args = None
 
-    @command
     def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
         """Check a list of files."""
         self.last_args = list(files)
         return self.check()
 
-    @command
     def cmd_recheck(self) -> Dict[str, object]:
         """Check the same list of files we checked most recently."""
         if not self.last_args:
@@ -410,7 +394,6 @@ def check(self) -> Dict[str, object]:
         else:
             return {'out': "", 'err': "", 'status': 0}
 
-    @command
     def cmd_hang(self) -> Dict[str, object]:
         """Hang for 100 seconds, as a debug hack."""
         time.sleep(100)

From 6bf9c7edc1951f9a54a53a1adbd7bcd0945a3831 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 8 Oct 2017 17:05:12 -0700
Subject: [PATCH 07/42] Fix crash in validate_meta() when --skip-version-check
 given

---
 mypy/build.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mypy/build.py b/mypy/build.py
index 891fc3c8d6ddf..5c233ab484ceb 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -999,6 +999,9 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
 
     # TODO: Share stat() outcome with find_module()
     path = os.path.abspath(path)
+    if not os.path.isfile(path):
+        manager.log('Metadata abandoned for {}: file {} does not exist'.format(id, path))
+        return None
     st = manager.get_stat(path)  # TODO: Errors
     size = st.st_size
     if size != meta.size:

From f92c84971af3a82aaf57d4c4ebf427ad64357eec Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 8 Oct 2017 17:56:48 -0700
Subject: [PATCH 08/42] Small fixes

- Don't die if client closed commection.
- Support `@file` to read arguments from a file.
- Call build.build() directly.
---
 mypy/dmypy.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 65c8f9bae5f6d..814e115d244b9 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -25,7 +25,8 @@
 # Argument parser.  Subparsers are tied to action functions by the
 # @action(subparse) decorator.
 
-parser = argparse.ArgumentParser(description="Client for mymy daemon mode")
+parser = argparse.ArgumentParser(description="Client for mymy daemon mode",
+                                 fromfile_prefix_chars='@')
 parser.set_defaults(action=None)
 subparsers = parser.add_subparsers()
 
@@ -319,7 +320,10 @@ def serve(self) -> NoReturn:
                 else:
                     command = data.pop('command')
                 resp = self.run_command(command, data)
-            conn.sendall(json.dumps(resp).encode('utf8'))
+            try:
+                conn.sendall(json.dumps(resp).encode('utf8'))
+            except OSError as err:
+                pass  # Maybe the client hung up
             conn.close()
             if command == 'stop':
                 sock.close()
@@ -384,7 +388,7 @@ def check(self) -> Dict[str, object]:
             sys.stdout = save_stdout
             sys.stderr = save_stderr
         try:
-            res = mypy.main.type_check_only(sources, None, self.options)
+            res = mypy.build.build(sources, self.options)
             msgs = res.errors
         except mypy.errors.CompileError as err:
             msgs = err.messages

From 34fe8747fa1c5aab46e7fbe77d3ae50830275c7a Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 9 Oct 2017 09:37:46 -0700
Subject: [PATCH 09/42] Disallow report-generating options, --no-incremental,
 and --quick

---
 mypy/dmypy.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 814e115d244b9..b7d975201a92f 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -296,9 +296,15 @@ class Server:
 
     def __init__(self, flags: List[str]) -> None:
         """Initialize the server with the desired mypy flags."""
-        sources, options = mypy.main.process_options(flags + ['-i'], False)
+        sources, options = mypy.main.process_options(['-i'] + flags, False)
         if sources:
             sys.exit("dmypy: start/restart does not accept sources")
+        if options.report_dirs:
+            sys.exit("dmypy: start/restart cannot generate reports")
+        if not options.incremental:
+            sys.exit("dmypy: start/restart should not disable incremental mode")
+        if options.quick_and_dirty:
+            sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
         self.options = options
 
     def serve(self) -> NoReturn:
@@ -388,6 +394,7 @@ def check(self) -> Dict[str, object]:
             sys.stdout = save_stdout
             sys.stderr = save_stderr
         try:
+            # TODO: This is where we can start reusing the graph.
             res = mypy.build.build(sources, self.options)
             msgs = res.errors
         except mypy.errors.CompileError as err:

From 6635e95c8a357c0ded7d1758a468ea541149ab44 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 9 Oct 2017 16:41:08 -0700
Subject: [PATCH 10/42] Save sources, not raw strings

---
 mypy/dmypy.py | 25 ++++++++++++-------------
 mypy/main.py  |  4 ++--
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index b7d975201a92f..53092aa07e9ff 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -367,35 +367,34 @@ def cmd_stop(self) -> Dict[str, object]:
         os.unlink(STATUS_FILE)
         return {}
 
-    last_args = None
+    last_sources = None
 
     def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
         """Check a list of files."""
-        self.last_args = list(files)
-        return self.check()
-
-    def cmd_recheck(self) -> Dict[str, object]:
-        """Check the same list of files we checked most recently."""
-        if not self.last_args:
-            return {'error': "Command 'recheck' is only valid after a 'check' command"}
-        return self.check()
-
-    def check(self) -> Dict[str, object]:
         # Capture stdout/stderr and catch SystemExit while processing the source list.
         save_stdout = sys.stdout
         save_stderr = sys.stderr
         try:
             sys.stdout = stdout = io.StringIO()
             sys.stderr = stderr = io.StringIO()
-            sources = mypy.main.create_source_list(self.last_args, self.options)
+            self.last_sources = mypy.main.create_source_list(files, self.options)
         except SystemExit as err:
             return {'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': err.code}
         finally:
             sys.stdout = save_stdout
             sys.stderr = save_stderr
+        return self.check()
+
+    def cmd_recheck(self) -> Dict[str, object]:
+        """Check the same list of files we checked most recently."""
+        if not self.last_sources:
+            return {'error': "Command 'recheck' is only valid after a 'check' command"}
+        return self.check()
+
+    def check(self) -> Dict[str, object]:
         try:
             # TODO: This is where we can start reusing the graph.
-            res = mypy.build.build(sources, self.options)
+            res = mypy.build.build(self.last_sources, self.options)
             msgs = res.errors
         except mypy.errors.CompileError as err:
             msgs = err.messages
diff --git a/mypy/main.py b/mypy/main.py
index 10a012107bc71..f1834e7259b65 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -8,7 +8,7 @@
 import sys
 import time
 
-from typing import Any, Dict, List, Mapping, Optional, Set, Tuple
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple
 
 from mypy import build
 from mypy import defaults
@@ -509,7 +509,7 @@ def add_invertible_flag(flag: str,
         return targets, options
 
 
-def create_source_list(files: List[str], options: Options) -> List[BuildSource]:
+def create_source_list(files: Sequence[str], options: Options) -> List[BuildSource]:
     targets = []
     for f in files:
         if f.endswith(PY_EXTENSIONS):

From 5c3786a361e8e3011f5805a0b8281c0bf63439ee Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 08:33:07 -0700
Subject: [PATCH 11/42] Primitive tracing of os.stat() calls

---
 mypy/main.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/mypy/main.py b/mypy/main.py
index f1834e7259b65..36d820ebbb482 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -28,6 +28,20 @@ class InvalidPackageName(Exception):
     """Exception indicating that a package name was invalid."""
 
 
+orig_stat = os.stat
+
+def stat_proxy(path: str) -> os.stat_result:
+    try:
+        st = orig_stat(path)
+    except os.error as err:
+        print("stat(%r) -> %s" % (path, err))
+        raise
+    else:
+        print("stat(%r) -> (st_mode=%o, st_mtime=%d, st_size=%d)" %
+              (path, st.st_mode, st.st_mtime, st.st_size))
+        return st
+
+
 def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None:
     """Main entry point to the type checker.
 
@@ -37,6 +51,7 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None:
         be used.
     """
     t0 = time.time()
+    ## os.stat = stat_proxy
     if script_path:
         bin_dir = find_bin_directory(script_path)  # type: Optional[str]
     else:

From 68cc680a90654c9b2b887afa643e1eeea88fbdda Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 10:46:51 -0700
Subject: [PATCH 12/42] Save and reuse meta and tree

---
 mypy/build.py | 72 ++++++++++++++++++++++++++++++++++++++++++++-------
 mypy/dmypy.py |  8 ++++--
 2 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 5c233ab484ceb..0c1fb7f0ad46d 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -122,7 +122,9 @@ def is_source(self, file: MypyFile) -> bool:
 def build(sources: List[BuildSource],
           options: Options,
           alt_lib_path: Optional[str] = None,
-          bin_dir: Optional[str] = None) -> BuildResult:
+          bin_dir: Optional[str] = None,
+          saved_cache: Optional['SavedCache'] = None,
+          ) -> BuildResult:
     """Analyze a program.
 
     A single call to build performs parsing, semantic analysis and optionally
@@ -195,7 +197,8 @@ def build(sources: List[BuildSource],
                            options=options,
                            version_id=__version__,
                            plugin=plugin,
-                           errors=errors)
+                           errors=errors,
+                           saved_cache=saved_cache)
 
     try:
         graph = dispatch(sources, manager)
@@ -452,6 +455,9 @@ def find_config_file_line_number(path: str, section: str, setting_name: str) ->
     return -1
 
 
+SavedCache = Dict[str, Tuple[CacheMeta, MypyFile]]
+
+
 # TODO: Get rid of all_types.  It's not used except for one log message.
 #       Maybe we could instead publish a map from module ID to its type_map.
 class BuildManager:
@@ -476,8 +482,15 @@ class BuildManager:
       version_id:      The current mypy version (based on commit id when possible)
       plugin:          Active mypy plugin(s)
       errors:          Used for reporting all errors
+      saved_cache:     Dict with saved cache state for dmypy (read-write!)
     """
 
+    # Counters for various cache-related events
+    fresh_metas = 0
+    fresh_trees = 0
+    reused_metas = 0
+    reused_trees = 0
+
     def __init__(self, data_dir: str,
                  lib_path: List[str],
                  ignore_prefix: str,
@@ -486,7 +499,9 @@ def __init__(self, data_dir: str,
                  options: Options,
                  version_id: str,
                  plugin: Plugin,
-                 errors: Errors) -> None:
+                 errors: Errors,
+                 saved_cache: Optional[SavedCache] = None,
+                 ) -> None:
         self.start_time = time.time()
         self.data_dir = data_dir
         self.errors = errors
@@ -501,7 +516,6 @@ def __init__(self, data_dir: str,
         self.plugin = plugin
         self.semantic_analyzer = SemanticAnalyzerPass2(self.modules, self.missing_modules,
                                                   lib_path, self.errors, self.plugin)
-        self.modules = self.semantic_analyzer.modules
         self.semantic_analyzer_pass3 = SemanticAnalyzerPass3(self.modules, self.errors,
                                                              self.semantic_analyzer)
         self.all_types = {}  # type: Dict[Expression, Type]
@@ -509,6 +523,7 @@ def __init__(self, data_dir: str,
         self.stale_modules = set()  # type: Set[str]
         self.rechecked_modules = set()  # type: Set[str]
         self.plugin = plugin
+        self.saved_cache = saved_cache if saved_cache is not None else {}  # type: SavedCache
 
     def maybe_swap_for_shadow_path(self, path: str) -> str:
         if (self.options.shadow_file and
@@ -880,6 +895,14 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
       A CacheMeta instance if the cache data was found and appears
       valid; otherwise None.
     """
+    saved_cache = manager.saved_cache
+    if id in saved_cache:
+        m, t = saved_cache[id]
+        manager.reused_metas += 1
+        manager.trace("Reusing saved metadata for %s" % id)
+        # Note: it could still be skipped if the mtime/size/hash mismatches.
+        return m
+
     # TODO: May need to take more build options into account
     meta_json, data_json = get_cache_names(id, path, manager)
     manager.trace('Looking for {} at {}'.format(id, meta_json))
@@ -948,6 +971,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
                                   .format(key, cached_options.get(key), current_options.get(key)))
         return None
 
+    manager.fresh_metas += 1
     return m
 
 
@@ -999,6 +1023,7 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
 
     # TODO: Share stat() outcome with find_module()
     path = os.path.abspath(path)
+    # TODO: Don't use isfile() but check st.st_mode
     if not os.path.isfile(path):
         manager.log('Metadata abandoned for {}: file {} does not exist'.format(id, path))
         return None
@@ -1047,7 +1072,7 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
             return meta
 
     # It's a match on (id, path, size, hash, mtime).
-    manager.log('Metadata fresh for {}: file {}'.format(id, path))
+    manager.trace('Metadata fresh for {}: file {}'.format(id, path))
     return meta
 
 
@@ -1596,6 +1621,7 @@ def load_tree(self) -> None:
         # TODO: Assert data file wasn't changed.
         self.tree = MypyFile.deserialize(data)
         self.manager.modules[self.id] = self.tree
+        self.manager.fresh_trees += 1
 
     def fix_cross_refs(self) -> None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
@@ -1858,6 +1884,7 @@ def write_cache(self) -> None:
 
 
 def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
+    manager.log("")
     manager.log("Mypy version %s" % __version__)
     graph = load_graph(sources, manager)
     if not graph:
@@ -1871,9 +1898,20 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
     if manager.options.warn_unused_ignores:
         # TODO: This could also be a per-module option.
         manager.errors.generate_unused_ignore_notes()
+    # TODO: What about cache entries deleted because of errors?
+    manager.saved_cache.update(preserve_cache(graph))
     return graph
 
 
+def preserve_cache(graph: Graph) -> SavedCache:
+    saved_cache = {}
+    for id, state in graph.items():
+        assert state.id == id
+        if state.meta is not None:
+            saved_cache[id] = (state.meta, state.tree)
+    return saved_cache
+
+
 class NodeInfo:
     """Some info about a node in the graph of SCCs."""
 
@@ -2095,7 +2133,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
 
         scc_str = " ".join(scc)
         if fresh:
-            manager.log("Queuing %s SCC (%s)" % (fresh_msg, scc_str))
+            manager.trace("Queuing %s SCC (%s)" % (fresh_msg, scc_str))
             fresh_scc_queue.append(scc)
         else:
             if len(fresh_scc_queue) > 0:
@@ -2110,7 +2148,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 # TODO: see if it's possible to determine if we need to process only a
                 # _subset_ of the past SCCs instead of having to process them all.
                 for prev_scc in fresh_scc_queue:
-                    process_fresh_scc(graph, prev_scc)
+                    process_fresh_scc(graph, prev_scc, manager)
                 fresh_scc_queue = []
             size = len(scc)
             if size == 1:
@@ -2119,9 +2157,14 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg))
             process_stale_scc(graph, scc, manager)
 
+    manager.log("Stats: fresh_metas=%d, fresh_trees=%d, reused_metas=%d, reused_trees=%d" %
+                (manager.fresh_metas, manager.fresh_trees,
+                 manager.reused_metas, manager.reused_trees))
     sccs_left = len(fresh_scc_queue)
     if sccs_left:
-        manager.log("{} fresh SCCs left in queue (and will remain unprocessed)".format(sccs_left))
+        nodes_left = sum(len(scc) for scc in fresh_scc_queue)
+        manager.log("{} fresh SCCs ({} nodes) left in queue (and will remain unprocessed)"
+                    .format(sccs_left, nodes_left))
         manager.trace(str(fresh_scc_queue))
     else:
         manager.log("No fresh SCCs left in queue")
@@ -2174,8 +2217,19 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) ->
     return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)]
 
 
-def process_fresh_scc(graph: Graph, scc: List[str]) -> None:
+def process_fresh_scc(graph: Graph, scc: List[str], manager: BuildManager) -> None:
     """Process the modules in one SCC from their cached data."""
+    # TODO: Clean this up, it's ugly.
+    saved_cache = manager.saved_cache
+    if all(id in saved_cache for id in scc):
+        trees = {id: saved_cache[id][1] for id in scc}
+        if all(trees.values()):
+            for id, tree in trees.items():
+                manager.reused_trees += 1
+                manager.trace("Reusing saved tree %s" % id)
+                graph[id].tree = tree
+                manager.modules[id] = tree
+            return
     for id in scc:
         graph[id].load_tree()
     for id in scc:
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 53092aa07e9ff..17b2476e53962 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -306,6 +306,7 @@ def __init__(self, flags: List[str]) -> None:
         if options.quick_and_dirty:
             sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
         self.options = options
+        ## os.stat = mypy.main.stat_proxy  # XXX
 
     def serve(self) -> NoReturn:
         """Serve a single request, synchronously (no thread or fork)."""
@@ -371,6 +372,7 @@ def cmd_stop(self) -> Dict[str, object]:
 
     def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
         """Check a list of files."""
+        # TODO: Move this into check(), in case one of the args is a directory.
         # Capture stdout/stderr and catch SystemExit while processing the source list.
         save_stdout = sys.stdout
         save_stderr = sys.stderr
@@ -391,10 +393,12 @@ def cmd_recheck(self) -> Dict[str, object]:
             return {'error': "Command 'recheck' is only valid after a 'check' command"}
         return self.check()
 
+    saved_cache = {}  # type: mypy.build.SavedCache
+
     def check(self) -> Dict[str, object]:
         try:
-            # TODO: This is where we can start reusing the graph.
-            res = mypy.build.build(self.last_sources, self.options)
+            # saved_cache is mutated in place.
+            res = mypy.build.build(self.last_sources, self.options, saved_cache=self.saved_cache)
             msgs = res.errors
         except mypy.errors.CompileError as err:
             msgs = err.messages

From 1f754b47130b22fc91316732bcf2b962ee420660 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 14:36:05 -0700
Subject: [PATCH 13/42] Revive self.meta when writing cache (hope this works)

---
 mypy/build.py | 61 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 0c1fb7f0ad46d..0000dedbd68b4 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -342,6 +342,27 @@ def default_lib_path(data_dir: str,
 # silent mode or simply not found.
 
 
+def cache_meta_from_dict(meta: Dict[str, Any], data_json: str) -> CacheMeta:
+    sentinel = None  # type: Any  # the values will be post-validated below
+    return CacheMeta(
+        meta.get('id', sentinel),
+        meta.get('path', sentinel),
+        int(meta['mtime']) if 'mtime' in meta else sentinel,
+        meta.get('size', sentinel),
+        meta.get('hash', sentinel),
+        meta.get('dependencies', []),
+        int(meta['data_mtime']) if 'data_mtime' in meta else sentinel,
+        data_json,
+        meta.get('suppressed', []),
+        meta.get('child_modules', []),
+        meta.get('options'),
+        meta.get('dep_prios', []),
+        meta.get('interface_hash', ''),
+        meta.get('version_id', sentinel),
+        meta.get('ignore_all', True),
+    )
+
+
 # Priorities used for imports.  (Here, top-level includes inside a class.)
 # These are used to determine a more predictable order in which the
 # nodes in an import cycle are processed.
@@ -917,24 +938,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
         manager.log('Could not load cache for {}: meta cache is not a dict: {}'
                     .format(id, repr(meta)))
         return None
-    sentinel = None  # type: Any  # the values will be post-validated below
-    m = CacheMeta(
-        meta.get('id', sentinel),
-        meta.get('path', sentinel),
-        int(meta['mtime']) if 'mtime' in meta else sentinel,
-        meta.get('size', sentinel),
-        meta.get('hash', sentinel),
-        meta.get('dependencies', []),
-        int(meta['data_mtime']) if 'data_mtime' in meta else sentinel,
-        data_json,
-        meta.get('suppressed', []),
-        meta.get('child_modules', []),
-        meta.get('options'),
-        meta.get('dep_prios', []),
-        meta.get('interface_hash', ''),
-        meta.get('version_id', sentinel),
-        meta.get('ignore_all', True),
-    )
+    m = cache_meta_from_dict(meta, data_json)
     # Don't check for path match, that is dealt with in validate_meta().
     if (m.id != id or
             m.mtime is None or m.size is None or
@@ -995,7 +999,7 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
                   ignore_all: bool, manager: BuildManager) -> Optional[CacheMeta]:
     '''Checks whether the cached AST of this module can be used.
 
-    Return:
+    Returns:
       None, if the cached AST is unusable.
       Original meta, if mtime/size matched.
       Meta with mtime updated to match source file, if hash/size matched but mtime/path didn't.
@@ -1087,7 +1091,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
                 dependencies: List[str], suppressed: List[str],
                 child_modules: List[str], dep_prios: List[int],
                 old_interface_hash: str, source_hash: str,
-                ignore_all: bool, manager: BuildManager) -> str:
+                ignore_all: bool, manager: BuildManager) -> Tuple[str, CacheMeta]:
     """Write cache files for a module.
 
     Note that this mypy's behavior is still correct when any given
@@ -1104,8 +1108,10 @@ def write_cache(id: str, path: str, tree: MypyFile,
       old_interface_hash: the hash from the previous version of the data cache file
       manager: the build manager (for pyversion, log/trace)
 
-    Return:
-      The new interface hash based on the serialized tree
+    Returns:
+      A tuple containing the interface hash and CacheMeta
+      corresponding to the metadata that was written (the latter may
+      be None if the cache could not be written).
     """
     # Obtain file paths
     path = os.path.abspath(path)
@@ -1138,7 +1144,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
             except OSError:
                 pass
         # Still return the interface hash we computed.
-        return interface_hash
+        return interface_hash, None
 
     # Write data cache file, if applicable
     if old_interface_hash == interface_hash:
@@ -1160,7 +1166,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
             # data_mtime field won't match the data file's mtime.
             # Both have the effect of slowing down the next run a
             # little bit due to an out-of-date cache file.
-            return interface_hash
+            return interface_hash, None
         data_mtime = getmtime(data_json)
 
     mtime = int(st.st_mtime)
@@ -1194,7 +1200,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
         # The next run will simply find the cache entry out of date.
         manager.log("Error writing meta JSON file {}".format(meta_json))
 
-    return interface_hash
+    return interface_hash, cache_meta_from_dict(meta, data_json)
 
 
 def delete_cache(id: str, path: str, manager: BuildManager) -> None:
@@ -1586,8 +1592,7 @@ def mark_as_rechecked(self) -> None:
         self.manager.rechecked_modules.add(self.id)
 
     def mark_interface_stale(self, *, on_errors: bool = False) -> None:
-        """Marks this module as having a stale public interface, and discards the cache data."""
-        self.meta = None
+        """Marks this module as having a stale public interface."""
         self.externally_same = False
         if not on_errors:
             self.manager.stale_modules.add(self.id)
@@ -1870,7 +1875,7 @@ def write_cache(self) -> None:
             self.mark_interface_stale(on_errors=True)
             return
         dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
-        new_interface_hash = write_cache(
+        new_interface_hash, self.meta = write_cache(
             self.id, self.path, self.tree,
             list(self.dependencies), list(self.suppressed), list(self.child_modules),
             dep_prios, self.interface_hash, self.source_hash, self.ignore_all,

From 9148acf72a518eb9ef248d93b88e0f2e5c7fd57c Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 15:05:00 -0700
Subject: [PATCH 14/42] Fix mypy and lint errors

---
 mypy/build.py |  4 ++--
 mypy/dmypy.py | 13 +++++++------
 mypy/main.py  |  3 ++-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 0000dedbd68b4..f04ebcf6c0c38 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -1091,7 +1091,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
                 dependencies: List[str], suppressed: List[str],
                 child_modules: List[str], dep_prios: List[int],
                 old_interface_hash: str, source_hash: str,
-                ignore_all: bool, manager: BuildManager) -> Tuple[str, CacheMeta]:
+                ignore_all: bool, manager: BuildManager) -> Tuple[str, Optional[CacheMeta]]:
     """Write cache files for a module.
 
     Note that this mypy's behavior is still correct when any given
@@ -1912,7 +1912,7 @@ def preserve_cache(graph: Graph) -> SavedCache:
     saved_cache = {}
     for id, state in graph.items():
         assert state.id == id
-        if state.meta is not None:
+        if state.meta is not None and state.tree is not None:
             saved_cache[id] = (state.meta, state.tree)
     return saved_cache
 
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 17b2476e53962..5569119ead349 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -16,7 +16,6 @@
 import time
 
 from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple, TypeVar
-from mypy_extensions import NoReturn
 
 import mypy.build
 import mypy.errors
@@ -71,6 +70,7 @@ def main() -> None:
 
 ActionFunction = Callable[[argparse.Namespace], None]
 
+
 def action(subparser: argparse.ArgumentParser) -> Callable[[ActionFunction], None]:
     """Decorator to tie an action function to a subparser."""
     def register(func: ActionFunction) -> None:
@@ -211,6 +211,7 @@ def do_help(args: argparse.Namespace) -> None:
 
 STATUS_FILE = 'dmypy.json'
 
+
 def request(command: str, **kwds: object) -> Dict[str, Any]:
     """Send a request to the daemon."""
     args = dict(kwds)
@@ -257,7 +258,7 @@ def get_status() -> Tuple[int, str]:
     return pid, sockname
 
 
-def daemonize(func: Callable[[], NoReturn]) -> None:
+def daemonize(func: Callable[[], None]) -> None:
     """Arrange to call func() in a grandchild of the current process."""
     # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python
     sys.stdout.flush()
@@ -289,6 +290,7 @@ def daemonize(func: Callable[[], NoReturn]) -> None:
 
 SOCKET_NAME = 'dmypy.sock'  # In current directory.
 
+
 class Server:
 
     # NOTE: the instance is constructed in the parent process but
@@ -306,9 +308,8 @@ def __init__(self, flags: List[str]) -> None:
         if options.quick_and_dirty:
             sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
         self.options = options
-        ## os.stat = mypy.main.stat_proxy  # XXX
 
-    def serve(self) -> NoReturn:
+    def serve(self) -> None:
         """Serve a single request, synchronously (no thread or fork)."""
         sock = self.create_listening_socket()
         with open(STATUS_FILE, 'w') as f:
@@ -317,7 +318,7 @@ def serve(self) -> NoReturn:
         while True:
             conn, addr = sock.accept()
             data = receive(conn)
-            resp = None  # type: Dict[str, Any]
+            resp = {}  # type: Dict[str, Any]
             if 'command' not in data:
                 resp = {'error': "No command found in request"}
             else:
@@ -361,7 +362,6 @@ def cmd_status(self) -> Dict[str, object]:
         """Return daemon status."""
         return {'status': "I'm alive!"}
 
-
     def cmd_stop(self) -> Dict[str, object]:
         """Stop daemon."""
         os.unlink(self.sockname)
@@ -396,6 +396,7 @@ def cmd_recheck(self) -> Dict[str, object]:
     saved_cache = {}  # type: mypy.build.SavedCache
 
     def check(self) -> Dict[str, object]:
+        assert self.last_sources
         try:
             # saved_cache is mutated in place.
             res = mypy.build.build(self.last_sources, self.options, saved_cache=self.saved_cache)
diff --git a/mypy/main.py b/mypy/main.py
index 36d820ebbb482..9de3054a927ed 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -30,6 +30,7 @@ class InvalidPackageName(Exception):
 
 orig_stat = os.stat
 
+
 def stat_proxy(path: str) -> os.stat_result:
     try:
         st = orig_stat(path)
@@ -51,7 +52,7 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None:
         be used.
     """
     t0 = time.time()
-    ## os.stat = stat_proxy
+    # To log stat() calls: os.stat = stat_proxy
     if script_path:
         bin_dir = find_bin_directory(script_path)  # type: Optional[str]
     else:

From 398d64924262383482d87dac076f848209446544 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 16:00:23 -0700
Subject: [PATCH 15/42] Tweak logging code

---
 mypy/build.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index f04ebcf6c0c38..7742f08b37da8 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -662,7 +662,10 @@ def report_file(self,
 
     def log(self, *message: str) -> None:
         if self.options.verbosity >= 1:
-            print('LOG: ', *message, file=sys.stderr)
+            if message:
+                print('LOG: ', *message, file=sys.stderr)
+            else:
+                print(file=sys.stderr)
             sys.stderr.flush()
 
     def trace(self, *message: str) -> None:
@@ -1889,7 +1892,7 @@ def write_cache(self) -> None:
 
 
 def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
-    manager.log("")
+    manager.log()
     manager.log("Mypy version %s" % __version__)
     graph = load_graph(sources, manager)
     if not graph:
@@ -2142,7 +2145,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             fresh_scc_queue.append(scc)
         else:
             if len(fresh_scc_queue) > 0:
-                manager.log("Processing the last {} queued SCCs".format(len(fresh_scc_queue)))
+                manager.log("Processing {} queued SCCs".format(len(fresh_scc_queue)))
                 # Defer processing fresh SCCs until we actually run into a stale SCC
                 # and need the earlier modules to be loaded.
                 #

From 75a9a16fb082f696457daf1b6f6f97e05368f7ff Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Wed, 11 Oct 2017 16:29:07 -0700
Subject: [PATCH 16/42] Delete saved cache entry; tweak log message

---
 mypy/build.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mypy/build.py b/mypy/build.py
index 7742f08b37da8..2bad7bcb3dd6c 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -1216,6 +1216,8 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None:
     path = os.path.abspath(path)
     meta_json, data_json = get_cache_names(id, path, manager)
     manager.log('Deleting {} {} {} {}'.format(id, path, meta_json, data_json))
+    if id in manager.saved_cache:
+        del manager.saved_cache[id]
 
     for filename in [data_json, meta_json]:
         try:
@@ -2145,7 +2147,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             fresh_scc_queue.append(scc)
         else:
             if len(fresh_scc_queue) > 0:
-                manager.log("Processing {} queued SCCs".format(len(fresh_scc_queue)))
+                manager.log("Processing {} queued fresh SCCs".format(len(fresh_scc_queue)))
                 # Defer processing fresh SCCs until we actually run into a stale SCC
                 # and need the earlier modules to be loaded.
                 #

From c221225e778542e662da1f4d7ffde90a0a037463 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 13:18:04 -0700
Subject: [PATCH 17/42] Added tests for dmypy (some still failing)

---
 mypy/dmypy.py          |  17 +--
 mypy/test/testdmypy.py | 298 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 307 insertions(+), 8 deletions(-)
 create mode 100644 mypy/test/testdmypy.py

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 5569119ead349..96bdd7e40055c 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -15,7 +15,7 @@
 import sys
 import time
 
-from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple, TypeVar
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar
 
 import mypy.build
 import mypy.errors
@@ -298,6 +298,7 @@ class Server:
 
     def __init__(self, flags: List[str]) -> None:
         """Initialize the server with the desired mypy flags."""
+        self.saved_cache = {}  # type: mypy.build.SavedCache
         sources, options = mypy.main.process_options(['-i'] + flags, False)
         if sources:
             sys.exit("dmypy: start/restart does not accept sources")
@@ -385,21 +386,21 @@ def cmd_check(self, files: Sequence[str]) -> Dict[str, object]:
         finally:
             sys.stdout = save_stdout
             sys.stderr = save_stderr
-        return self.check()
+        return self.check(self.last_sources)
 
     def cmd_recheck(self) -> Dict[str, object]:
         """Check the same list of files we checked most recently."""
         if not self.last_sources:
             return {'error': "Command 'recheck' is only valid after a 'check' command"}
-        return self.check()
+        return self.check(self.last_sources)
 
-    saved_cache = {}  # type: mypy.build.SavedCache
-
-    def check(self) -> Dict[str, object]:
-        assert self.last_sources
+    def check(self, sources: List[mypy.build.BuildSource],
+              alt_lib_path: Optional[str] = None) -> Dict[str, object]:
         try:
             # saved_cache is mutated in place.
-            res = mypy.build.build(self.last_sources, self.options, saved_cache=self.saved_cache)
+            res = mypy.build.build(sources, self.options,
+                                   saved_cache=self.saved_cache,
+                                   alt_lib_path=alt_lib_path)
             msgs = res.errors
         except mypy.errors.CompileError as err:
             msgs = err.messages
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
new file mode 100644
index 0000000000000..d576d98a5dd27
--- /dev/null
+++ b/mypy/test/testdmypy.py
@@ -0,0 +1,298 @@
+"""Type checker test cases"""
+
+import os
+import re
+import shutil
+import sys
+import time
+import typed_ast
+
+from typing import Dict, List, Optional, Set, Tuple
+
+from mypy import build
+from mypy import defaults
+from mypy.main import process_options
+from mypy.myunit import AssertionFailure
+from mypy.test.config import test_temp_dir, test_data_prefix
+from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.helpers import (
+    assert_string_arrays_equal, normalize_error_messages,
+    retry_on_error, testcase_pyversion, update_testcase_output,
+)
+from mypy.errors import CompileError
+from mypy.options import Options
+
+from mypy import experiments
+from mypy import dmypy
+
+# List of files that contain test case descriptions.
+files = [
+    'check-incremental.test',
+    # TODO: Add other files that have (some) incremental test cases
+]
+
+
+class TypeCheckSuite(DataSuite):
+
+    @classmethod
+    def cases(cls) -> List[DataDrivenTestCase]:
+        c = []  # type: List[DataDrivenTestCase]
+        for f in files:
+            # TODO: Filter for name containing 'incremental'
+            c += parse_test_cases(os.path.join(test_data_prefix, f),
+                                  None, test_temp_dir, True)
+        return c
+
+    def run_case(self, testcase: DataDrivenTestCase) -> None:
+        incremental = ('incremental' in testcase.name.lower()
+                       or 'incremental' in testcase.file)
+        assert incremental, "cases() did not filter out non-incremental tests"
+        # TODO: Skip tests that vary flags across runs.
+        # Incremental tests are run once with a cold cache, then
+        # at least once with a warm cache and maybe changed files.
+        # Expected output is specified separately for each run.
+        self.clear_cache()
+        num_steps = max([2] + list(testcase.output2.keys()))
+        # Check that there are no file changes beyond the last run (they would be ignored).
+        for dn, dirs, files in os.walk(os.curdir):
+            for file in files:
+                m = re.search(r'\.([2-9])$', file)
+                if m and int(m.group(1)) > num_steps:
+                    raise ValueError(
+                        'Output file {} exists though test case only has {} runs'.format(
+                            file, num_steps))
+        self.server = None
+        for step in range(1, num_steps + 1):
+            self.run_case_once(testcase, step)
+
+    def clear_cache(self) -> None:
+        dn = defaults.CACHE_DIR
+        if os.path.exists(dn):
+            shutil.rmtree(dn)
+
+    def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) -> None:
+        assert incremental_step >= 1
+        build.find_module_clear_caches()
+        original_program_text = '\n'.join(testcase.input)
+        module_data = self.parse_module(original_program_text, incremental_step)
+
+        if incremental_step == 1:
+            # In run 1, copy program text to program file.
+            for module_name, program_path, program_text in module_data:
+                if module_name == '__main__':
+                    with open(program_path, 'w') as f:
+                        f.write(program_text)
+                    break
+        elif incremental_step > 1:
+            # In runs 2+, copy *.[num] files to * files.
+            for dn, dirs, files in os.walk(os.curdir):
+                for file in files:
+                    if file.endswith('.' + str(incremental_step)):
+                        full = os.path.join(dn, file)
+                        target = full[:-2]
+                        # Use retries to work around potential flakiness on Windows (AppVeyor).
+                        retry_on_error(lambda: shutil.copy(full, target))
+
+                        # In some systems, mtime has a resolution of 1 second which can cause
+                        # annoying-to-debug issues when a file has the same size after a
+                        # change. We manually set the mtime to circumvent this.
+                        new_time = os.stat(target).st_mtime + 1
+                        os.utime(target, times=(new_time, new_time))
+            # Delete files scheduled to be deleted in [delete <path>.num] sections.
+            for path in testcase.deleted_paths.get(incremental_step, set()):
+                # Use retries to work around potential flakiness on Windows (AppVeyor).
+                retry_on_error(lambda: os.remove(path))
+
+        # Parse options after moving files (in case mypy.ini is being moved).
+        # TODO: Guard against options changing for subsequent runs
+        options = self.parse_options(original_program_text, testcase, incremental_step)
+        if incremental_step == 1:
+            self.server = dmypy.Server([])  # TODO: Fix ugly API
+            self.server.options = options
+        else:
+            assert self.server.options == options
+
+        sources = []
+        for module_name, program_path, program_text in module_data:
+            # Always set to none so we're forced to reread the module in incremental mode
+            sources.append(build.BuildSource(program_path, module_name, None))
+        response = self.server.check(sources, alt_lib_path=test_temp_dir)
+        a = (response['out'] or response['err']).splitlines()
+        a = normalize_error_messages(a)
+
+        # Make sure error messages match
+        if incremental_step == 1:
+            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
+            output = testcase.output
+        elif incremental_step > 1:
+            msg = ('Unexpected type checker output in incremental, run {}'.format(
+                incremental_step) + ' ({}, line {})')
+            output = testcase.output2.get(incremental_step, [])
+        else:
+            raise AssertionError()
+
+        if output != a and self.update_data:
+            update_testcase_output(testcase, a)
+        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
+
+        # TODO: This requires access to the manager.
+        if False:
+            if options.follow_imports == 'normal' and testcase.output is None:
+                self.verify_cache(module_data, a, res.manager)
+            if incremental_step > 1:
+                suffix = '' if incremental_step == 2 else str(incremental_step - 1)
+                self.check_module_equivalence(
+                    'rechecked' + suffix,
+                    testcase.expected_rechecked_modules.get(incremental_step - 1),
+                    res.manager.rechecked_modules)
+                self.check_module_equivalence(
+                    'stale' + suffix,
+                    testcase.expected_stale_modules.get(incremental_step - 1),
+                    res.manager.stale_modules)
+
+    def check_module_equivalence(self, name: str,
+                                 expected: Optional[Set[str]], actual: Set[str]) -> None:
+        if expected is not None:
+            expected_normalized = sorted(expected)
+            actual_normalized = sorted(actual.difference({"__main__"}))
+            assert_string_arrays_equal(
+                expected_normalized,
+                actual_normalized,
+                ('Actual modules ({}) do not match expected modules ({}) '
+                 'for "[{} ...]"').format(
+                    ', '.join(actual_normalized),
+                    ', '.join(expected_normalized),
+                    name))
+
+    def verify_cache(self, module_data: List[Tuple[str, str, str]], a: List[str],
+                     manager: build.BuildManager) -> None:
+        # There should be valid cache metadata for each module except
+        # those in error_paths; for those there should not be.
+        #
+        # NOTE: When A imports B and there's an error in B, the cache
+        # data for B is invalidated, but the cache data for A remains.
+        # However build.process_graphs() will ignore A's cache data.
+        #
+        # Also note that when A imports B, and there's an error in A
+        # _due to a valid change in B_, the cache data for B will be
+        # invalidated and updated, but the old cache data for A will
+        # remain unchanged. As before, build.process_graphs() will
+        # ignore A's (old) cache data.
+        error_paths = self.find_error_paths(a)
+        modules = self.find_module_files()
+        modules.update({module_name: path for module_name, path, text in module_data})
+        missing_paths = self.find_missing_cache_files(modules, manager)
+        if not missing_paths.issubset(error_paths):
+            raise AssertionFailure("cache data discrepancy %s != %s" %
+                                   (missing_paths, error_paths))
+
+    def find_error_paths(self, a: List[str]) -> Set[str]:
+        hits = set()
+        for line in a:
+            m = re.match(r'([^\s:]+):\d+: error:', line)
+            if m:
+                # Normalize to Linux paths.
+                p = m.group(1).replace(os.path.sep, '/')
+                hits.add(p)
+        return hits
+
+    def find_module_files(self) -> Dict[str, str]:
+        modules = {}
+        for dn, dirs, files in os.walk(test_temp_dir):
+            dnparts = dn.split(os.sep)
+            assert dnparts[0] == test_temp_dir
+            del dnparts[0]
+            for file in files:
+                if file.endswith('.py'):
+                    if file == "__init__.py":
+                        # If the file path is `a/b/__init__.py`, exclude the file name
+                        # and make sure the module id is just `a.b`, not `a.b.__init__`.
+                        id = '.'.join(dnparts)
+                    else:
+                        base, ext = os.path.splitext(file)
+                        id = '.'.join(dnparts + [base])
+                    modules[id] = os.path.join(dn, file)
+        return modules
+
+    def find_missing_cache_files(self, modules: Dict[str, str],
+                                 manager: build.BuildManager) -> Set[str]:
+        ignore_errors = True
+        missing = {}
+        for id, path in modules.items():
+            meta = build.find_cache_meta(id, path, manager)
+            if not build.validate_meta(meta, id, path, ignore_errors, manager):
+                missing[id] = path
+        return set(missing.values())
+
+    def parse_module(self,
+                     program_text: str,
+                     incremental_step: int) -> List[Tuple[str, str, str]]:
+        """Return the module and program names for a test case.
+
+        Normally, the unit tests will parse the default ('__main__')
+        module and follow all the imports listed there. You can override
+        this behavior and instruct the tests to check multiple modules
+        by using a comment like this in the test case input:
+
+          # cmd: mypy -m foo.bar foo.baz
+
+        You can also use `# cmdN:` to have a different cmd for incremental
+        step N (2, 3, ...).
+
+        Return a list of tuples (module name, file name, program text).
+        """
+        m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE)
+        regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step)
+        alt_m = re.search(regex, program_text, flags=re.MULTILINE)
+        if alt_m is not None and incremental_step > 1:
+            # Optionally return a different command if in a later step
+            # of incremental mode, otherwise default to reusing the
+            # original cmd.
+            m = alt_m
+
+        if m:
+            # The test case wants to use a non-default main
+            # module. Look up the module and give it as the thing to
+            # analyze.
+            module_names = m.group(1)
+            out = []
+            for module_name in module_names.split(' '):
+                path = build.find_module(module_name, [test_temp_dir])
+                assert path is not None, "Can't find ad hoc case file"
+                with open(path) as f:
+                    program_text = f.read()
+                out.append((module_name, path, program_text))
+            return out
+        else:
+            return [('__main__', 'main', program_text)]
+
+    def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
+                      incremental_step: int) -> Options:
+        options = Options()
+        flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
+        if incremental_step > 1:
+            flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
+                               flags=re.MULTILINE)
+            if flags2:
+                flags = flags2
+
+        flag_list = None
+        if flags:
+            flag_list = flags.group(1).split()
+            targets, options = process_options(flag_list, require_targets=False)
+            if targets:
+                # TODO: support specifying targets via the flags pragma
+                raise RuntimeError('Specifying targets via the flags pragma is not supported.')
+        else:
+            options = Options()
+
+        # Allow custom python version to override testcase_pyversion
+        if (not flag_list or
+                all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
+            options.python_version = testcase_pyversion(testcase.file, testcase.name)
+
+        options.use_builtins_fixtures = True
+        options.show_traceback = True
+        options.incremental = True
+
+        return options

From 6739a0653f69835c1b4e2ce4877d094f450a3419 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 14:48:08 -0700
Subject: [PATCH 18/42] Skip tests that specify varying flags/options

---
 mypy/test/testdmypy.py                | 17 +++++++++++++----
 test-data/unit/check-incremental.test |  1 -
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index d576d98a5dd27..db68e1966b5b6 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -39,15 +39,16 @@ def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             # TODO: Filter for name containing 'incremental'
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
+            tc = parse_test_cases(os.path.join(test_data_prefix, f),
                                   None, test_temp_dir, True)
+            c += [case for case in tc if cls.has_stable_flags(case)]
         return c
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         incremental = ('incremental' in testcase.name.lower()
                        or 'incremental' in testcase.file)
-        assert incremental, "cases() did not filter out non-incremental tests"
-        # TODO: Skip tests that vary flags across runs.
+        assert incremental, "Testcase is not incremental"
+        assert self.has_stable_flags(testcase), "Testcase has varying flags"
         # Incremental tests are run once with a cold cache, then
         # at least once with a warm cache and maybe changed files.
         # Expected output is specified separately for each run.
@@ -65,6 +66,14 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         for step in range(1, num_steps + 1):
             self.run_case_once(testcase, step)
 
+    @classmethod
+    def has_stable_flags(cls, testcase: DataDrivenTestCase) -> bool:
+        if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
+            return False
+        if any(re.match(r'^\[file mypy.ini\.[2-9]\]', line) for line in testcase.input):
+            return False
+        return True
+
     def clear_cache(self) -> None:
         dn = defaults.CACHE_DIR
         if os.path.exists(dn):
@@ -104,12 +113,12 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
                 retry_on_error(lambda: os.remove(path))
 
         # Parse options after moving files (in case mypy.ini is being moved).
-        # TODO: Guard against options changing for subsequent runs
         options = self.parse_options(original_program_text, testcase, incremental_step)
         if incremental_step == 1:
             self.server = dmypy.Server([])  # TODO: Fix ugly API
             self.server.options = options
         else:
+            # Guard against options in mypy.ini changing for subsequent runs.
             assert self.server.options == options
 
         sources = []
diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test
index 5c952c86a92d5..525a4564fb253 100644
--- a/test-data/unit/check-incremental.test
+++ b/test-data/unit/check-incremental.test
@@ -3020,7 +3020,6 @@ tmp/mod.py:7: error: Revealed type is 'builtins.bytes'
 # cmd: mypy -m a
 # cmd2: mypy -m b
 # flags: --follow-imports=silent
-# flags2: --follow-imports=silent
 [file a.py]
 import b
 

From af6db88c2a6e5a205c7047f52f31ad4e350622d0 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 15:11:42 -0700
Subject: [PATCH 19/42] Include incremental tests from other files (not many)

---
 mypy/test/testdmypy.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index db68e1966b5b6..88d4e3194e778 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -27,8 +27,9 @@
 
 # List of files that contain test case descriptions.
 files = [
+    'check-enum.test',
     'check-incremental.test',
-    # TODO: Add other files that have (some) incremental test cases
+    'check-newtype.test',
 ]
 
 
@@ -38,20 +39,18 @@ class TypeCheckSuite(DataSuite):
     def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
-            # TODO: Filter for name containing 'incremental'
             tc = parse_test_cases(os.path.join(test_data_prefix, f),
                                   None, test_temp_dir, True)
-            c += [case for case in tc if cls.has_stable_flags(case)]
+            c += [case for case in tc
+                  if cls.has_stable_flags(case) and cls.is_incremental(case)]
         return c
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
-        incremental = ('incremental' in testcase.name.lower()
-                       or 'incremental' in testcase.file)
-        assert incremental, "Testcase is not incremental"
+        assert self.is_incremental(testcase), "Testcase is not incremental"
         assert self.has_stable_flags(testcase), "Testcase has varying flags"
-        # Incremental tests are run once with a cold cache, then
-        # at least once with a warm cache and maybe changed files.
-        # Expected output is specified separately for each run.
+        # All tests run once with a cold cache, then at least once
+        # with a warm cache and maybe changed files.  Expected output
+        # is specified separately for each run.
         self.clear_cache()
         num_steps = max([2] + list(testcase.output2.keys()))
         # Check that there are no file changes beyond the last run (they would be ignored).
@@ -66,6 +65,11 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         for step in range(1, num_steps + 1):
             self.run_case_once(testcase, step)
 
+    @classmethod
+    def is_incremental(cls, testcase: DataDrivenTestCase) -> bool:
+        return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
+        
+
     @classmethod
     def has_stable_flags(cls, testcase: DataDrivenTestCase) -> bool:
         if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
@@ -290,7 +294,6 @@ def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
             flag_list = flags.group(1).split()
             targets, options = process_options(flag_list, require_targets=False)
             if targets:
-                # TODO: support specifying targets via the flags pragma
                 raise RuntimeError('Specifying targets via the flags pragma is not supported.')
         else:
             options = Options()

From 7cd4e443a83433a9beb4670961972e8e576061dc Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 16:33:07 -0700
Subject: [PATCH 20/42] Fix 6 test errors caused by not deleting meta for files
 with errors

---
 mypy/build.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 2bad7bcb3dd6c..df5ca31dddfe1 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -1424,6 +1424,9 @@ class State:
     # Whether to ignore all errors
     ignore_all = False
 
+    # Whether this module was found to have errors
+    has_errors = False
+
     def __init__(self,
                  id: Optional[str],
                  path: Optional[str],
@@ -1599,6 +1602,7 @@ def mark_as_rechecked(self) -> None:
     def mark_interface_stale(self, *, on_errors: bool = False) -> None:
         """Marks this module as having a stale public interface."""
         self.externally_same = False
+        self.has_errors = on_errors
         if not on_errors:
             self.manager.stale_modules.add(self.id)
 
@@ -1908,7 +1912,6 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
     if manager.options.warn_unused_ignores:
         # TODO: This could also be a per-module option.
         manager.errors.generate_unused_ignore_notes()
-    # TODO: What about cache entries deleted because of errors?
     manager.saved_cache.update(preserve_cache(graph))
     return graph
 
@@ -1917,7 +1920,7 @@ def preserve_cache(graph: Graph) -> SavedCache:
     saved_cache = {}
     for id, state in graph.items():
         assert state.id == id
-        if state.meta is not None and state.tree is not None:
+        if state.meta is not None and state.tree is not None and not state.has_errors:
             saved_cache[id] = (state.meta, state.tree)
     return saved_cache
 

From cf0469e4b3c00f90dce14c6551aee14500cf12a6 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 16:53:11 -0700
Subject: [PATCH 21/42] Fix broken check for mypy.ini.* in has_stable_flags()

---
 mypy/test/testdmypy.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 88d4e3194e778..4f1b491164253 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -74,8 +74,9 @@ def is_incremental(cls, testcase: DataDrivenTestCase) -> bool:
     def has_stable_flags(cls, testcase: DataDrivenTestCase) -> bool:
         if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
             return False
-        if any(re.match(r'^\[file mypy.ini\.[2-9]\]', line) for line in testcase.input):
-            return False
+        for filename, contents in testcase.files:
+            if os.path.basename(filename).startswith('mypy.ini.'):
+                return False
         return True
 
     def clear_cache(self) -> None:

From 798b7e801fb856a521c70b1f530cdb387db2bf02 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Fri, 13 Oct 2017 16:55:07 -0700
Subject: [PATCH 22/42] Don't compare options; it doesn't work right

---
 mypy/test/testdmypy.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 4f1b491164253..8b9e304e19220 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -122,9 +122,6 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
         if incremental_step == 1:
             self.server = dmypy.Server([])  # TODO: Fix ugly API
             self.server.options = options
-        else:
-            # Guard against options in mypy.ini changing for subsequent runs.
-            assert self.server.options == options
 
         sources = []
         for module_name, program_path, program_text in module_data:

From d916f29a72445a73e6666183872d96a14e2683d4 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 15 Oct 2017 15:09:36 -0700
Subject: [PATCH 23/42] Revive cache verification after each test step

---
 mypy/dmypy.py          |  4 ++++
 mypy/test/testdmypy.py | 10 +++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 96bdd7e40055c..73d89bdd32861 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -394,6 +394,8 @@ def cmd_recheck(self) -> Dict[str, object]:
             return {'error': "Command 'recheck' is only valid after a 'check' command"}
         return self.check(self.last_sources)
 
+    last_mananager = None  # type: Optional[mypy.build.BuildManager]
+
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, object]:
         try:
@@ -402,8 +404,10 @@ def check(self, sources: List[mypy.build.BuildSource],
                                    saved_cache=self.saved_cache,
                                    alt_lib_path=alt_lib_path)
             msgs = res.errors
+            self.last_manager = res.manager
         except mypy.errors.CompileError as err:
             msgs = err.messages
+            self.last_manager = None
         if msgs:
             msgs.append("")
             return {'out': "\n".join(msgs), 'err': "", 'status': 1}
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 8b9e304e19220..e4b879638808d 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -146,20 +146,20 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
             update_testcase_output(testcase, a)
         assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
 
-        # TODO: This requires access to the manager.
-        if False:
+        manager = self.server.last_manager
+        if manager is not None:
             if options.follow_imports == 'normal' and testcase.output is None:
-                self.verify_cache(module_data, a, res.manager)
+                self.verify_cache(module_data, a, manager)
             if incremental_step > 1:
                 suffix = '' if incremental_step == 2 else str(incremental_step - 1)
                 self.check_module_equivalence(
                     'rechecked' + suffix,
                     testcase.expected_rechecked_modules.get(incremental_step - 1),
-                    res.manager.rechecked_modules)
+                    manager.rechecked_modules)
                 self.check_module_equivalence(
                     'stale' + suffix,
                     testcase.expected_stale_modules.get(incremental_step - 1),
-                    res.manager.stale_modules)
+                    manager.stale_modules)
 
     def check_module_equivalence(self, name: str,
                                  expected: Optional[Set[str]], actual: Set[str]) -> None:

From 69de33ab42a23603fed7061ed33de977f92a50b8 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 15 Oct 2017 15:16:41 -0700
Subject: [PATCH 24/42] Fix final two failing tests (it was wrong to delete the
 line 'self.meta = None')

---
 mypy/build.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mypy/build.py b/mypy/build.py
index df5ca31dddfe1..2227b57c3ee4d 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -1600,7 +1600,8 @@ def mark_as_rechecked(self) -> None:
         self.manager.rechecked_modules.add(self.id)
 
     def mark_interface_stale(self, *, on_errors: bool = False) -> None:
-        """Marks this module as having a stale public interface."""
+        """Marks this module as having a stale public interface, and discards the cache data."""
+        self.meta = None
         self.externally_same = False
         self.has_errors = on_errors
         if not on_errors:

From d4bcd071040941b81ec5176a842b838143ae90a3 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Sun, 15 Oct 2017 16:27:00 -0700
Subject: [PATCH 25/42] Fix mypy errors; clean up file/socket better

---
 mypy/dmypy.py          | 58 ++++++++++++++++++++++--------------------
 mypy/test/testdmypy.py |  2 +-
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 73d89bdd32861..ef64acde3766a 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -312,31 +312,37 @@ def __init__(self, flags: List[str]) -> None:
 
     def serve(self) -> None:
         """Serve a single request, synchronously (no thread or fork)."""
-        sock = self.create_listening_socket()
-        with open(STATUS_FILE, 'w') as f:
-            json.dump({'pid': os.getpid(), 'sockname': sock.getsockname()}, f)
-            f.write('\n')  # I like my JSON with trailing newline
-        while True:
-            conn, addr = sock.accept()
-            data = receive(conn)
-            resp = {}  # type: Dict[str, Any]
-            if 'command' not in data:
-                resp = {'error': "No command found in request"}
-            else:
-                command = data['command']
-                if not isinstance(command, str):
-                    resp = {'error': "Command is not a string"}
-                else:
-                    command = data.pop('command')
-                resp = self.run_command(command, data)
+        try:
+            sock = self.create_listening_socket()
             try:
-                conn.sendall(json.dumps(resp).encode('utf8'))
-            except OSError as err:
-                pass  # Maybe the client hung up
-            conn.close()
-            if command == 'stop':
-                sock.close()
-                sys.exit(0)
+                with open(STATUS_FILE, 'w') as f:
+                    json.dump({'pid': os.getpid(), 'sockname': sock.getsockname()}, f)
+                    f.write('\n')  # I like my JSON with trailing newline
+                while True:
+                    conn, addr = sock.accept()
+                    data = receive(conn)
+                    resp = {}  # type: Dict[str, Any]
+                    if 'command' not in data:
+                        resp = {'error': "No command found in request"}
+                    else:
+                        command = data['command']
+                        if not isinstance(command, str):
+                            resp = {'error': "Command is not a string"}
+                        else:
+                            command = data.pop('command')
+                        resp = self.run_command(command, data)
+                    try:
+                        conn.sendall(json.dumps(resp).encode('utf8'))
+                    except OSError as err:
+                        pass  # Maybe the client hung up
+                    conn.close()
+                    if command == 'stop':
+                        sock.close()
+                        sys.exit(0)
+            finally:
+                os.unlink(STATUS_FILE)
+        finally:
+            os.unlink(self.sockname)
 
     def create_listening_socket(self) -> socket.socket:
         """Create the socket and set it up for listening."""
@@ -365,8 +371,6 @@ def cmd_status(self) -> Dict[str, object]:
 
     def cmd_stop(self) -> Dict[str, object]:
         """Stop daemon."""
-        os.unlink(self.sockname)
-        os.unlink(STATUS_FILE)
         return {}
 
     last_sources = None
@@ -397,7 +401,7 @@ def cmd_recheck(self) -> Dict[str, object]:
     last_mananager = None  # type: Optional[mypy.build.BuildManager]
 
     def check(self, sources: List[mypy.build.BuildSource],
-              alt_lib_path: Optional[str] = None) -> Dict[str, object]:
+              alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
         try:
             # saved_cache is mutated in place.
             res = mypy.build.build(sources, self.options,
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index e4b879638808d..f6e531b724606 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -61,7 +61,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                     raise ValueError(
                         'Output file {} exists though test case only has {} runs'.format(
                             file, num_steps))
-        self.server = None
+        self.server = None  # type: Optional[dmypy.Server]
         for step in range(1, num_steps + 1):
             self.run_case_once(testcase, step)
 

From f8034d3733a63a070991dd94287a89e55e375081 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 12:40:46 -0700
Subject: [PATCH 26/42] New approach to instrumentation

---
 mypy/build.py | 41 +++++++++++++++++++++++++----------------
 mypy/dmypy.py | 19 +++++++++++++++++--
 2 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 2227b57c3ee4d..e00c0f8e21cf1 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -23,8 +23,8 @@
 from os.path import dirname, basename
 import errno
 
-from typing import (AbstractSet, Dict, Iterable, Iterator, List, cast, Any,
-                    NamedTuple, Optional, Set, Tuple, Union, Callable)
+from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List,
+                    Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable)
 # Can't use TYPE_CHECKING because it's not in the Python 3.5.1 stdlib
 MYPY = False
 if MYPY:
@@ -504,14 +504,9 @@ class BuildManager:
       plugin:          Active mypy plugin(s)
       errors:          Used for reporting all errors
       saved_cache:     Dict with saved cache state for dmypy (read-write!)
+      stats:           Dict with various instrumentation numbers
     """
 
-    # Counters for various cache-related events
-    fresh_metas = 0
-    fresh_trees = 0
-    reused_metas = 0
-    reused_trees = 0
-
     def __init__(self, data_dir: str,
                  lib_path: List[str],
                  ignore_prefix: str,
@@ -545,6 +540,7 @@ def __init__(self, data_dir: str,
         self.rechecked_modules = set()  # type: Set[str]
         self.plugin = plugin
         self.saved_cache = saved_cache if saved_cache is not None else {}  # type: SavedCache
+        self.stats = {}  # type: Dict[str, Any]  # Values are ints or floats
 
     def maybe_swap_for_shadow_path(self, path: str) -> str:
         if (self.options.shadow_file and
@@ -628,6 +624,7 @@ def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> My
         num_errs = self.errors.num_messages()
         tree = parse(source, path, self.errors, options=self.options)
         tree._fullname = id
+        self.add_stats(files_parsed=1, modules_parsed=int(not tree.is_stub), stubs_parsed=int(tree.is_stub))
 
         if self.errors.num_messages() != num_errs:
             self.log("Bailing due to parse errors")
@@ -673,6 +670,15 @@ def trace(self, *message: str) -> None:
             print('TRACE:', *message, file=sys.stderr)
             sys.stderr.flush()
 
+    def add_stats(self, **kwds: Any) -> None:
+        for key, value in kwds.items():
+            if key in self.stats:
+                self.stats[key] += value
+            else:
+                self.stats[key] = value
+
+    def stats_summary(self) -> Mapping[str, object]:
+        return self.stats
 
 def remove_cwd_prefix_from_path(p: str) -> str:
     """Remove current working directory prefix from p, if present.
@@ -922,7 +928,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
     saved_cache = manager.saved_cache
     if id in saved_cache:
         m, t = saved_cache[id]
-        manager.reused_metas += 1
+        manager.add_stats(reused_metas=1)
         manager.trace("Reusing saved metadata for %s" % id)
         # Note: it could still be skipped if the mtime/size/hash mismatches.
         return m
@@ -978,7 +984,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
                                   .format(key, cached_options.get(key), current_options.get(key)))
         return None
 
-    manager.fresh_metas += 1
+    manager.add_stats(fresh_metas=1)
     return m
 
 
@@ -1636,7 +1642,7 @@ def load_tree(self) -> None:
         # TODO: Assert data file wasn't changed.
         self.tree = MypyFile.deserialize(data)
         self.manager.modules[self.id] = self.tree
-        self.manager.fresh_trees += 1
+        self.manager.add_stats(fresh_trees=1)
 
     def fix_cross_refs(self) -> None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
@@ -1901,7 +1907,12 @@ def write_cache(self) -> None:
 def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
     manager.log()
     manager.log("Mypy version %s" % __version__)
+    t0 = time.time()
     graph = load_graph(sources, manager)
+    t1 = time.time()
+    manager.add_stats(graph_size=len(graph),
+                      stubs_found=sum(g.path.endswith('.pyi') for g in graph.values()),
+                      graph_load_time=(t1 - t0))
     if not graph:
         print("Nothing to do?!")
         return graph
@@ -2171,12 +2182,10 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg))
             process_stale_scc(graph, scc, manager)
 
-    manager.log("Stats: fresh_metas=%d, fresh_trees=%d, reused_metas=%d, reused_trees=%d" %
-                (manager.fresh_metas, manager.fresh_trees,
-                 manager.reused_metas, manager.reused_trees))
     sccs_left = len(fresh_scc_queue)
+    nodes_left = sum(len(scc) for scc in fresh_scc_queue)
+    manager.add_stats(sccs_left=sccs_left, nodes_left=nodes_left)
     if sccs_left:
-        nodes_left = sum(len(scc) for scc in fresh_scc_queue)
         manager.log("{} fresh SCCs ({} nodes) left in queue (and will remain unprocessed)"
                     .format(sccs_left, nodes_left))
         manager.trace(str(fresh_scc_queue))
@@ -2239,7 +2248,7 @@ def process_fresh_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
         trees = {id: saved_cache[id][1] for id in scc}
         if all(trees.values()):
             for id, tree in trees.items():
-                manager.reused_trees += 1
+                manager.add_stats(reused_trees=1)
                 manager.trace("Reusing saved tree %s" % id)
                 graph[id].tree = tree
                 manager.modules[id] = tree
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index ef64acde3766a..fbea45b9ff122 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -164,7 +164,10 @@ def do_restart(args: argparse.Namespace) -> None:
 @action(check_parser)
 def do_check(args: argparse.Namespace) -> None:
     """Ask the daemon to check a list of files."""
+    t0 = time.time()
     response = request('check', files=args.files)
+    t1 = time.time()
+    response['roundtrip_time'] = t1 - t0
     check_output(response)
 
 
@@ -174,7 +177,10 @@ def do_recheck(args: argparse.Namespace) -> None:
 
     This doesn't work across daemon restarts.
     """
+    t0 = time.time()
     response = request('recheck')
+    t1 = time.time()
+    response['roundtrip_time'] = t1 - t0
     check_output(response)
 
 
@@ -186,6 +192,9 @@ def check_output(response: Dict[str, Any]) -> None:
         sys.exit("Response: %s" % str(response))
     sys.stdout.write(out)
     sys.stderr.write(err)
+    for key, value in sorted(response.items()):
+        if key not in ('out', 'err'):
+            print("%20s: %10s" % (key, "%.3f" % value if isinstance(value, float) else value))
     sys.exit(status)
 
 
@@ -402,6 +411,7 @@ def cmd_recheck(self) -> Dict[str, object]:
 
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+        t0 = time.time()
         try:
             # saved_cache is mutated in place.
             res = mypy.build.build(sources, self.options,
@@ -412,11 +422,16 @@ def check(self, sources: List[mypy.build.BuildSource],
         except mypy.errors.CompileError as err:
             msgs = err.messages
             self.last_manager = None
+        t1 = time.time()
         if msgs:
             msgs.append("")
-            return {'out': "\n".join(msgs), 'err': "", 'status': 1}
+            response = {'out': "\n".join(msgs), 'err': "", 'status': 1}
         else:
-            return {'out': "", 'err': "", 'status': 0}
+            response = {'out': "", 'err': "", 'status': 0}
+        response['build_time'] = t1 - t0
+        if self.last_manager is not None:
+            response.update(self.last_manager.stats_summary())
+        return response
 
     def cmd_hang(self) -> Dict[str, object]:
         """Hang for 100 seconds, as a debug hack."""

From a4d5d79a412ad5214804430258325eaac029d269 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 13:39:52 -0700
Subject: [PATCH 27/42] Add GC statistics (and increase GC threshold!)

---
 mypy/build.py |  3 +++
 mypy/dmypy.py | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/mypy/build.py b/mypy/build.py
index e00c0f8e21cf1..c9868caf442bf 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -13,6 +13,7 @@
 import binascii
 import collections
 import contextlib
+import gc
 import hashlib
 import json
 import os.path
@@ -141,6 +142,8 @@ def build(sources: List[BuildSource],
       bin_dir: directory containing the mypy script, used for finding data
         directories; if omitted, use '.' as the data directory
     """
+    # This seems the most reasonable place to tune garbage collection.
+    gc.set_threshold(50000)
 
     data_dir = default_data_dir(bin_dir)
 
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index fbea45b9ff122..05fec8c8532bc 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -7,6 +7,7 @@
 """
 
 import argparse
+import gc
 import io
 import json
 import os
@@ -320,7 +321,7 @@ def __init__(self, flags: List[str]) -> None:
         self.options = options
 
     def serve(self) -> None:
-        """Serve a single request, synchronously (no thread or fork)."""
+        """Serve requests, synchronously (no thread or fork)."""
         try:
             sock = self.create_listening_socket()
             try:
@@ -411,8 +412,15 @@ def cmd_recheck(self) -> Dict[str, object]:
 
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+        bound_gc_callback = lambda phase, info: self.gc_callback(phase, info)
+        self.gc_start_time = None  # type: Optional[float]
+        self.gc_time = 0.0
+        self.gc_calls = 0
+        self.gc_collected = 0
+        self.gc_uncollectable = 0
         t0 = time.time()
         try:
+            gc.callbacks.append(bound_gc_callback)
             # saved_cache is mutated in place.
             res = mypy.build.build(sources, self.options,
                                    saved_cache=self.saved_cache,
@@ -422,6 +430,9 @@ def check(self, sources: List[mypy.build.BuildSource],
         except mypy.errors.CompileError as err:
             msgs = err.messages
             self.last_manager = None
+        finally:
+            while bound_gc_callback in gc.callbacks:
+                gc.callbacks.remove(bound_gc_callback)
         t1 = time.time()
         if msgs:
             msgs.append("")
@@ -429,6 +440,10 @@ def check(self, sources: List[mypy.build.BuildSource],
         else:
             response = {'out': "", 'err': "", 'status': 0}
         response['build_time'] = t1 - t0
+        response['gc_time'] = self.gc_time
+        response['gc_calls'] = self.gc_calls
+        response['gc_collected'] = self.gc_collected
+        response['gc_uncollectable'] = self.gc_uncollectable
         if self.last_manager is not None:
             response.update(self.last_manager.stats_summary())
         return response
@@ -438,6 +453,20 @@ def cmd_hang(self) -> Dict[str, object]:
         time.sleep(100)
         return {}
 
+    def gc_callback(self, phase: str, info: Mapping[str, int]) -> None:
+        if phase == 'start':
+            assert self.gc_start_time is None, "Start phase out of sequence"
+            self.gc_start_time = time.time()
+        elif phase == 'stop':
+            assert self.gc_start_time is not None, "Stop phase out of sequence"
+            self.gc_calls += 1
+            self.gc_time += time.time() - self.gc_start_time
+            self.gc_start_time = None
+            self.gc_collected += info['collected']
+            self.gc_uncollectable += info['uncollectable']
+        else:
+            assert False, "Unrecognized gc phase (%r)" % (phase,)
+
 
 # Network utilities.
 

From 657a55d41957481cba9d6aee56d8d9d3f23940dc Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 14:51:06 -0700
Subject: [PATCH 28/42] Report memory usage using getrusage() and psutil

---
 mypy/dmypy.py | 37 +++++++++++++++++++++++++++++++++----
 setup.py      |  8 ++++++--
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 05fec8c8532bc..43f5866921dd9 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -11,6 +11,7 @@
 import io
 import json
 import os
+import resource
 import signal
 import socket
 import sys
@@ -18,6 +19,8 @@
 
 from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, TypeVar
 
+import psutil  # type: ignore  # It's not in typeshed yet
+
 import mypy.build
 import mypy.errors
 import mypy.main
@@ -114,7 +117,7 @@ def do_status(args: argparse.Namespace) -> None:
     except Exception as err:
         print("Daemon is stuck; consider %s kill" % sys.argv[0])
     else:
-        print("Status response:", status)
+        show_stats(status)
 
 
 @action(stop_parser)
@@ -193,10 +196,14 @@ def check_output(response: Dict[str, Any]) -> None:
         sys.exit("Response: %s" % str(response))
     sys.stdout.write(out)
     sys.stderr.write(err)
+    show_stats(response)
+    sys.exit(status)
+
+
+def show_stats(response: Mapping[str, object]):
     for key, value in sorted(response.items()):
         if key not in ('out', 'err'):
             print("%20s: %10s" % (key, "%.3f" % value if isinstance(value, float) else value))
-    sys.exit(status)
 
 
 @action(hang_parser)
@@ -377,7 +384,9 @@ def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, obj
 
     def cmd_status(self) -> Dict[str, object]:
         """Return daemon status."""
-        return {'status': "I'm alive!"}
+        res = {}  # type: Dict[str, object]
+        res.update(get_meminfo())
+        return res
 
     def cmd_stop(self) -> Dict[str, object]:
         """Stop daemon."""
@@ -444,6 +453,7 @@ def check(self, sources: List[mypy.build.BuildSource],
         response['gc_calls'] = self.gc_calls
         response['gc_collected'] = self.gc_collected
         response['gc_uncollectable'] = self.gc_uncollectable
+        response.update(get_meminfo())
         if self.last_manager is not None:
             response.update(self.last_manager.stats_summary())
         return response
@@ -468,7 +478,7 @@ def gc_callback(self, phase: str, info: Mapping[str, int]) -> None:
             assert False, "Unrecognized gc phase (%r)" % (phase,)
 
 
-# Network utilities.
+# Misc utilities.
 
 def receive(sock: socket.socket) -> Any:
     """Receive data from a socket until EOF."""
@@ -483,6 +493,25 @@ def receive(sock: socket.socket) -> Any:
     return json.loads(bdata)
 
 
+MiB = 2**20
+
+
+def get_meminfo() -> Mapping[str, float]:
+    # See https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
+    res = {}
+    rusage = resource.getrusage(resource.RUSAGE_SELF)
+    if sys.platform == 'darwin':
+        factor = 1
+    else:
+        factor = 1024  # Linux
+    res['memory_maxrss_mib'] = rusage.ru_maxrss * factor / MiB
+    process = psutil.Process(os.getpid())
+    meminfo = process.memory_info()
+    res['memory_rss_mib'] = meminfo.rss / MiB
+    res['memory_vms_mib'] = meminfo.vms / MiB
+    return res
+
+
 # Run main().
 
 if __name__ == '__main__':
diff --git a/setup.py b/setup.py
index 64d5fac9d74ec..364f75321615d 100644
--- a/setup.py
+++ b/setup.py
@@ -103,11 +103,15 @@ def run(self):
       py_modules=[],
       packages=['mypy', 'mypy.test', 'mypy.myunit', 'mypy.server'],
       entry_points={'console_scripts': ['mypy=mypy.__main__:console_entry',
-                                        'stubgen=mypy.stubgen:main']},
+                                        'stubgen=mypy.stubgen:main',
+                                        'dmypy=mypy.dmypy:main',
+                                        ]},
       data_files=data_files,
       classifiers=classifiers,
       cmdclass={'build_py': CustomPythonBuild},
-      install_requires = ['typed-ast >= 1.1.0, < 1.2.0'],
+      install_requires = ['typed-ast >= 1.1.0, < 1.2.0',
+                          'psutil >= 5.4.0, < 5.5.0',
+                          ],
       extras_require = {
           ':python_version < "3.5"': 'typing >= 3.5.3',
       },

From b72c16d0489a3dd8b8ecd1a4d5e9c344c98fb155 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 15:02:47 -0700
Subject: [PATCH 29/42] Older Pythons don't load JSON from bytearray

---
 mypy/dmypy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 43f5866921dd9..901559f843567 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -490,7 +490,7 @@ def receive(sock: socket.socket) -> Any:
         bdata.extend(more)
     if not bdata:
         raise OSError("No data received")
-    return json.loads(bdata)
+    return json.loads(bdata.decode('utf8'))
 
 
 MiB = 2**20

From 9e0df243d982cf71d47c13d0b8fbf6a5c27bccad Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 15:16:07 -0700
Subject: [PATCH 30/42] Refactor status reading/reporting

---
 mypy/dmypy.py | 69 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 22 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 901559f843567..7f92868704b8c 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -109,27 +109,27 @@ def do_status(args: argparse.Namespace) -> None:
 
     This verifies that it is responsive to requests.
     """
-    pid, sockname = get_status()
-    print("pid: %d" % pid)
-    print("sockname: %s" % sockname)
+    status = read_status()
+    show_stats(status)
+    check_status(status)
     try:
-        status = request('status')
+        response = request('status')
     except Exception as err:
         print("Daemon is stuck; consider %s kill" % sys.argv[0])
     else:
-        show_stats(status)
+        show_stats(response)
 
 
 @action(stop_parser)
 def do_stop(args: argparse.Namespace) -> None:
     """Stop daemon politely (via a request)."""
     try:
-        status = request('stop')
+        response = request('stop')
     except Exception as err:
         sys.exit("Daemon is stuck; consider %s kill" % sys.argv[0])
     else:
-        if status:
-            print("Stop response:", status)
+        if response:
+            print("Stop response:", response)
         else:
             print("Daemon stopped")
 
@@ -154,12 +154,12 @@ def do_restart(args: argparse.Namespace) -> None:
     mypy flags (and has the same issues as start).
     """
     try:
-        status = request('stop')
+        response = request('stop')
     except SystemExit:
         pass
     else:
-        if status:
-            sys.exit("Status: %s" % str(status))
+        if response:
+            sys.exit("Status: %s" % str(response))
         else:
             print("Daemon stopped")
     daemonize(Server(args.flags).serve)
@@ -197,7 +197,8 @@ def check_output(response: Dict[str, Any]) -> None:
     sys.stdout.write(out)
     sys.stderr.write(err)
     show_stats(response)
-    sys.exit(status)
+    if status:
+        sys.exit(status)
 
 
 def show_stats(response: Mapping[str, object]):
@@ -257,24 +258,48 @@ def get_status() -> Tuple[int, str]:
 
     Raise SystemExit(<message>) if something's wrong.
     """
-    if not os.path.isfile(STATUS_FILE):
-        raise SystemExit("No status file found")
-    with open(STATUS_FILE) as f:
-        try:
-            data = json.load(f)
-        except Exception as err:
-            raise SystemExit("Malformed status file")
-    if not isinstance(data, dict) or 'pid' not in data:
-        raise SystemExit("Invalid status file")
+    data = read_status()
+    return check_status(data)
+
+
+def check_status(data: Dict[str, Any]) -> Tuple[int, str]:
+    """Check if the process is alive.
+
+    Return (pid, sockname) on success.
+
+    Raise SystemExit(<message>) if something's wrong.
+    """
+    if 'pid' not in data:
+        raise SystemExit("Invalid status file (no pid field)")
     pid = data['pid']
-    sockname = data['sockname']
+    if not isinstance(pid, int):
+        raise SystemExit("pid field is not an int")
     try:
         os.kill(pid, 0)
     except OSError as err:
         raise SystemExit("Daemon has died")
+    if 'sockname' not in data:
+        raise SystemExit("Invalid status file (no sockname field)")
+    sockname = data['sockname']
+    if not isinstance(sockname, str):
+        raise SystemExit("sockname field is not a string")
     return pid, sockname
 
 
+def read_status() -> Dict[str, object]:
+    """Read status file."""
+    if not os.path.isfile(STATUS_FILE):
+        raise SystemExit("No status file found")
+    with open(STATUS_FILE) as f:
+        try:
+            data = json.load(f)
+        except Exception as err:
+            raise SystemExit("Malformed status file (not JSON)")
+    if not isinstance(data, dict):
+        raise SystemExit("Invalid status file (not a dict)")
+    return data
+
+
 def daemonize(func: Callable[[], None]) -> None:
     """Arrange to call func() in a grandchild of the current process."""
     # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python

From fc663c8cef2412036957da9d9d865cd22d6dbaf4 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Mon, 16 Oct 2017 15:38:37 -0700
Subject: [PATCH 31/42] Don't update all_types unless in a test

---
 mypy/build.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index c9868caf442bf..a99ba6644d1d2 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -81,7 +81,7 @@ def __init__(self, manager: 'BuildManager', graph: Graph) -> None:
         self.manager = manager
         self.graph = graph
         self.files = manager.modules
-        self.types = manager.all_types
+        self.types = manager.all_types  # Non-empty for tests only
         self.errors = manager.errors.messages()
 
 
@@ -207,10 +207,9 @@ def build(sources: List[BuildSource],
         graph = dispatch(sources, manager)
         return BuildResult(manager, graph)
     finally:
-        manager.log("Build finished in %.3f seconds with %d modules, %d types, and %d errors" %
+        manager.log("Build finished in %.3f seconds with %d modules, and %d errors" %
                     (time.time() - manager.start_time,
                      len(manager.modules),
-                     len(manager.all_types),
                      manager.errors.num_messages()))
         # Finish the HTML or XML reports even if CompileError was raised.
         reports.finish()
@@ -482,8 +481,6 @@ def find_config_file_line_number(path: str, section: str, setting_name: str) ->
 SavedCache = Dict[str, Tuple[CacheMeta, MypyFile]]
 
 
-# TODO: Get rid of all_types.  It's not used except for one log message.
-#       Maybe we could instead publish a map from module ID to its type_map.
 class BuildManager:
     """This class holds shared state for building a mypy program.
 
@@ -499,7 +496,7 @@ class BuildManager:
                        Semantic analyzer, pass 2
       semantic_analyzer_pass3:
                        Semantic analyzer, pass 3
-      all_types:       Map {Expression: Type} collected from all modules
+      all_types:       Map {Expression: Type} collected from all modules (tests only)
       options:         Build options
       missing_modules: Set of modules that could not be imported encountered so far
       stale_modules:   Set of modules that needed to be rechecked (only used by tests)
@@ -537,7 +534,7 @@ def __init__(self, data_dir: str,
                                                   lib_path, self.errors, self.plugin)
         self.semantic_analyzer_pass3 = SemanticAnalyzerPass3(self.modules, self.errors,
                                                              self.semantic_analyzer)
-        self.all_types = {}  # type: Dict[Expression, Type]
+        self.all_types = {}  # type: Dict[Expression, Type]  # Used by tests only
         self.indirection_detector = TypeIndirectionVisitor()
         self.stale_modules = set()  # type: Set[str]
         self.rechecked_modules = set()  # type: Set[str]
@@ -1841,7 +1838,9 @@ def finish_passes(self) -> None:
         if self.options.semantic_analysis_only:
             return
         with self.wrap_context():
-            manager.all_types.update(self.type_checker.type_map)
+            # Some tests want to look at the set of all types.
+            if manager.options.use_builtins_fixtures:
+                manager.all_types.update(self.type_checker.type_map)
 
             if self.options.incremental:
                 self._patch_indirect_dependencies(self.type_checker.module_refs,
@@ -1914,7 +1913,8 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
     graph = load_graph(sources, manager)
     t1 = time.time()
     manager.add_stats(graph_size=len(graph),
-                      stubs_found=sum(g.path.endswith('.pyi') for g in graph.values()),
+                      stubs_found=sum(g.path is not None and g.path.endswith('.pyi')
+                                      for g in graph.values()),
                       graph_load_time=(t1 - t0))
     if not graph:
         print("Nothing to do?!")

From 46ffbc7be425313b91ea2fe3027483e102576470 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 13:49:04 -0700
Subject: [PATCH 32/42] Add -q/--quiet flag to suppress printing stats

---
 mypy/dmypy.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 7f92868704b8c..92ed9ca7d073b 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -49,10 +49,14 @@
                             help="Regular mypy flags (precede with --)")
 
 check_parser = subparsers.add_parser('check', help="Check some files (requires running daemon)")
+check_parser.add_argument('-q', '--quiet', action='store_true',
+                          help="Suppress instrumentation stats")
 check_parser.add_argument('files', metavar='FILE', nargs='+', help="File (or directory) to check")
 
 recheck_parser = subparsers.add_parser('recheck',
     help="Check the same files as the most previous  check run (requires running daemon)")
+recheck_parser.add_argument('-q', '--quiet', action='store_true',
+                            help="Suppress instrumentation stats")
 
 hang_parser = subparsers.add_parser('hang', help="Hang for 100 seconds")
 
@@ -172,7 +176,7 @@ def do_check(args: argparse.Namespace) -> None:
     response = request('check', files=args.files)
     t1 = time.time()
     response['roundtrip_time'] = t1 - t0
-    check_output(response)
+    check_output(response, args.quiet)
 
 
 @action(recheck_parser)
@@ -185,10 +189,10 @@ def do_recheck(args: argparse.Namespace) -> None:
     response = request('recheck')
     t1 = time.time()
     response['roundtrip_time'] = t1 - t0
-    check_output(response)
+    check_output(response, args.quiet)
 
 
-def check_output(response: Dict[str, Any]) -> None:
+def check_output(response: Dict[str, Any], quiet: bool) -> None:
     """Print the output from a check or recheck command."""
     try:
         out, err, status = response['out'], response['err'], response['status']
@@ -196,7 +200,8 @@ def check_output(response: Dict[str, Any]) -> None:
         sys.exit("Response: %s" % str(response))
     sys.stdout.write(out)
     sys.stderr.write(err)
-    show_stats(response)
+    if not quiet:
+        show_stats(response)
     if status:
         sys.exit(status)
 

From 81da5b14480be23b723acd536f8fa5aa24482e40 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 14:12:31 -0700
Subject: [PATCH 33/42] When starting daemon, wait for child process and report
 status

---
 mypy/dmypy.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 92ed9ca7d073b..64392f7da371e 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -102,7 +102,8 @@ def do_start(args: argparse.Namespace) -> None:
     try:
         pid, sockname = get_status()
     except SystemExit as err:
-        daemonize(Server(args.flags).serve)
+        if daemonize(Server(args.flags).serve):
+            sys.exit(1)
     else:
         sys.exit("Daemon is still alive")
 
@@ -166,7 +167,8 @@ def do_restart(args: argparse.Namespace) -> None:
             sys.exit("Status: %s" % str(response))
         else:
             print("Daemon stopped")
-    daemonize(Server(args.flags).serve)
+    if daemonize(Server(args.flags).serve):
+        sys.exit(1)
 
 
 @action(check_parser)
@@ -305,15 +307,29 @@ def read_status() -> Dict[str, object]:
     return data
 
 
-def daemonize(func: Callable[[], None]) -> None:
-    """Arrange to call func() in a grandchild of the current process."""
+def daemonize(func: Callable[[], None]) -> int:
+    """Arrange to call func() in a grandchild of the current process.
+
+    Return 0 for success, proposed exit status for failure.
+    """
     # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python
     sys.stdout.flush()
     sys.stderr.flush()
     pid = os.fork()
     if pid:
-        print("Daemon started")
-        return
+        npid, sts = os.waitpid(pid, 0)
+        if sts == 0:
+            # TODO: Wait until STATUS_FILE appears and is readable.
+            print("Daemon started")
+            return 0
+        else:
+            sig = sts & 0xff
+            if sig:
+                print("Child killed by signal", sig)
+                return -sig
+            sts = sts >> 8
+            print("Child exit status", sts)
+            return sts
     # Child
     try:
         os.setsid()  # Detach controlling terminal

From 80e60b0b113e1129898d486c656b4ff5c8eee57b Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 17:13:06 -0700
Subject: [PATCH 34/42] Add support for daemon mode to
 misc/incremental_checker.py

---
 misc/incremental_checker.py | 59 +++++++++++++++++++++++++++++--------
 1 file changed, 46 insertions(+), 13 deletions(-)

diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py
index 4bd30551a5cb3..c3b009cb4032c 100755
--- a/misc/incremental_checker.py
+++ b/misc/incremental_checker.py
@@ -49,6 +49,7 @@
 CACHE_PATH = ".incremental_checker_cache.json"
 MYPY_REPO_URL = "https://github.com/python/mypy.git"
 MYPY_TARGET_FILE = "mypy"
+DAEMON_CMD = ["python3", "-m", "mypy.dmypy"]
 
 JsonDict = Dict[str, Any]
 
@@ -121,23 +122,30 @@ def get_nth_commit(repo_folder_path, n: int) -> Tuple[str, str]:
 def run_mypy(target_file_path: Optional[str],
              mypy_cache_path: str,
              mypy_script: Optional[str],
-             incremental: bool = True,
+             *,
+             incremental: bool = False,
+             daemon: bool = False,
              verbose: bool = False) -> Tuple[float, str]:
     """Runs mypy against `target_file_path` and returns what mypy prints to stdout as a string.
 
     If `incremental` is set to True, this function will use store and retrieve all caching data
     inside `mypy_cache_path`. If `verbose` is set to True, this function will pass the "-v -v"
     flags to mypy to make it output debugging information.
+
+    If `daemon` is True, we use daemon mode; the daemon must be started and stopped by the caller.
     """
-    if mypy_script is None:
-        command = ["python3", "-m", "mypy"]
+    if daemon:
+        command = DAEMON_CMD + ["check", "-q"]
     else:
-        command = [mypy_script]
-    command.extend(["--cache-dir", mypy_cache_path])
-    if incremental:
-        command.append("--incremental")
-    if verbose:
-        command.extend(["-v", "-v"])
+        if mypy_script is None:
+            command = ["python3", "-m", "mypy"]
+        else:
+            command = [mypy_script]
+        command.extend(["--cache-dir", mypy_cache_path])
+        if incremental:
+            command.append("--incremental")
+        if verbose:
+            command.extend(["-v", "-v"])
     if target_file_path is not None:
         command.append(target_file_path)
     start = time.time()
@@ -148,6 +156,23 @@ def run_mypy(target_file_path: Optional[str],
     return runtime, output
 
 
+def start_daemon(mypy_cache_path: str, verbose: bool) -> None:
+    stdout, stderr, status = execute(DAEMON_CMD + ["status"], fail_on_error=False)
+    if status:
+        cmd = DAEMON_CMD + ["start", "--", "--cache-dir", mypy_cache_path]
+        if verbose:
+            cmd.extend(["-v", "-v"])
+        execute(cmd)
+        time.sleep(1)
+        execute(DAEMON_CMD + ["status"])
+
+
+def stop_daemon() -> None:
+    stdout, stderr, status = execute(DAEMON_CMD + ["status"], fail_on_error=False)
+    if status == 0:
+        execute(DAEMON_CMD + ["stop"])
+
+
 def load_cache(incremental_cache_path: str = CACHE_PATH) -> JsonDict:
     if os.path.exists(incremental_cache_path):
         with open(incremental_cache_path, 'r') as stream:
@@ -196,7 +221,9 @@ def test_incremental(commits: List[Tuple[str, str]],
                      temp_repo_path: str,
                      target_file_path: Optional[str],
                      mypy_cache_path: str,
-                     mypy_script: Optional[str]) -> None:
+                     *,
+                     mypy_script: Optional[str] = None,
+                     daemon: bool = False) -> None:
     """Runs incremental mode on all `commits` to verify the output matches the expected output.
 
     This function runs mypy on the `target_file_path` inside the `temp_repo_path`. The
@@ -208,7 +235,7 @@ def test_incremental(commits: List[Tuple[str, str]],
         print('Now testing commit {0}: "{1}"'.format(commit_id, message))
         execute(["git", "-C", temp_repo_path, "checkout", commit_id])
         runtime, output = run_mypy(target_file_path, mypy_cache_path, mypy_script,
-                                   incremental=True)
+                                   incremental=True, daemon=daemon)
         expected_runtime = cache[commit_id]['runtime']  # type: float
         expected_output = cache[commit_id]['output']  # type: str
         if output != expected_output:
@@ -278,11 +305,15 @@ def test_repo(target_repo_url: str, temp_repo_path: str,
     save_cache(cache, incremental_cache_path)
 
     # Stage 4: Rewind and re-run mypy (with incremental mode enabled)
+    if params.daemon:
+        start_daemon(mypy_cache_path, False)
     test_incremental(commits, cache, temp_repo_path, target_file_path, mypy_cache_path,
-                     mypy_script=params.mypy_script)
+                     mypy_script=params.mypy_script, daemon=params.daemon)
 
-    # Stage 5: Remove temp files
+    # Stage 5: Remove temp files, stop daemon
     cleanup(temp_repo_path, mypy_cache_path)
+    if params.daemon:
+        stop_daemon()
 
 
 def main() -> None:
@@ -309,6 +340,8 @@ def main() -> None:
     parser.add_argument("--sample", type=int, help="use a random sample of size SAMPLE")
     parser.add_argument("--seed", type=str, help="random seed")
     parser.add_argument("--mypy-script", type=str, help="alternate mypy script to run")
+    parser.add_argument("--daemon", action='store_true',
+                        help="use mypy daemon instead of incremental (highly experimental)")
 
     if len(sys.argv[1:]) == 0:
         parser.print_help()

From 2caaf0b5c0927d966f6335beea51f0ae2cb8d9ff Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 18:57:58 -0700
Subject: [PATCH 35/42] When starting daemon wait for it to be serving

---
 misc/incremental_checker.py |  2 --
 mypy/dmypy.py               | 45 +++++++++++++++++++++++++++----------
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py
index c3b009cb4032c..9d0c7209c48bd 100755
--- a/misc/incremental_checker.py
+++ b/misc/incremental_checker.py
@@ -163,8 +163,6 @@ def start_daemon(mypy_cache_path: str, verbose: bool) -> None:
         if verbose:
             cmd.extend(["-v", "-v"])
         execute(cmd)
-        time.sleep(1)
-        execute(DAEMON_CMD + ["status"])
 
 
 def stop_daemon() -> None:
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 64392f7da371e..88545cc046344 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -104,6 +104,7 @@ def do_start(args: argparse.Namespace) -> None:
     except SystemExit as err:
         if daemonize(Server(args.flags).serve):
             sys.exit(1)
+        wait_for_server()
     else:
         sys.exit("Daemon is still alive")
 
@@ -169,6 +170,27 @@ def do_restart(args: argparse.Namespace) -> None:
             print("Daemon stopped")
     if daemonize(Server(args.flags).serve):
         sys.exit(1)
+    wait_for_server()
+
+
+def wait_for_server(timeout: float = 5.0) -> None:
+    """Wait until the server is up.
+
+    Exit if it doesn't happen within the timeout.
+    """
+    endtime = time.time() + timeout
+    while time.time() < endtime:
+        try:
+            data = read_status()
+        except SystemExit:
+            # If the file isn't there yet, retry later.
+            time.sleep(0.1)
+            continue
+        # If the file's content is bogus or the process is dead, fail.
+        pid, sockname = check_status(data)
+        print("Daemon started")
+        return
+    sys.exit("Timed out waiting for daemon to start")
 
 
 @action(check_parser)
@@ -310,7 +332,8 @@ def read_status() -> Dict[str, object]:
 def daemonize(func: Callable[[], None]) -> int:
     """Arrange to call func() in a grandchild of the current process.
 
-    Return 0 for success, proposed exit status for failure.
+    Return 0 for success, exit status for failure, negative if
+    subprocess killed by signal.
     """
     # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python
     sys.stdout.flush()
@@ -318,18 +341,14 @@ def daemonize(func: Callable[[], None]) -> int:
     pid = os.fork()
     if pid:
         npid, sts = os.waitpid(pid, 0)
-        if sts == 0:
-            # TODO: Wait until STATUS_FILE appears and is readable.
-            print("Daemon started")
-            return 0
-        else:
-            sig = sts & 0xff
-            if sig:
-                print("Child killed by signal", sig)
-                return -sig
-            sts = sts >> 8
+        sig = sts & 0xff
+        if sig:
+            print("Child killed by signal", sig)
+            return -sig
+        sts = sts >> 8
+        if sts:
             print("Child exit status", sts)
-            return sts
+        return sts
     # Child
     try:
         os.setsid()  # Detach controlling terminal
@@ -372,6 +391,8 @@ def __init__(self, flags: List[str]) -> None:
         if options.quick_and_dirty:
             sys.exit("dmypy: start/restart should not specify quick_and_dirty mode")
         self.options = options
+        if os.path.isfile(STATUS_FILE):
+            os.unlink(STATUS_FILE)
 
     def serve(self) -> None:
         """Serve requests, synchronously (no thread or fork)."""

From 808ac7fed3a4319d19a51fb0675155f7f378a03c Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 21:15:00 -0700
Subject: [PATCH 36/42] Add testdmypy to PYTEST_FILES (fixes runtests.py)

---
 runtests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtests.py b/runtests.py
index 2f1bc7aa94cce..d4712bbfbabb8 100755
--- a/runtests.py
+++ b/runtests.py
@@ -204,6 +204,7 @@ def test_path(*names: str):
 
 PYTEST_FILES = test_path(
     'testcheck',
+    'testdmypy',
     'testextensions',
     'testdeps',
     'testdiff',

From e6b46eabe5ba2f1b38c25dd41a2d2b787953ea72 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 21:19:18 -0700
Subject: [PATCH 37/42] Skip dmypy tests on Windows

---
 mypy/test/testdmypy.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index f6e531b724606..349e68663f1d5 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -37,6 +37,8 @@ class TypeCheckSuite(DataSuite):
 
     @classmethod
     def cases(cls) -> List[DataDrivenTestCase]:
+        if sys.platform == 'win32':
+            return []  # Nothing here works on Windows.
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             tc = parse_test_cases(os.path.join(test_data_prefix, f),

From bc151063b845755e71d6ca71cac32fc4c7a16503 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Tue, 17 Oct 2017 21:27:31 -0700
Subject: [PATCH 38/42] Move 'import resource' into the function that uses it

---
 mypy/dmypy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index 88545cc046344..b354956bfc04f 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -11,7 +11,6 @@
 import io
 import json
 import os
-import resource
 import signal
 import socket
 import sys
@@ -565,6 +564,7 @@ def receive(sock: socket.socket) -> Any:
 
 def get_meminfo() -> Mapping[str, float]:
     # See https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
+    import resource  # Since it doesn't exist on Windows.
     res = {}
     rusage = resource.getrusage(resource.RUSAGE_SELF)
     if sys.platform == 'darwin':

From c9bb0d1c6dbe0fe9217f5b9807c377b8ea457e86 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Thu, 19 Oct 2017 09:20:06 -0700
Subject: [PATCH 39/42] Respond to code review (in some cases by adding a TODO)

---
 mypy/build.py | 11 +++++++----
 mypy/dmypy.py | 23 +++++++++++++++++------
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index a99ba6644d1d2..8fe505e10e79b 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -120,11 +120,16 @@ def is_source(self, file: MypyFile) -> bool:
             return False
 
 
+# A dict containing saved cache data from a previous run.  This will
+# be updated in place with newly computed cache data.  See dmypy.py.
+SavedCache = Dict[str, Tuple['CacheMeta', MypyFile]]
+
+
 def build(sources: List[BuildSource],
           options: Options,
           alt_lib_path: Optional[str] = None,
           bin_dir: Optional[str] = None,
-          saved_cache: Optional['SavedCache'] = None,
+          saved_cache: Optional[SavedCache] = None,
           ) -> BuildResult:
     """Analyze a program.
 
@@ -141,6 +146,7 @@ def build(sources: List[BuildSource],
         (takes precedence over other directories)
       bin_dir: directory containing the mypy script, used for finding data
         directories; if omitted, use '.' as the data directory
+      saved_cache: optional dict with saved cache state for dmypy (read-write!)
     """
     # This seems the most reasonable place to tune garbage collection.
     gc.set_threshold(50000)
@@ -478,9 +484,6 @@ def find_config_file_line_number(path: str, section: str, setting_name: str) ->
     return -1
 
 
-SavedCache = Dict[str, Tuple[CacheMeta, MypyFile]]
-
-
 class BuildManager:
     """This class holds shared state for building a mypy program.
 
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index b354956bfc04f..cfd0a5b1c04b3 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -20,6 +20,7 @@
 
 import psutil  # type: ignore  # It's not in typeshed yet
 
+# TODO: Import all mypy modules lazily to speed up client startup time.
 import mypy.build
 import mypy.errors
 import mypy.main
@@ -27,7 +28,7 @@
 # Argument parser.  Subparsers are tied to action functions by the
 # @action(subparse) decorator.
 
-parser = argparse.ArgumentParser(description="Client for mymy daemon mode",
+parser = argparse.ArgumentParser(description="Client for mypy daemon mode",
                                  fromfile_prefix_chars='@')
 parser.set_defaults(action=None)
 subparsers = parser.add_subparsers()
@@ -86,6 +87,7 @@ def register(func: ActionFunction) -> None:
 
 
 # Action functions (run in client from command line).
+# TODO: Use a separate exception instead of SystemExit to indicate failures.
 
 @action(start_parser)
 def do_start(args: argparse.Namespace) -> None:
@@ -259,7 +261,10 @@ def do_help(args: argparse.Namespace) -> None:
 
 
 def request(command: str, **kwds: object) -> Dict[str, Any]:
-    """Send a request to the daemon."""
+    """Send a request to the daemon.
+
+    Return the JSON dict with the response.
+    """
     args = dict(kwds)
     if command:
         args.update(command=command)
@@ -339,6 +344,7 @@ def daemonize(func: Callable[[], None]) -> int:
     sys.stderr.flush()
     pid = os.fork()
     if pid:
+        # Parent process: wait for child in case things go bad there.
         npid, sts = os.waitpid(pid, 0)
         sig = sts & 0xff
         if sig:
@@ -348,7 +354,7 @@ def daemonize(func: Callable[[], None]) -> int:
         if sts:
             print("Child exit status", sts)
         return sts
-    # Child
+    # Child process: do a bunch of UNIX stuff and then fork a grandchild.
     try:
         os.setsid()  # Detach controlling terminal
         os.umask(0o27)
@@ -359,8 +365,9 @@ def daemonize(func: Callable[[], None]) -> int:
         os.close(devnull)
         pid = os.fork()
         if pid:
+            # Child is done, exit to parent.
             os._exit(0)
-        # Grandchild
+        # Grandchild: run the server.
         func()
     finally:
         # Make sure we never get back into the caller.
@@ -487,6 +494,7 @@ def cmd_recheck(self) -> Dict[str, object]:
 
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
+        # TODO: Move stats handling code to make the logic here less cluttered.
         bound_gc_callback = lambda phase, info: self.gc_callback(phase, info)
         self.gc_start_time = None  # type: Optional[float]
         self.gc_time = 0.0
@@ -547,7 +555,7 @@ def gc_callback(self, phase: str, info: Mapping[str, int]) -> None:
 # Misc utilities.
 
 def receive(sock: socket.socket) -> Any:
-    """Receive data from a socket until EOF."""
+    """Receive JSON data from a socket until EOF."""
     bdata = bytearray()
     while True:
         more = sock.recv(100000)
@@ -556,7 +564,10 @@ def receive(sock: socket.socket) -> Any:
         bdata.extend(more)
     if not bdata:
         raise OSError("No data received")
-    return json.loads(bdata.decode('utf8'))
+    data = json.loads(bdata.decode('utf8'))
+    if not isinstance(data, dict):
+        raise OSError("Data received is not a dict (%s)" % str(type(data)))
+    return data
 
 
 MiB = 2**20

From 517129d93899c0508b94e4d02434733bffd0f033 Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Thu, 19 Oct 2017 09:49:12 -0700
Subject: [PATCH 40/42] Fix stricter mypy errors

---
 mypy/dmypy.py          | 4 ++--
 mypy/test/testdmypy.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index cfd0a5b1c04b3..d9615ab995356 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -231,7 +231,7 @@ def check_output(response: Dict[str, Any], quiet: bool) -> None:
         sys.exit(status)
 
 
-def show_stats(response: Mapping[str, object]):
+def show_stats(response: Mapping[str, object]) -> None:
     for key, value in sorted(response.items()):
         if key not in ('out', 'err'):
             print("%20s: %10s" % (key, "%.3f" % value if isinstance(value, float) else value))
@@ -509,7 +509,7 @@ def check(self, sources: List[mypy.build.BuildSource],
                                    saved_cache=self.saved_cache,
                                    alt_lib_path=alt_lib_path)
             msgs = res.errors
-            self.last_manager = res.manager
+            self.last_manager = res.manager  # type: Optional[mypy.build.BuildManager]
         except mypy.errors.CompileError as err:
             msgs = err.messages
             self.last_manager = None
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 349e68663f1d5..8b13a7af7d7eb 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -125,6 +125,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) ->
             self.server = dmypy.Server([])  # TODO: Fix ugly API
             self.server.options = options
 
+        assert self.server is not None  # Set in step 1 and survives into next steps
         sources = []
         for module_name, program_path, program_text in module_data:
             # Always set to none so we're forced to reread the module in incremental mode

From ba827e2f369a0cc4f3cc9317c09b7b906c88394c Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Thu, 19 Oct 2017 09:53:41 -0700
Subject: [PATCH 41/42] Fix lint errors

---
 mypy/build.py          | 5 ++++-
 mypy/dmypy.py          | 3 ++-
 mypy/test/testdmypy.py | 1 -
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/mypy/build.py b/mypy/build.py
index 8fe505e10e79b..beb30544ad190 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -627,7 +627,9 @@ def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> My
         num_errs = self.errors.num_messages()
         tree = parse(source, path, self.errors, options=self.options)
         tree._fullname = id
-        self.add_stats(files_parsed=1, modules_parsed=int(not tree.is_stub), stubs_parsed=int(tree.is_stub))
+        self.add_stats(files_parsed=1,
+                       modules_parsed=int(not tree.is_stub),
+                       stubs_parsed=int(tree.is_stub))
 
         if self.errors.num_messages() != num_errs:
             self.log("Bailing due to parse errors")
@@ -683,6 +685,7 @@ def add_stats(self, **kwds: Any) -> None:
     def stats_summary(self) -> Mapping[str, object]:
         return self.stats
 
+
 def remove_cwd_prefix_from_path(p: str) -> str:
     """Remove current working directory prefix from p, if present.
 
diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index d9615ab995356..ff2c9259e97c3 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -495,7 +495,8 @@ def cmd_recheck(self) -> Dict[str, object]:
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
         # TODO: Move stats handling code to make the logic here less cluttered.
-        bound_gc_callback = lambda phase, info: self.gc_callback(phase, info)
+        def bound_gc_callback(phase, info):
+            return self.gc_callback(phase, info)
         self.gc_start_time = None  # type: Optional[float]
         self.gc_time = 0.0
         self.gc_calls = 0
diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py
index 8b13a7af7d7eb..1f483e1bc3154 100644
--- a/mypy/test/testdmypy.py
+++ b/mypy/test/testdmypy.py
@@ -70,7 +70,6 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
     @classmethod
     def is_incremental(cls, testcase: DataDrivenTestCase) -> bool:
         return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
-        
 
     @classmethod
     def has_stable_flags(cls, testcase: DataDrivenTestCase) -> bool:

From 626566b6f11ccdc110207d4430485d0346a3457c Mon Sep 17 00:00:00 2001
From: Guido van Rossum <guido@dropbox.com>
Date: Thu, 19 Oct 2017 16:28:32 -0700
Subject: [PATCH 42/42] Simplify bound_gc_callback

---
 mypy/dmypy.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mypy/dmypy.py b/mypy/dmypy.py
index ff2c9259e97c3..87d9e85f8a009 100644
--- a/mypy/dmypy.py
+++ b/mypy/dmypy.py
@@ -495,8 +495,7 @@ def cmd_recheck(self) -> Dict[str, object]:
     def check(self, sources: List[mypy.build.BuildSource],
               alt_lib_path: Optional[str] = None) -> Dict[str, Any]:
         # TODO: Move stats handling code to make the logic here less cluttered.
-        def bound_gc_callback(phase, info):
-            return self.gc_callback(phase, info)
+        bound_gc_callback = self.gc_callback
         self.gc_start_time = None  # type: Optional[float]
         self.gc_time = 0.0
         self.gc_calls = 0