From ff21f6565b530f2dbeeb1ae9070c6b110d356aef Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 10 Apr 2018 06:39:54 +0200 Subject: [PATCH 01/39] Support for PY37 without contexvars This MR aligns the uvloop code to become compatible, not production ready, with PY37 adding the following changes: - Support for the new API used behind the `set_debug` function. - Adds the `context=None` kw to those functions that will receive it when uvloop runs with PY37. - Former asyncio.test_utils moved as an uvloop test resource. --- .travis.yml | 7 +++ tests/test_pipes.py | 2 +- tests/test_process.py | 2 +- tests/test_tasks.py | 7 +-- tests/test_udp.py | 2 +- tests/test_unix.py | 2 + tests/utils.py | 88 ++++++++++++++++++++++++++++++++++++++ uvloop/includes/stdlib.pxi | 5 +-- uvloop/loop.pxd | 7 ++- uvloop/loop.pyx | 69 ++++++++++++++++++------------ 10 files changed, 152 insertions(+), 39 deletions(-) create mode 100644 tests/utils.py diff --git a/.travis.yml b/.travis.yml index b7ba7e7f..23d4cbb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,6 +56,13 @@ matrix: python: "3.6" env: BUILD=tests + - os: linux + dist: trusty + sudo: false + language: python + python: "3.7-dev" + env: BUILD=tests + - os: linux dist: trusty branches: {only: [releases]} diff --git a/tests/test_pipes.py b/tests/test_pipes.py index 8bec52b3..97b3c03e 100644 --- a/tests/test_pipes.py +++ b/tests/test_pipes.py @@ -2,7 +2,7 @@ import io import os -from asyncio import test_utils +from tests import utils as test_utils from uvloop import _testbase as tb diff --git a/tests/test_process.py b/tests/test_process.py index 51e1970a..2db2cb43 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -9,7 +9,7 @@ import time import unittest -from asyncio import test_utils +from tests import utils as test_utils from uvloop import _testbase as tb diff --git a/tests/test_tasks.py b/tests/test_tasks.py index b237ab67..c5f52bfe 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -2,7 +2,7 @@ import asyncio -from asyncio import test_utils +from tests import utils as test_utils from uvloop import _testbase as tb @@ -232,7 +232,8 @@ def notmutch(): raise BaseException() task = self.create_task(notmutch()) - self.assertRaises(BaseException, task._step) + with self.assertRaises(BaseException): + test_utils.run_briefly(self.loop) self.assertTrue(task.done()) self.assertIsInstance(task.exception(), BaseException) @@ -245,7 +246,7 @@ def __init__(self, *args, **kwds): self.cb_added = False super().__init__(*args, **kwds) - def add_done_callback(self, fn): + def add_done_callback(self, fn, context=None): self.cb_added = True super().add_done_callback(fn) diff --git a/tests/test_udp.py b/tests/test_udp.py index b3f79e8b..fb2c7fb7 100644 --- a/tests/test_udp.py +++ b/tests/test_udp.py @@ -3,7 +3,7 @@ import unittest import sys -from asyncio import test_utils +from tests import utils as test_utils from uvloop import _testbase as tb diff --git a/tests/test_unix.py b/tests/test_unix.py index 87466c10..e9900d3e 100644 --- a/tests/test_unix.py +++ b/tests/test_unix.py @@ -5,6 +5,7 @@ import tempfile import time import unittest +import sys from uvloop import _testbase as tb @@ -362,6 +363,7 @@ async def client(): self.assertIn(excs[0].__class__, (BrokenPipeError, ConnectionResetError)) + @unittest.skipUnless(sys.version_info < (3, 7), 'Python version must be < 3.7') def test_transport_unclosed_warning(self): async def test(sock): return await self.loop.create_unix_connection( diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..d413581a --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,88 @@ +"""Utilities shared by tests.""" +import sys +import contextlib +import logging +import socket +import time +import inspect +import functools + +from asyncio.log import logger +from asyncio import futures +from asyncio import tasks +from socket import socketpair + + +def run_briefly(loop): + async def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() + + +def run_until(loop, pred, timeout=30): + deadline = time.time() + timeout + while not pred(): + if timeout is not None: + timeout = deadline - time.time() + if timeout <= 0: + raise futures.TimeoutError() + loop.run_until_complete(tasks.sleep(0.001, loop=loop)) + + +def run_once(loop): + """Legacy API to run once through the event loop. + + This is the recommended pattern for test code. It will poll the + selector once and run all callbacks scheduled in response to I/O + events. + """ + loop.call_soon(loop.stop) + loop.run_forever() + + +@contextlib.contextmanager +def disable_logger(): + """Context manager to disable asyncio logger. + + For example, it can be used to ignore warnings in debug mode. + """ + old_level = logger.level + try: + logger.setLevel(logging.CRITICAL+1) + yield + finally: + logger.setLevel(old_level) + + +def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, + family=socket.AF_INET): + """Create a mock of a non-blocking socket.""" + sock = mock.MagicMock(socket.socket) + sock.proto = proto + sock.type = type + sock.family = family + sock.gettimeout.return_value = 0.0 + return sock + + +def get_function_source(func): + if sys.version_info >= (3, 4): + func = inspect.unwrap(func) + elif hasattr(func, '__wrapped__'): + func = func.__wrapped__ + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if compat.PY34 and isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None diff --git a/uvloop/includes/stdlib.pxi b/uvloop/includes/stdlib.pxi index c09cec57..aff70e03 100644 --- a/uvloop/includes/stdlib.pxi +++ b/uvloop/includes/stdlib.pxi @@ -38,7 +38,6 @@ cdef aio_iscoroutinefunction = asyncio.iscoroutinefunction cdef aio_BaseProtocol = asyncio.BaseProtocol cdef aio_Protocol = asyncio.Protocol cdef aio_SSLProtocol = asyncio.sslproto.SSLProtocol -cdef aio_debug_wrapper = asyncio.coroutines.debug_wrapper cdef aio_isfuture = getattr(asyncio, 'isfuture', None) cdef aio_get_running_loop = getattr(asyncio, '_get_running_loop', None) cdef aio_set_running_loop = getattr(asyncio, '_set_running_loop', None) @@ -146,8 +145,8 @@ cdef py_inf = float('inf') # Cython doesn't clean-up imported objects properly in Py3 mode, -# so we delete refs to all modules manually (except sys) -del asyncio, concurrent, collections, errno +# so we delete refs to all modules manually (except sys, asyncio) +del concurrent, collections, errno del functools, inspect, itertools, socket, os, threading del signal, subprocess, ssl del time, traceback, warnings, weakref diff --git a/uvloop/loop.pxd b/uvloop/loop.pxd index c715e028..8afacfa9 100644 --- a/uvloop/loop.pxd +++ b/uvloop/loop.pxd @@ -29,11 +29,14 @@ ctypedef object (*method2_t)(object, object, object) ctypedef object (*method3_t)(object, object, object, object) + cdef class Loop: cdef: uv.uv_loop_t *uvloop - bint _coroutine_wrapper_set + + bint _coroutine_debug_set + int _coroutine_origin_tracking_saved_depth public slow_callback_duration @@ -194,7 +197,7 @@ cdef class Loop: cdef _read_from_self(self) cdef _process_self_data(self, data) - cdef _set_coroutine_wrapper(self, bint enabled) + cdef _set_coroutine_debug(self, bint enabled) cdef _print_debug_info(self) diff --git a/uvloop/loop.pyx b/uvloop/loop.pyx index 1e505675..f33f622a 100644 --- a/uvloop/loop.pyx +++ b/uvloop/loop.pyx @@ -152,7 +152,7 @@ cdef class Loop: self._signal_handlers = {} self._listening_signals = False - self._coroutine_wrapper_set = False + self._coroutine_debug_set = False if hasattr(sys, 'get_asyncgen_hooks'): # Python >= 3.6 @@ -970,32 +970,45 @@ cdef class Loop: if err < 0: raise convert_error(-errno.errno) - cdef _set_coroutine_wrapper(self, bint enabled): + cdef _set_coroutine_debug(self, bint enabled): enabled = bool(enabled) - if self._coroutine_wrapper_set == enabled: + if self._coroutine_debug_set == enabled: return - wrapper = aio_debug_wrapper - current_wrapper = sys_get_coroutine_wrapper() - - if enabled: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(True): cannot set debug coroutine " - "wrapper; another wrapper is already set %r" % - current_wrapper, RuntimeWarning) + if sys_version_info >= (3, 7, 0): + if enabled: + self._coroutine_origin_tracking_saved_depth = ( + sys.get_coroutine_origin_tracking_depth()) + sys.set_coroutine_origin_tracking_depth( + DEBUG_STACK_DEPTH) else: - sys_set_coroutine_wrapper(wrapper) - self._coroutine_wrapper_set = True + sys.set_coroutine_origin_tracking_depth( + self._coroutine_origin_tracking_saved_depth) + + self._coroutine_debug_set = enabled else: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(False): cannot unset debug coroutine " - "wrapper; another wrapper was set %r" % - current_wrapper, RuntimeWarning) + wrapper = asyncio.coroutines.debug_wrapper + current_wrapper = sys_get_coroutine_wrapper() + + if enabled: + if current_wrapper not in (None, wrapper): + warnings.warn( + "loop.set_debug(True): cannot set debug coroutine " + "wrapper; another wrapper is already set %r" % + current_wrapper, RuntimeWarning) + else: + sys_set_coroutine_wrapper(wrapper) + self._coroutine_debug_set = True else: - sys_set_coroutine_wrapper(None) - self._coroutine_wrapper_set = False + if current_wrapper not in (None, wrapper): + warnings.warn( + "loop.set_debug(False): cannot unset debug coroutine " + "wrapper; another wrapper was set %r" % + current_wrapper, RuntimeWarning) + else: + sys_set_coroutine_wrapper(None) + self._coroutine_debug_set = False + cdef _create_server(self, system.sockaddr *addr, object protocol_factory, @@ -1151,7 +1164,7 @@ cdef class Loop: self.is_closed(), self.get_debug()) - def call_soon(self, callback, *args): + def call_soon(self, callback, *args, context=None): """Arrange for a callback to be called as soon as possible. This operates as a FIFO queue: callbacks are called in the @@ -1168,7 +1181,7 @@ cdef class Loop: else: return self._call_soon(callback, None) - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): """Like call_soon(), but thread-safe.""" if not args: args = None @@ -1176,7 +1189,7 @@ cdef class Loop: self.handler_async.send() return handle - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): """Arrange for a callback to be called at a given time. Return a Handle: an opaque object with a cancel() method that @@ -1213,7 +1226,7 @@ cdef class Loop: else: return self._call_later(when, callback, args) - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): """Like call_later(), but uses an absolute time. Absolute time corresponds to the event loop's time() method. @@ -1251,7 +1264,7 @@ cdef class Loop: # loop.stop() was called right before loop.run_forever(). # This is how asyncio loop behaves. mode = uv.UV_RUN_NOWAIT - self._set_coroutine_wrapper(self._debug) + self._set_coroutine_debug(self._debug) if self._asyncgens is not None: old_agen_hooks = sys.get_asyncgen_hooks() sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, @@ -1259,7 +1272,7 @@ cdef class Loop: try: self._run(mode) finally: - self._set_coroutine_wrapper(False) + self._set_coroutine_debug(False) if self._asyncgens is not None: sys.set_asyncgen_hooks(*old_agen_hooks) @@ -1280,7 +1293,7 @@ cdef class Loop: def set_debug(self, enabled): self._debug = bool(enabled) if self.is_running(): - self._set_coroutine_wrapper(self._debug) + self.call_soon_threadsafe(self._set_coroutine_debug, self, self._debug) def is_running(self): """Return whether the event loop is currently running.""" From efc3a9e4954410353120169562a96b7c5e91fff2 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 10 Apr 2018 07:40:58 +0200 Subject: [PATCH 02/39] Testing with Ubuntu Xenial for PY37 step, comes with 3.7.0b1+ --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 23d4cbb6..c45be1d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ matrix: env: BUILD=tests - os: linux - dist: trusty + dist: xenial sudo: false language: python python: "3.7-dev" From 1f01d3f9edf188bc475fbe7d0948b0be680b239c Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 15 Apr 2018 09:04:59 +0200 Subject: [PATCH 03/39] Run Python 3.7 Travis step using docker --- .travis.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c45be1d6..38e0e64c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,9 @@ branches: - releases - /^v\d+(\.\d+)*$/ +services: + - docker + matrix: fast_finish: true @@ -57,10 +60,12 @@ matrix: env: BUILD=tests - os: linux - dist: xenial - sudo: false + dist: trusty + sudo: required language: python - python: "3.7-dev" + python: "3.7" + before_install: + - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc env: BUILD=tests - os: linux @@ -76,10 +81,20 @@ cache: pip install: - - .ci/travis-install.sh + - | + if [ $TRAVIS_PYTHON_VERSION == "3.7" ]; then + docker container exec -it uvloop-ci-py37 .ci/travis-install.sh + else + .ci/travis-install.sh + fi script: - - .ci/travis-tests.sh && .ci/travis-build-wheels.sh + - | + if [ $TRAVIS_PYTHON_VERSION == "3.7" ]; then + docker container exec -e BUILD='tests' -it uvloop-ci-py37 sh -c ".ci/travis-tests.sh" + else + .ci/travis-tests.sh && .ci/travis-build-wheels.sh + fi deploy: provider: script From 0d1dc9c8045e21005ee478ab79963cc3519d8978 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 15 Apr 2018 09:10:59 +0200 Subject: [PATCH 04/39] Dont relay on Travis PY37 environment --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38e0e64c..6ce3d0e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,11 +62,12 @@ matrix: - os: linux dist: trusty sudo: required - language: python - python: "3.7" + # Travis has removed the py37 image from their repos + # language: python + # python: "3.7" before_install: - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc - env: BUILD=tests + env: BUILD=tests UVLOOP_PY37_DOCKER=true - os: linux dist: trusty @@ -82,7 +83,7 @@ cache: install: - | - if [ $TRAVIS_PYTHON_VERSION == "3.7" ]; then + if [ $UVLOOP_PY37_DOCKER == "true" ]; then docker container exec -it uvloop-ci-py37 .ci/travis-install.sh else .ci/travis-install.sh @@ -90,7 +91,7 @@ install: script: - | - if [ $TRAVIS_PYTHON_VERSION == "3.7" ]; then + if [ $UVLOOP_PY37_DOCKER == "true" ]; then docker container exec -e BUILD='tests' -it uvloop-ci-py37 sh -c ".ci/travis-tests.sh" else .ci/travis-tests.sh && .ci/travis-build-wheels.sh From 2b3b1c7f599e9f5f6dfa86ded91d090a647fec4e Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 15 Apr 2018 09:28:36 +0200 Subject: [PATCH 05/39] Fixes an issue testing the environment in the travis file --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ce3d0e6..cf9e2d9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,7 +83,7 @@ cache: install: - | - if [ $UVLOOP_PY37_DOCKER == "true" ]; then + if [ "${UVLOOP_PY37_DOCKER}" == "true" ]; then docker container exec -it uvloop-ci-py37 .ci/travis-install.sh else .ci/travis-install.sh @@ -91,7 +91,7 @@ install: script: - | - if [ $UVLOOP_PY37_DOCKER == "true" ]; then + if [ "${UVLOOP_PY37_DOCKER}" == "true" ]; then docker container exec -e BUILD='tests' -it uvloop-ci-py37 sh -c ".ci/travis-tests.sh" else .ci/travis-tests.sh && .ci/travis-build-wheels.sh From 9ea9e297d8b99b67f6b3532465ed9bccd04ad289 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Mon, 16 Apr 2018 06:26:01 +0200 Subject: [PATCH 06/39] Disable pip upgrade at Travis CI --- .ci/travis-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 91215150..055aaa81 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -22,6 +22,7 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew outdated automake || brew upgrade automake --with-default-names fi -pip install --upgrade pip wheel +#pip install --upgrade pip +pip install --upgrade wheel pip install --upgrade setuptools pip install -r .ci/requirements.txt From 6714d991ad6dd941ee5d95e9a46ea8541b5774b6 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 06:10:39 +0200 Subject: [PATCH 07/39] Added back the explicit del asyncio and other minor stuff --- tests/test_pipes.py | 20 ++++----- tests/test_process.py | 7 ++- tests/test_tasks.py | 31 +++++++------- tests/test_udp.py | 5 +-- tests/utils.py | 88 -------------------------------------- uvloop/_testbase.py | 79 ++++++++++++++++++++++++++++++++++ uvloop/includes/stdlib.pxi | 5 ++- uvloop/loop.pyx | 2 +- 8 files changed, 113 insertions(+), 124 deletions(-) delete mode 100644 tests/utils.py diff --git a/tests/test_pipes.py b/tests/test_pipes.py index 97b3c03e..6304bc81 100644 --- a/tests/test_pipes.py +++ b/tests/test_pipes.py @@ -1,8 +1,8 @@ import asyncio import io import os +import socket -from tests import utils as test_utils from uvloop import _testbase as tb @@ -81,11 +81,11 @@ def connect(): self.loop.run_until_complete(connect()) os.write(wpipe, b'1') - test_utils.run_until(self.loop, lambda: proto.nbytes >= 1) + tb.run_until(self.loop, lambda: proto.nbytes >= 1) self.assertEqual(1, proto.nbytes) os.write(wpipe, b'2345') - test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + tb.run_until(self.loop, lambda: proto.nbytes >= 5) self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) self.assertEqual(5, proto.nbytes) @@ -114,11 +114,11 @@ def connect(): self.loop.run_until_complete(connect()) os.write(slave, b'1') - test_utils.run_until(self.loop, lambda: proto.nbytes) + tb.run_until(self.loop, lambda: proto.nbytes) self.assertEqual(1, proto.nbytes) os.write(slave, b'2345') - test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + tb.run_until(self.loop, lambda: proto.nbytes >= 5) self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) self.assertEqual(5, proto.nbytes) @@ -157,11 +157,11 @@ def reader(data): data += chunk return len(data) - test_utils.run_until(self.loop, lambda: reader(data) >= 1) + tb.run_until(self.loop, lambda: reader(data) >= 1) self.assertEqual(b'1', data) transport.write(b'2345') - test_utils.run_until(self.loop, lambda: reader(data) >= 5) + tb.run_until(self.loop, lambda: reader(data) >= 5) self.assertEqual(b'12345', data) self.assertEqual('CONNECTED', proto.state) @@ -176,7 +176,7 @@ def reader(data): self.assertEqual('CLOSED', proto.state) def test_write_pipe_disconnect_on_close(self): - rsock, wsock = test_utils.socketpair() + rsock, wsock = socket.socketpair() rsock.setblocking(False) pipeobj = io.open(wsock.detach(), 'wb', 1024) @@ -222,12 +222,12 @@ def reader(data): data += chunk return len(data) - test_utils.run_until(self.loop, lambda: reader(data) >= 1, + tb.run_until(self.loop, lambda: reader(data) >= 1, timeout=10) self.assertEqual(b'1', data) transport.write(b'2345') - test_utils.run_until(self.loop, lambda: reader(data) >= 5, + tb.run_until(self.loop, lambda: reader(data) >= 5, timeout=10) self.assertEqual(b'12345', data) self.assertEqual('CONNECTED', proto.state) diff --git a/tests/test_process.py b/tests/test_process.py index 2db2cb43..fced8562 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -9,7 +9,6 @@ import time import unittest -from tests import utils as test_utils from uvloop import _testbase as tb @@ -504,7 +503,7 @@ def cancel_make_transport(): # ignore the log: # "Exception during subprocess creation, kill the subprocess" - with test_utils.disable_logger(): + with tb.disable_logger(): self.loop.run_until_complete(cancel_make_transport()) def test_cancel_post_init(self): @@ -522,9 +521,9 @@ def cancel_make_transport(): # ignore the log: # "Exception during subprocess creation, kill the subprocess" - with test_utils.disable_logger(): + with tb.disable_logger(): self.loop.run_until_complete(cancel_make_transport()) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) class Test_UV_Process(_TestProcess, tb.UVTestCase): diff --git a/tests/test_tasks.py b/tests/test_tasks.py index c5f52bfe..6d0effed 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -2,7 +2,6 @@ import asyncio -from tests import utils as test_utils from uvloop import _testbase as tb @@ -44,7 +43,7 @@ def notmuch(): '_TestTasks.test_task_repr..notmuch') self.assertEqual(notmuch.__module__, __name__) - filename, lineno = test_utils.get_function_source(notmuch) + filename, lineno = tb.get_function_source(notmuch) src = "%s:%s" % (filename, lineno) # test coroutine object @@ -109,7 +108,7 @@ def task(): return 12 t = self.create_task(task()) - test_utils.run_briefly(self.loop) # start coro + tb.run_briefly(self.loop) # start coro t.cancel() self.assertRaises( asyncio.CancelledError, self.loop.run_until_complete, t) @@ -126,7 +125,7 @@ def task(): return 12 t = self.create_task(task()) - test_utils.run_briefly(self.loop) # start task + tb.run_briefly(self.loop) # start task f.cancel() with self.assertRaises(asyncio.CancelledError): self.loop.run_until_complete(t) @@ -143,7 +142,7 @@ def task(): t = self.create_task(task()) self.assertEqual(asyncio.Task.all_tasks(loop=self.loop), {t}) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) f.cancel() t.cancel() @@ -168,10 +167,10 @@ def task(): return 42 t = self.create_task(task()) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut1) # White-box test. fut1.set_result(None) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut2) # White-box test. t.cancel() self.assertTrue(fut2.cancelled()) @@ -195,14 +194,14 @@ def task(): return res t = self.create_task(task()) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut1) # White-box test. fut1.set_result(None) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut2) # White-box test. t.cancel() self.assertTrue(fut2.cancelled()) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(t._fut_waiter, fut3) # White-box test. fut3.set_result(42) res = self.loop.run_until_complete(t) @@ -233,7 +232,7 @@ def notmutch(): task = self.create_task(notmutch()) with self.assertRaises(BaseException): - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertTrue(task.done()) self.assertIsInstance(task.exception(), BaseException) @@ -259,12 +258,12 @@ def wait_for_future(): result = yield from fut t = self.create_task(wait_for_future()) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertTrue(fut.cb_added) res = object() fut.set_result(res) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertIs(res, result) self.assertTrue(t.done()) self.assertIsNone(t.result()) @@ -357,7 +356,7 @@ def outer(): proof += 10 f = asyncio.ensure_future(outer(), loop=self.loop) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) f.cancel() self.loop.run_until_complete(f) self.assertEqual(proof, 101) @@ -382,12 +381,12 @@ def outer(): proof += 100 f = asyncio.ensure_future(outer(), loop=self.loop) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) f.cancel() self.assertRaises( asyncio.CancelledError, self.loop.run_until_complete, f) waiter.set_result(None) - test_utils.run_briefly(self.loop) + tb.run_briefly(self.loop) self.assertEqual(proof, 1) diff --git a/tests/test_udp.py b/tests/test_udp.py index fb2c7fb7..1e5be9b5 100644 --- a/tests/test_udp.py +++ b/tests/test_udp.py @@ -3,7 +3,6 @@ import unittest import sys -from tests import utils as test_utils from uvloop import _testbase as tb @@ -77,9 +76,9 @@ def datagram_received(self, data, addr): self.assertIs(client.transport, transport) transport.sendto(b'xxx') - test_utils.run_until(self.loop, lambda: server.nbytes) + tb.run_until(self.loop, lambda: server.nbytes) self.assertEqual(3, server.nbytes) - test_utils.run_until(self.loop, lambda: client.nbytes) + tb.run_until(self.loop, lambda: client.nbytes) # received self.assertEqual(8, client.nbytes) diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index d413581a..00000000 --- a/tests/utils.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Utilities shared by tests.""" -import sys -import contextlib -import logging -import socket -import time -import inspect -import functools - -from asyncio.log import logger -from asyncio import futures -from asyncio import tasks -from socket import socketpair - - -def run_briefly(loop): - async def once(): - pass - gen = once() - t = loop.create_task(gen) - # Don't log a warning if the task is not done after run_until_complete(). - # It occurs if the loop is stopped or if a task raises a BaseException. - t._log_destroy_pending = False - try: - loop.run_until_complete(t) - finally: - gen.close() - - -def run_until(loop, pred, timeout=30): - deadline = time.time() + timeout - while not pred(): - if timeout is not None: - timeout = deadline - time.time() - if timeout <= 0: - raise futures.TimeoutError() - loop.run_until_complete(tasks.sleep(0.001, loop=loop)) - - -def run_once(loop): - """Legacy API to run once through the event loop. - - This is the recommended pattern for test code. It will poll the - selector once and run all callbacks scheduled in response to I/O - events. - """ - loop.call_soon(loop.stop) - loop.run_forever() - - -@contextlib.contextmanager -def disable_logger(): - """Context manager to disable asyncio logger. - - For example, it can be used to ignore warnings in debug mode. - """ - old_level = logger.level - try: - logger.setLevel(logging.CRITICAL+1) - yield - finally: - logger.setLevel(old_level) - - -def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, - family=socket.AF_INET): - """Create a mock of a non-blocking socket.""" - sock = mock.MagicMock(socket.socket) - sock.proto = proto - sock.type = type - sock.family = family - sock.gettimeout.return_value = 0.0 - return sock - - -def get_function_source(func): - if sys.version_info >= (3, 4): - func = inspect.unwrap(func) - elif hasattr(func, '__wrapped__'): - func = func.__wrapped__ - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if compat.PY34 and isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None diff --git a/uvloop/_testbase.py b/uvloop/_testbase.py index 22ebda08..680bc8dd 100644 --- a/uvloop/_testbase.py +++ b/uvloop/_testbase.py @@ -5,6 +5,7 @@ import asyncio.events import collections import contextlib +import functools import gc import logging import os @@ -12,14 +13,17 @@ import re import select import socket +import sys import ssl import tempfile import threading import time +import inspect import unittest import uvloop + class MockPattern(str): def __eq__(self, other): return bool(re.search(str(self), other, re.S)) @@ -483,3 +487,78 @@ def _handle_client(self, sock): @property def addr(self): return self._sock.getsockname() + + +def run_briefly(loop): + async def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() + + +def run_until(loop, pred, timeout=30): + deadline = time.time() + timeout + while not pred(): + if timeout is not None: + timeout = deadline - time.time() + if timeout <= 0: + raise asyncio.futures.TimeoutError() + loop.run_until_complete(asyncio.tasks.sleep(0.001, loop=loop)) + + +def run_once(loop): + """Legacy API to run once through the event loop. + + This is the recommended pattern for test code. It will poll the + selector once and run all callbacks scheduled in response to I/O + events. + """ + loop.call_soon(loop.stop) + loop.run_forever() + + +@contextlib.contextmanager +def disable_logger(): + """Context manager to disable asyncio logger. + + For example, it can be used to ignore warnings in debug mode. + """ + old_level = asyncio.log.logger.level + try: + asyncio.log.logger.setLevel(logging.CRITICAL+1) + yield + finally: + asyncio.log.logger.setLevel(old_level) + + +def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, + family=socket.AF_INET): + """Create a mock of a non-blocking socket.""" + sock = mock.MagicMock(socket.socket) + sock.proto = proto + sock.type = type + sock.family = family + sock.gettimeout.return_value = 0.0 + return sock + + +def get_function_source(func): + if sys.version_info >= (3, 4): + func = inspect.unwrap(func) + elif hasattr(func, '__wrapped__'): + func = func.__wrapped__ + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if compat.PY34 and isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None diff --git a/uvloop/includes/stdlib.pxi b/uvloop/includes/stdlib.pxi index aff70e03..94c2dd7c 100644 --- a/uvloop/includes/stdlib.pxi +++ b/uvloop/includes/stdlib.pxi @@ -41,6 +41,7 @@ cdef aio_SSLProtocol = asyncio.sslproto.SSLProtocol cdef aio_isfuture = getattr(asyncio, 'isfuture', None) cdef aio_get_running_loop = getattr(asyncio, '_get_running_loop', None) cdef aio_set_running_loop = getattr(asyncio, '_set_running_loop', None) +cdef aio_debug_wrapper = getattr(asyncio.coroutines, 'debug_wrapper', None) cdef col_deque = collections.deque cdef col_Iterable = collections.Iterable @@ -145,8 +146,8 @@ cdef py_inf = float('inf') # Cython doesn't clean-up imported objects properly in Py3 mode, -# so we delete refs to all modules manually (except sys, asyncio) -del concurrent, collections, errno +# so we delete refs to all modules manually (except sys) +del asyncio, concurrent, collections, errno del functools, inspect, itertools, socket, os, threading del signal, subprocess, ssl del time, traceback, warnings, weakref diff --git a/uvloop/loop.pyx b/uvloop/loop.pyx index f33f622a..54f9b68a 100644 --- a/uvloop/loop.pyx +++ b/uvloop/loop.pyx @@ -987,7 +987,7 @@ cdef class Loop: self._coroutine_debug_set = enabled else: - wrapper = asyncio.coroutines.debug_wrapper + wrapper = aio_debug_wrapper current_wrapper = sys_get_coroutine_wrapper() if enabled: From 842110737f2fa5780d093fdd632fa0d8baf01a5c Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 06:15:39 +0200 Subject: [PATCH 08/39] Temporary commit to get more info about Python env and Pip env --- .ci/travis-install.sh | 3 ++- .travis.yml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 055aaa81..7999c3be 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -22,7 +22,8 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew outdated automake || brew upgrade automake --with-default-names fi -#pip install --upgrade pip +pip --version +pip install --upgrade pip pip install --upgrade wheel pip install --upgrade setuptools pip install -r .ci/requirements.txt diff --git a/.travis.yml b/.travis.yml index cf9e2d9f..1ea54483 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,8 +63,8 @@ matrix: dist: trusty sudo: required # Travis has removed the py37 image from their repos - # language: python - # python: "3.7" + language: python + python: "3.7-dev" before_install: - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc env: BUILD=tests UVLOOP_PY37_DOCKER=true @@ -84,6 +84,7 @@ cache: install: - | if [ "${UVLOOP_PY37_DOCKER}" == "true" ]; then + python -V docker container exec -it uvloop-ci-py37 .ci/travis-install.sh else .ci/travis-install.sh From 36407ac5c90692b01283c2d62697080591654bae Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 06:41:33 +0200 Subject: [PATCH 09/39] Test again the 3.7 Travis Python image --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1ea54483..2064f28a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: sudo: required # Travis has removed the py37 image from their repos language: python - python: "3.7-dev" + python: "3.7" before_install: - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc env: BUILD=tests UVLOOP_PY37_DOCKER=true From 9dfd99d66f8bf7bd471eb792ef30acdb781cff44 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 06:50:53 +0200 Subject: [PATCH 10/39] Get rid of none functional Travis Python 3.7 configuration --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2064f28a..d9a0b0ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,9 +62,10 @@ matrix: - os: linux dist: trusty sudo: required - # Travis has removed the py37 image from their repos language: python - python: "3.7" + # Travis has removed the py37 image from their repos, while 3.7-dev still + # points to 3.7.0a4. So we use the docker version of 3.7-rc that points to the + # the last release candidate. before_install: - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc env: BUILD=tests UVLOOP_PY37_DOCKER=true From 5cb302c6de354585ea02cbbbb9f667aa93c60b15 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 06:54:06 +0200 Subject: [PATCH 11/39] Do not upgrade pip for Travis CI --- .ci/travis-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 7999c3be..db487554 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -22,8 +22,9 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew outdated automake || brew upgrade automake --with-default-names fi -pip --version -pip install --upgrade pip +# Pined to the 9.0 Pip version, having issues +# with Mac environments +# pip install --upgrade pip pip install --upgrade wheel pip install --upgrade setuptools pip install -r .ci/requirements.txt From f0375bc4332c6fe737a28e4cd940412a357bc292 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 07:26:58 +0200 Subject: [PATCH 12/39] Travis CI Pins PIP to 9.X for OSX enviornment --- .ci/travis-install.sh | 10 +++++++--- .travis.yml | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index db487554..bfd0fa1f 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -20,11 +20,15 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew outdated libtool || brew upgrade libtool brew outdated autoconf || brew upgrade autoconf --with-default-names brew outdated automake || brew upgrade automake --with-default-names + + # Pined to 9.0.X till following issues are addressed + # https://github.com/pypa/pip/issues/5240 + # https://github.com/pyenv/pyenv/issues/1141 + pip install --upgrade pip=~9.0.3 +else + pip install --upgrade pip fi -# Pined to the 9.0 Pip version, having issues -# with Mac environments -# pip install --upgrade pip pip install --upgrade wheel pip install --upgrade setuptools pip install -r .ci/requirements.txt diff --git a/.travis.yml b/.travis.yml index d9a0b0ed..bb13c7be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,6 +66,7 @@ matrix: # Travis has removed the py37 image from their repos, while 3.7-dev still # points to 3.7.0a4. So we use the docker version of 3.7-rc that points to the # the last release candidate. + # https://github.com/travis-ci/travis-ci/issues/9069 before_install: - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc env: BUILD=tests UVLOOP_PY37_DOCKER=true From 3a3e4cc697f8f2d911676a17f2a33543a9445ab3 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Tue, 17 Apr 2018 07:39:07 +0200 Subject: [PATCH 13/39] Fixed invalid pined expresion --- .ci/travis-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index bfd0fa1f..0f000b06 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -24,7 +24,7 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then # Pined to 9.0.X till following issues are addressed # https://github.com/pypa/pip/issues/5240 # https://github.com/pyenv/pyenv/issues/1141 - pip install --upgrade pip=~9.0.3 + pip install --upgrade pip~=9.0.0 else pip install --upgrade pip fi From d1bdc502a6ca89414db37fa6e557ea43213b9460 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 06:29:40 +0200 Subject: [PATCH 14/39] Test py37 recipe using an external PPA --- .ci/travis-install.sh | 2 ++ .travis.yml | 27 +++++++++------------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 0f000b06..5d28a9a0 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -2,6 +2,8 @@ set -e -x +python -V + if [ "${TRAVIS_OS_NAME}" == "osx" ]; then git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv PYENV_ROOT="$HOME/.pyenv" diff --git a/.travis.yml b/.travis.yml index bb13c7be..6b3aec2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,16 +60,18 @@ matrix: env: BUILD=tests - os: linux - dist: trusty + dist: xenial sudo: required language: python + python: "3.7-dev" # Travis has removed the py37 image from their repos, while 3.7-dev still - # points to 3.7.0a4. So we use the docker version of 3.7-rc that points to the - # the last release candidate. + # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 before_install: - - docker container run -dt -v $PWD:/app -w /app --name uvloop-ci-py37 --rm python:3.7-rc - env: BUILD=tests UVLOOP_PY37_DOCKER=true + - sudo add-apt-repository ppa:deadsnakes/ppa -y + - sudo sudo apt-get update + - sudo apt-get --yes install python3.7 + env: BUILD=tests - os: linux dist: trusty @@ -84,21 +86,10 @@ cache: pip install: - - | - if [ "${UVLOOP_PY37_DOCKER}" == "true" ]; then - python -V - docker container exec -it uvloop-ci-py37 .ci/travis-install.sh - else - .ci/travis-install.sh - fi + - .ci/travis-install.sh script: - - | - if [ "${UVLOOP_PY37_DOCKER}" == "true" ]; then - docker container exec -e BUILD='tests' -it uvloop-ci-py37 sh -c ".ci/travis-tests.sh" - else - .ci/travis-tests.sh && .ci/travis-build-wheels.sh - fi + - .ci/travis-tests.sh && .ci/travis-build-wheels.sh deploy: provider: script From 4c1d9cfac8ea85b7d147bff5557445dc5e21c477 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 07:22:38 +0200 Subject: [PATCH 15/39] Test with pyenv --- .ci/travis-build-wheels.sh | 2 +- .ci/travis-install.sh | 23 ++++++++++++++++++++++- .ci/travis-tests.sh | 2 +- .travis.yml | 6 +----- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index c1d9b77e..4359187c 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -9,7 +9,7 @@ if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then fi -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then +if [ "${TRAVIS_OS_NAME}" == "osx" ] || [ "${PYTHON_VERSION}" == "3.7" ]; then PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 5d28a9a0..760ee9f5 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -2,7 +2,6 @@ set -e -x -python -V if [ "${TRAVIS_OS_NAME}" == "osx" ]; then git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv @@ -23,6 +22,24 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew outdated autoconf || brew upgrade autoconf --with-default-names brew outdated automake || brew upgrade automake --with-default-names + # Pined to 9.0.X till following issues are addressed + # https://github.com/pypa/pip/issues/5240 + # https://github.com/pyenv/pyenv/issues/1141 + pip install --upgrade pip~=9.0.0 +elif [ "${PYTHON_VERSION}" == "3.7" ]; then + # Travis has removed the py37 image from their repos, while 3.7-dev still + # points to 3.7.0a4. https://github.com/travis-ci/travis-ci/issues/9069 + # Forcing to use the pyenv version. + + git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv + PYENV_ROOT="$HOME/.pyenv" + PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" + + pyenv install ${PYTHON_VERSION} + pyenv global ${PYTHON_VERSION} + pyenv rehash + # Pined to 9.0.X till following issues are addressed # https://github.com/pypa/pip/issues/5240 # https://github.com/pyenv/pyenv/issues/1141 @@ -31,6 +48,10 @@ else pip install --upgrade pip fi +echo "Travis Python version vs Python system version" +echo ${PYTHON_VERSION} +python -V + pip install --upgrade wheel pip install --upgrade setuptools pip install -r .ci/requirements.txt diff --git a/.ci/travis-tests.sh b/.ci/travis-tests.sh index 0fc4aa23..2a5dee7e 100755 --- a/.ci/travis-tests.sh +++ b/.ci/travis-tests.sh @@ -7,7 +7,7 @@ if [[ "${BUILD}" != *tests* ]]; then exit 0 fi -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then +if [ "${TRAVIS_OS_NAME}" == "osx" ] || [ "${PYTHON_VERSION}" == "3.7" ]; then PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" diff --git a/.travis.yml b/.travis.yml index 6b3aec2b..af2839b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,14 +63,10 @@ matrix: dist: xenial sudo: required language: python - python: "3.7-dev" # Travis has removed the py37 image from their repos, while 3.7-dev still # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 - before_install: - - sudo add-apt-repository ppa:deadsnakes/ppa -y - - sudo sudo apt-get update - - sudo apt-get --yes install python3.7 + python: "3.7-dev" env: BUILD=tests - os: linux From b1638314d38b9f1cec15c4c4421b20fb6ebc4baa Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 07:35:03 +0200 Subject: [PATCH 16/39] Test with pyenv refatorized --- .ci/travis-build-wheels.sh | 2 +- .ci/travis-install.sh | 35 +++++++++-------------------------- .ci/travis-tests.sh | 2 +- .travis.yml | 6 +++--- 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 4359187c..30a77b18 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -9,7 +9,7 @@ if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then fi -if [ "${TRAVIS_OS_NAME}" == "osx" ] || [ "${PYTHON_VERSION}" == "3.7" ]; then +if [ "${PYENV}" == "true" ]; then PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh index 760ee9f5..34918808 100755 --- a/.ci/travis-install.sh +++ b/.ci/travis-install.sh @@ -3,7 +3,7 @@ set -e -x -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then +if [ "${PYENV}" == "true" ]; then git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" @@ -15,30 +15,16 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then pyenv global ${PYTHON_VERSION} pyenv rehash - brew update + python --version - brew install gnu-sed --with-default-names - brew outdated libtool || brew upgrade libtool - brew outdated autoconf || brew upgrade autoconf --with-default-names - brew outdated automake || brew upgrade automake --with-default-names + if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + brew update - # Pined to 9.0.X till following issues are addressed - # https://github.com/pypa/pip/issues/5240 - # https://github.com/pyenv/pyenv/issues/1141 - pip install --upgrade pip~=9.0.0 -elif [ "${PYTHON_VERSION}" == "3.7" ]; then - # Travis has removed the py37 image from their repos, while 3.7-dev still - # points to 3.7.0a4. https://github.com/travis-ci/travis-ci/issues/9069 - # Forcing to use the pyenv version. - - git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - - pyenv install ${PYTHON_VERSION} - pyenv global ${PYTHON_VERSION} - pyenv rehash + brew install gnu-sed --with-default-names + brew outdated libtool || brew upgrade libtool + brew outdated autoconf || brew upgrade autoconf --with-default-names + brew outdated automake || brew upgrade automake --with-default-names + fi # Pined to 9.0.X till following issues are addressed # https://github.com/pypa/pip/issues/5240 @@ -48,9 +34,6 @@ else pip install --upgrade pip fi -echo "Travis Python version vs Python system version" -echo ${PYTHON_VERSION} -python -V pip install --upgrade wheel pip install --upgrade setuptools diff --git a/.ci/travis-tests.sh b/.ci/travis-tests.sh index 2a5dee7e..7dcac80f 100755 --- a/.ci/travis-tests.sh +++ b/.ci/travis-tests.sh @@ -7,7 +7,7 @@ if [[ "${BUILD}" != *tests* ]]; then exit 0 fi -if [ "${TRAVIS_OS_NAME}" == "osx" ] || [ "${PYTHON_VERSION}" == "3.7" ]; then +if [ "${PYENV}" == "true" ]; then PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" diff --git a/.travis.yml b/.travis.yml index af2839b8..6cc93c59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,14 +35,14 @@ matrix: osx_image: xcode7.3 # Travis macOS env does not support Python yet, # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.5.4 PIP_USER=1 + env: BUILD=tests,wheels PYTHON_VERSION=3.5.4 PIP_USER=1 PYENV=true branches: {only: [releases]} - os: osx osx_image: xcode7.3 # Travis macOS env does not support Python yet, # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.6.3 PIP_USER=1 + env: BUILD=tests,wheels PYTHON_VERSION=3.6.3 PIP_USER=1 PYENV=true branches: {only: [releases]} - os: linux @@ -67,7 +67,7 @@ matrix: # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 python: "3.7-dev" - env: BUILD=tests + env: BUILD=tests PYENV=true PYTHON_VERSION=3.7 - os: linux dist: trusty From 5850466bfc99f1d8bab941d89513765ff0efa16d Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 07:51:40 +0200 Subject: [PATCH 17/39] Explicit Python 3.7 to 3.7-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6cc93c59..1e15b842 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ matrix: # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 python: "3.7-dev" - env: BUILD=tests PYENV=true PYTHON_VERSION=3.7 + env: BUILD=tests PYENV=true PYTHON_VERSION=3.7-dev - os: linux dist: trusty From 04b272fd3cdf4f45532bffeabcb6c05efce94525 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 09:00:13 +0200 Subject: [PATCH 18/39] Force use PYENV PYTHON_VERSION when PYENV is used rather than the system one --- .ci/travis-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/travis-tests.sh b/.ci/travis-tests.sh index 7dcac80f..61aeb3eb 100755 --- a/.ci/travis-tests.sh +++ b/.ci/travis-tests.sh @@ -11,7 +11,10 @@ if [ "${PYENV}" == "true" ]; then PYENV_ROOT="$HOME/.pyenv" PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" + pyenv global ${PYTHON_VERSION} fi +python --version + make distclean && make && make test make distclean && make debug && make test From 51776f1369d353cfa9c96cc61452a70e81c9ab60 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 19 Apr 2018 09:28:58 +0200 Subject: [PATCH 19/39] Trigger Travis From 1705d78866c5bf15dbc217503e47652cf87c6a8d Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 20 Apr 2018 06:25:32 +0200 Subject: [PATCH 20/39] Force last beta release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1e15b842..d4b78984 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ matrix: # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 python: "3.7-dev" - env: BUILD=tests PYENV=true PYTHON_VERSION=3.7-dev + env: BUILD=tests PYENV=true PYTHON_VERSION=3.7.0b3 - os: linux dist: trusty From 2e358c906845bdcdfedea5dcb09d29a0518c7875 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 20 Apr 2018 07:12:49 +0200 Subject: [PATCH 21/39] Removed no longer used Docker service --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4b78984..0452201a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,6 @@ branches: - releases - /^v\d+(\.\d+)*$/ -services: - - docker - matrix: fast_finish: true From ed32c95511794d354333ebd8c317cbb6c3813941 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 06:44:57 +0200 Subject: [PATCH 22/39] debug --- tests/test_tcp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_tcp.py b/tests/test_tcp.py index 79e36227..19c4e2b3 100644 --- a/tests/test_tcp.py +++ b/tests/test_tcp.py @@ -427,10 +427,13 @@ def test_create_connection_3(self): def server(sock): data = sock.recv_all(4) + print("server received") + print(data) self.assertEqual(data, b'AAAA') sock.close() async def client(addr): + print(addr) reader, writer = await asyncio.open_connection( *addr, loop=self.loop) @@ -438,6 +441,7 @@ async def client(addr): writer.write(b'AAAA') with self.assertRaises(asyncio.IncompleteReadError): + print("reading") await reader.readexactly(10) writer.close() From b9efa8d1aa5e1b1fc2b450f2a3ba3321c444f59b Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 07:01:18 +0200 Subject: [PATCH 23/39] Debug failure test in CI --- tests/test_tcp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_tcp.py b/tests/test_tcp.py index 19c4e2b3..e6747859 100644 --- a/tests/test_tcp.py +++ b/tests/test_tcp.py @@ -443,6 +443,7 @@ async def client(addr): with self.assertRaises(asyncio.IncompleteReadError): print("reading") await reader.readexactly(10) + print("finish reading") writer.close() From 8e6184c79e8aeb728e3abb70c366ca1620578da8 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 14:16:53 +0200 Subject: [PATCH 24/39] Test with trusty --- .travis.yml | 2 +- tests/test_tcp.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 424d222b..10c7d6e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ matrix: env: BUILD=tests - os: linux - dist: xenial + dist: trusty sudo: required language: python # Travis has removed the py37 image from their repos, while 3.7-dev still diff --git a/tests/test_tcp.py b/tests/test_tcp.py index e6747859..79e36227 100644 --- a/tests/test_tcp.py +++ b/tests/test_tcp.py @@ -427,13 +427,10 @@ def test_create_connection_3(self): def server(sock): data = sock.recv_all(4) - print("server received") - print(data) self.assertEqual(data, b'AAAA') sock.close() async def client(addr): - print(addr) reader, writer = await asyncio.open_connection( *addr, loop=self.loop) @@ -441,9 +438,7 @@ async def client(addr): writer.write(b'AAAA') with self.assertRaises(asyncio.IncompleteReadError): - print("reading") await reader.readexactly(10) - print("finish reading") writer.close() From 8480b4616f1d729edb8d9b9011753f9699c1f7ea Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 17:41:35 +0200 Subject: [PATCH 25/39] Get network interfaces in Travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 10c7d6e8..2f151de8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ matrix: env: BUILD=tests - os: linux - dist: trusty + dist: xenial sudo: required language: python # Travis has removed the py37 image from their repos, while 3.7-dev still @@ -82,6 +82,7 @@ install: - .ci/travis-install.sh script: + - ifconfig && exit 1 - .ci/travis-tests.sh && .ci/travis-build-wheels.sh deploy: From 8fe70b949f4825a367c0635ff3b5a85358241d29 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 17:47:49 +0200 Subject: [PATCH 26/39] Python 3.7.0b4 back for xenial wihout pyenv? --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f151de8..32396e83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: # points to 3.7.0a4. # https://github.com/travis-ci/travis-ci/issues/9069 python: "3.7-dev" - env: BUILD=tests PYENV=true PYTHON_VERSION=3.7.0b3 + env: BUILD=tests - os: linux dist: trusty @@ -82,7 +82,6 @@ install: - .ci/travis-install.sh script: - - ifconfig && exit 1 - .ci/travis-tests.sh && .ci/travis-build-wheels.sh deploy: From 8646ff4423ed2962d98ade8c36943ee5d389807a Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 18:17:51 +0200 Subject: [PATCH 27/39] Trigger CI From 3bd621f60192cdf681870dd0620289d34112e2db Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Fri, 4 May 2018 20:00:13 +0200 Subject: [PATCH 28/39] Trigger CI 2 From 0f36879a985bf6ce1a42fb8acde314f210c94966 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sat, 5 May 2018 15:23:49 +0200 Subject: [PATCH 29/39] Run only problematic test --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index acb9796d..5d5b8d84 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,8 @@ docs: test: - PYTHONASYNCIODEBUG=1 $(PYTHON) setup.py test - $(PYTHON) setup.py test + PYTHONASYNCIODEBUG=1 $(PYTHON) -m unittest tests.test_tcp -v + $(PYTHON) -m unittest tests.test_tcp -v testinstalled: From 5bdb2381ed5a7aa8b95fcd98189e76dca96b0300 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 07:44:46 +0200 Subject: [PATCH 30/39] Test without debug asyncio --- .travis.yml | 30 ------------------------------ Makefile | 1 + 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 32396e83..2e0066fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,27 +28,6 @@ matrix: fast_finish: true include: - - os: osx - osx_image: xcode7.3 - # Travis macOS env does not support Python yet, - # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.5.5 PYENV=true - branches: {only: [releases]} - - - os: osx - osx_image: xcode7.3 - # Travis macOS env does not support Python yet, - # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.6.5 PYENV=true - branches: {only: [releases]} - - - os: linux - dist: trusty - sudo: false - language: python - python: "3.5" - env: BUILD=tests - - os: linux dist: trusty sudo: false @@ -66,15 +45,6 @@ matrix: python: "3.7-dev" env: BUILD=tests - - os: linux - dist: trusty - branches: {only: [releases]} - sudo: required - language: python - python: "3.5" - services: [docker] - env: BUILD=tests,wheels,release - cache: pip diff --git a/Makefile b/Makefile index 5d5b8d84..82e12033 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ docs: test: + $(PYTHON) -m unittest tests.test_tcp -v PYTHONASYNCIODEBUG=1 $(PYTHON) -m unittest tests.test_tcp -v $(PYTHON) -m unittest tests.test_tcp -v From 09cffad3e3917e7d332d9cc8388f951fc87e390e Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 14:17:07 +0200 Subject: [PATCH 31/39] Test 3.6 against Xenial --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e0066fa..bf2a7cd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: include: - os: linux - dist: trusty + dist: xenial sudo: false language: python python: "3.6" @@ -37,7 +37,7 @@ matrix: - os: linux dist: xenial - sudo: required + sudo: false language: python # Travis has removed the py37 image from their repos, while 3.7-dev still # points to 3.7.0a4. From 7638a59839d13e93fb2c411f2a8278d093aaf137 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 14:28:59 +0200 Subject: [PATCH 32/39] Test all Travis matrix --- .travis.yml | 35 +++++++++++++++++++++++++++++++---- Makefile | 5 ++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf2a7cd5..38038dbe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,20 +28,47 @@ matrix: fast_finish: true include: + - os: osx + osx_image: xcode7.3 + # Travis macOS env does not support Python yet, + # so we have to set things up manually in install.sh. + env: BUILD=tests,wheels PYTHON_VERSION=3.5.5 + branches: {only: [releases]} + + - os: osx + osx_image: xcode7.3 + # Travis macOS env does not support Python yet, + # so we have to set things up manually in install.sh. + env: BUILD=tests,wheels PYTHON_VERSION=3.6.5 + branches: {only: [releases]} + - os: linux - dist: xenial + dist: trusty + sudo: false + language: python + python: "3.5" + env: BUILD=tests + + - os: linux + dist: trusty sudo: false language: python python: "3.6" env: BUILD=tests + - os: linux + dist: trusty + branches: {only: [releases]} + sudo: required + language: python + python: "3.5" + services: [docker] + env: BUILD=tests,wheels,release + - os: linux dist: xenial sudo: false language: python - # Travis has removed the py37 image from their repos, while 3.7-dev still - # points to 3.7.0a4. - # https://github.com/travis-ci/travis-ci/issues/9069 python: "3.7-dev" env: BUILD=tests diff --git a/Makefile b/Makefile index 82e12033..acb9796d 100644 --- a/Makefile +++ b/Makefile @@ -43,9 +43,8 @@ docs: test: - $(PYTHON) -m unittest tests.test_tcp -v - PYTHONASYNCIODEBUG=1 $(PYTHON) -m unittest tests.test_tcp -v - $(PYTHON) -m unittest tests.test_tcp -v + PYTHONASYNCIODEBUG=1 $(PYTHON) setup.py test + $(PYTHON) setup.py test testinstalled: From 8c3b28a4390391da48b5822f6a45df0e0f682066 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 14:54:42 +0200 Subject: [PATCH 33/39] Fix use of PIPENV for Mac OSX --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38038dbe..06755c36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,16 +32,23 @@ matrix: osx_image: xcode7.3 # Travis macOS env does not support Python yet, # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.5.5 + env: BUILD=tests,wheels PYTHON_VERSION=3.5.5 PYENV=true branches: {only: [releases]} - os: osx osx_image: xcode7.3 # Travis macOS env does not support Python yet, # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.6.5 + env: BUILD=tests,wheels PYTHON_VERSION=3.6.5 PYENV=true branches: {only: [releases]} + - os: linux + dist: xenial + sudo: false + language: python + python: "3.7-dev" + env: BUILD=tests + - os: linux dist: trusty sudo: false @@ -65,13 +72,6 @@ matrix: services: [docker] env: BUILD=tests,wheels,release - - os: linux - dist: xenial - sudo: false - language: python - python: "3.7-dev" - env: BUILD=tests - cache: pip From 46480da56c3a06d47405536917da35be9abd73ec Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 16:07:09 +0200 Subject: [PATCH 34/39] Test with only 3.5 and 3.7-dev --- .travis.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 06755c36..3410cbc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,20 +28,6 @@ matrix: fast_finish: true include: - - os: osx - osx_image: xcode7.3 - # Travis macOS env does not support Python yet, - # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.5.5 PYENV=true - branches: {only: [releases]} - - - os: osx - osx_image: xcode7.3 - # Travis macOS env does not support Python yet, - # so we have to set things up manually in install.sh. - env: BUILD=tests,wheels PYTHON_VERSION=3.6.5 PYENV=true - branches: {only: [releases]} - - os: linux dist: xenial sudo: false @@ -49,20 +35,6 @@ matrix: python: "3.7-dev" env: BUILD=tests - - os: linux - dist: trusty - sudo: false - language: python - python: "3.5" - env: BUILD=tests - - - os: linux - dist: trusty - sudo: false - language: python - python: "3.6" - env: BUILD=tests - - os: linux dist: trusty branches: {only: [releases]} From 5338ae6b6e011d42085e0fe6f3649f325c35dedc Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 16:22:04 +0200 Subject: [PATCH 35/39] Only 3.7-dev --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3410cbc8..2adcba46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,15 +35,6 @@ matrix: python: "3.7-dev" env: BUILD=tests - - os: linux - dist: trusty - branches: {only: [releases]} - sudo: required - language: python - python: "3.5" - services: [docker] - env: BUILD=tests,wheels,release - cache: pip From b3a9de1458de5e2f96e080a66105abf72a8357b3 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 16:37:43 +0200 Subject: [PATCH 36/39] Test with PYENV=true PYTHON_VERSION=3.7.0b3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2adcba46..df6c9033 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: sudo: false language: python python: "3.7-dev" - env: BUILD=tests + env: BUILD=tests PYENV=true PYTHON_VERSION=3.7.0b4 cache: pip From 7d7ac60e40c19443f0da0efaa85b3ea84d62b872 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 6 May 2018 16:42:39 +0200 Subject: [PATCH 37/39] Test with PYENV=true PYTHON_VERSION=3.7.0b3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df6c9033..3c5ffa79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: sudo: false language: python python: "3.7-dev" - env: BUILD=tests PYENV=true PYTHON_VERSION=3.7.0b4 + env: BUILD=tests PYENV=true PYTHON_VERSION=3.7.0b3 cache: pip From 4b35859fa4cbfdf1a77db3ec3a64ab59816cf2ee Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Sun, 20 May 2018 13:08:23 +0200 Subject: [PATCH 38/39] POC UVloop tracing, support for DNS traces This is the first PR that implements POC that gives to uvloop the capacity of emit traces that later can be collected by the upper layers. What does this commit implement? - DNS trace events triggered by the calls to the `getaddrinfo` loop method. - The enabler and disabler context manager called `tracing`. - The collector interface used to gather the trace events called `TracingCollector. This POC assumes that the caller is the last responsible implementing a context that isolates calling to traces that belong to different tasks. Missing pices in this PoC: - Allow many collectors at a time. - The exceptions have to be propagated at tracing level? perhaps, what should we do when there is an exception and we have to call to `dns_request_end` ? right now a `None` is passed as argument --- tests/test_dns.py | 28 ++++++++++++++++++++++++++++ tests/test_tracing.py | 17 +++++++++++++++++ uvloop/__init__.py | 3 ++- uvloop/loop.pxd | 2 ++ uvloop/loop.pyx | 8 ++++++++ uvloop/tracing.pxd | 8 ++++++++ uvloop/tracing.pyx | 39 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 tests/test_tracing.py create mode 100644 uvloop/tracing.pxd create mode 100644 uvloop/tracing.pyx diff --git a/tests/test_dns.py b/tests/test_dns.py index 47820284..9c2f5843 100644 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -2,6 +2,7 @@ import unittest from uvloop import _testbase as tb +from uvloop import tracing, TracingCollector class BaseTestDNS: @@ -174,6 +175,33 @@ async def run(): finally: self.loop.close() + def test_getaddrinfo_tracing(self): + + class DNSCollector(TracingCollector): + dns_request_begin_call = False + dns_request_end_called = False + + def dns_request_begin(self, *args): + self.dns_request_begin_called = True + + def dns_request_end(self, *args): + self.dns_request_end_called = True + + collector = DNSCollector() + with tracing(collector): + self.loop.run_until_complete( + self.loop.getaddrinfo('example.com', 80) + ) + assert collector.dns_request_begin_called + assert collector.dns_request_end_called + + collector.dns_request_begin_called = False + collector.dns_request_end_called = False + self.loop.run_until_complete( + self.loop.getaddrinfo('example.com', 80) + ) + assert not collector.dns_request_begin_called + assert not collector.dns_request_end_called class Test_AIO_DNS(BaseTestDNS, tb.AIOTestCase): pass diff --git a/tests/test_tracing.py b/tests/test_tracing.py new file mode 100644 index 00000000..f52bae99 --- /dev/null +++ b/tests/test_tracing.py @@ -0,0 +1,17 @@ +import unittest + +from uvloop import tracing, TracingCollector + + +class TestTracingCollector(unittest.TestCase): + def test_tracing_methods(self): + collector = TracingCollector() + assert hasattr(collector, 'dns_request_begin') + assert hasattr(collector, 'dns_request_end') + + +class TestTracing(unittest.TestCase): + def test_invalid_collector(self): + with self.assertRaises(ValueError): + with tracing(None): + pass diff --git a/uvloop/__init__.py b/uvloop/__init__.py index 8f09dd07..38ea126a 100644 --- a/uvloop/__init__.py +++ b/uvloop/__init__.py @@ -5,9 +5,10 @@ from . import includes as __includes # NOQA from . import _patch # NOQA from .loop import Loop as __BaseLoop # NOQA +from .loop import tracing, TracingCollector -__all__ = ('new_event_loop', 'EventLoopPolicy') +__all__ = ('new_event_loop', 'EventLoopPolicy', 'tracing', 'TracingCollector') class Loop(__BaseLoop, asyncio.AbstractEventLoop): diff --git a/uvloop/loop.pxd b/uvloop/loop.pxd index 8afacfa9..c13fd6f6 100644 --- a/uvloop/loop.pxd +++ b/uvloop/loop.pxd @@ -222,3 +222,5 @@ include "request.pxd" include "handles/udp.pxd" include "server.pxd" + +include "tracing.pxd" diff --git a/uvloop/loop.pyx b/uvloop/loop.pyx index 54f9b68a..6fb98228 100644 --- a/uvloop/loop.pyx +++ b/uvloop/loop.pyx @@ -38,6 +38,7 @@ include "includes/stdlib.pxi" include "errors.pyx" + cdef _is_sock_stream(sock_type): if SOCK_NONBLOCK == -1: return sock_type == uv.SOCK_STREAM @@ -778,12 +779,17 @@ cdef class Loop: except Exception as ex: if not fut.cancelled(): fut.set_exception(ex) + __trace_dns_request_end(None) else: if not fut.cancelled(): fut.set_result(data) + __trace_dns_request_end(data) else: if not fut.cancelled(): fut.set_exception(result) + __trace_dns_request_end(result) + + __trace_dns_request_begin(host, port, family, type, proto, flags) AddrInfoRequest(self, host, port, family, type, proto, flags, callback) return fut @@ -2786,6 +2792,8 @@ include "handles/udp.pyx" include "server.pyx" +include "tracing.pyx" + # Used in UVProcess cdef vint __atfork_installed = 0 diff --git a/uvloop/tracing.pxd b/uvloop/tracing.pxd new file mode 100644 index 00000000..052062b9 --- /dev/null +++ b/uvloop/tracing.pxd @@ -0,0 +1,8 @@ +cdef class TracingCollector: + cpdef dns_request_begin(self, object arg1, object arg2, object arg3, + object arg4, object arg5, object arg6) + cpdef dns_request_end(self, object arg1) + +cdef __trace_dns_request_end(object arg1) +cdef __trace_dns_request_begin(object arg1, object arg2, object arg3, + object arg4, object arg5, object arg6) diff --git a/uvloop/tracing.pyx b/uvloop/tracing.pyx new file mode 100644 index 00000000..f5241df8 --- /dev/null +++ b/uvloop/tracing.pyx @@ -0,0 +1,39 @@ +import contextvars +from contextlib import contextmanager + + +__tracing_ctx = contextvars.ContextVar('__tracing_ctx') + + +cdef class TracingCollector: + cpdef dns_request_begin(self, arg1, arg2, arg3, arg4, arg5, arg6): + pass + + cpdef dns_request_end(self, arg1): + pass + + +@contextmanager +def tracing(collector): + if not isinstance(collector, TracingCollector): + raise ValueError("collector must be a TracingCollector class instance") + + __tracing_ctx.set(collector) + yield + __tracing_ctx.set(None) + + +cdef inline __trace_dns_request_end(arg1): + cdef TracingCollector collector = __tracing_ctx.get(None) + if not collector: + return + + collector.dns_request_end(arg1) + + +cdef inline __trace_dns_request_begin(arg1, arg2, arg3, arg4, arg5, arg6): + cdef TracingCollector collector = __tracing_ctx.get(None) + if not collector: + return + + collector.dns_request_begin(arg1, arg2, arg3, arg4, arg5, arg6) From 36b9433b53fd90de03bc4b4dfc1217a13c527fed Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Mon, 21 May 2018 08:00:27 +0200 Subject: [PATCH 39/39] UVloop tracing, support for task created trace This trace will help us to measure the cost of having a tracing system taking into account that this trace does not entail any IO operation under the hood. --- tests/test_dns.py | 11 ++++++----- tests/test_tasks.py | 24 ++++++++++++++++++++++++ tests/test_tracing.py | 1 + uvloop/loop.pyx | 1 + uvloop/tracing.pxd | 2 ++ uvloop/tracing.pyx | 18 +++++++++++------- 6 files changed, 45 insertions(+), 12 deletions(-) diff --git a/tests/test_dns.py b/tests/test_dns.py index 9c2f5843..958a03e8 100644 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -178,7 +178,7 @@ async def run(): def test_getaddrinfo_tracing(self): class DNSCollector(TracingCollector): - dns_request_begin_call = False + dns_request_begin_called = False dns_request_end_called = False def dns_request_begin(self, *args): @@ -192,16 +192,17 @@ def dns_request_end(self, *args): self.loop.run_until_complete( self.loop.getaddrinfo('example.com', 80) ) - assert collector.dns_request_begin_called - assert collector.dns_request_end_called + assert collector.dns_request_begin_called + assert collector.dns_request_end_called collector.dns_request_begin_called = False collector.dns_request_end_called = False self.loop.run_until_complete( self.loop.getaddrinfo('example.com', 80) ) - assert not collector.dns_request_begin_called - assert not collector.dns_request_end_called + assert not collector.dns_request_begin_called + assert not collector.dns_request_end_called + class Test_AIO_DNS(BaseTestDNS, tb.AIOTestCase): pass diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 6d0effed..f157452a 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -3,6 +3,7 @@ import asyncio from uvloop import _testbase as tb +from uvloop import tracing, TracingCollector class Dummy: @@ -402,6 +403,29 @@ def create_future(self): def create_task(self, coro): return self.loop.create_task(coro) + def test_create_task_tracing(self): + + @asyncio.coroutine + def coro(): + pass + + class CreateTaskCollector(TracingCollector): + task_created_called = False + + def task_created(self, *args): + self.task_created_called = True + + collector = CreateTaskCollector() + with tracing(collector): + self.create_task(coro()) + assert collector.task_created_called + + collector.task_created_called = False + self.create_task(coro()) + assert not collector.task_created_called + + tb.run_briefly(self.loop) + class Test_UV_UV_Tasks_AIO_Future(_TestTasks, tb.UVTestCase): def create_future(self): diff --git a/tests/test_tracing.py b/tests/test_tracing.py index f52bae99..19e1ee3f 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -8,6 +8,7 @@ def test_tracing_methods(self): collector = TracingCollector() assert hasattr(collector, 'dns_request_begin') assert hasattr(collector, 'dns_request_end') + assert hasattr(collector, 'task_created') class TestTracing(unittest.TestCase): diff --git a/uvloop/loop.pyx b/uvloop/loop.pyx index 6fb98228..4d53470c 100644 --- a/uvloop/loop.pyx +++ b/uvloop/loop.pyx @@ -1323,6 +1323,7 @@ cdef class Loop: task = aio_Task(coro, loop=self) else: task = self._task_factory(self, coro) + __trace_task_created(task) return task def set_task_factory(self, factory): diff --git a/uvloop/tracing.pxd b/uvloop/tracing.pxd index 052062b9..64dcab4c 100644 --- a/uvloop/tracing.pxd +++ b/uvloop/tracing.pxd @@ -2,7 +2,9 @@ cdef class TracingCollector: cpdef dns_request_begin(self, object arg1, object arg2, object arg3, object arg4, object arg5, object arg6) cpdef dns_request_end(self, object arg1) + cpdef task_created(self, object arg1) cdef __trace_dns_request_end(object arg1) cdef __trace_dns_request_begin(object arg1, object arg2, object arg3, object arg4, object arg5, object arg6) +cdef __trace_task_created(object arg1) diff --git a/uvloop/tracing.pyx b/uvloop/tracing.pyx index f5241df8..80dd9f5d 100644 --- a/uvloop/tracing.pyx +++ b/uvloop/tracing.pyx @@ -12,6 +12,9 @@ cdef class TracingCollector: cpdef dns_request_end(self, arg1): pass + cpdef task_created(self, arg1): + pass + @contextmanager def tracing(collector): @@ -25,15 +28,16 @@ def tracing(collector): cdef inline __trace_dns_request_end(arg1): cdef TracingCollector collector = __tracing_ctx.get(None) - if not collector: - return - - collector.dns_request_end(arg1) + if collector: + collector.dns_request_end(arg1) cdef inline __trace_dns_request_begin(arg1, arg2, arg3, arg4, arg5, arg6): cdef TracingCollector collector = __tracing_ctx.get(None) - if not collector: - return + if collector: + collector.dns_request_begin(arg1, arg2, arg3, arg4, arg5, arg6) - collector.dns_request_begin(arg1, arg2, arg3, arg4, arg5, arg6) +cdef inline __trace_task_created(arg1): + cdef TracingCollector collector = __tracing_ctx.get(None) + if collector: + collector.task_created(arg1)