From e46f82e9ce5b58c8e1b0d7dc9985837378805fae Mon Sep 17 00:00:00 2001 From: d-ryzhikov Date: Fri, 25 Oct 2019 18:52:41 +0300 Subject: [PATCH 01/17] Add iscoroutine with replacement for older pythons --- memory_profiler.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/memory_profiler.py b/memory_profiler.py index 6edd226..5e77129 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -29,6 +29,16 @@ import psutil +if sys.version_info > (3, 3): + if sys.version_info < (3, 5): + from asyncio import iscoroutinefunction + else: + from inspect import iscoroutinefunction +else: + def iscoroutinefunction(_): + return False + + # TODO: provide alternative when multiprocessing is not available try: from multiprocessing import Process, Pipe From 71a61fb0b1d4b8b5ebf6deb6cb6462e4479adbad Mon Sep 17 00:00:00 2001 From: d-ryzhikov Date: Fri, 25 Oct 2019 19:31:08 +0300 Subject: [PATCH 02/17] Include version-specific module with async support --- setup.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a89ee6..4050cd4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import os import io import re +import sys from setuptools import setup @@ -21,6 +22,14 @@ def find_version(*file_paths): return version_match.group(1) +async_modules = [] +if sys.version_info > (3, 3): + if sys.version_info < (3, 5): + async_modules = ['_aio_34'] + else: + async_modules = ['_aio_35'] + + CLASSIFIERS = """\ Development Status :: 5 - Production/Stable Intended Audience :: Science/Research @@ -39,6 +48,7 @@ def find_version(*file_paths): """ + setup( name='memory_profiler', description='A module for monitoring memory usage of a python program', @@ -47,7 +57,7 @@ def find_version(*file_paths): author='Fabian Pedregosa', author_email='f@bianp.net', url='https://github.com/pythonprofilers/memory_profiler', - py_modules=['memory_profiler', 'mprof'], + py_modules=['memory_profiler', 'mprof'] + async_modules, entry_points={ 'console_scripts' : ['mprof = mprof:main'], }, From d65fb3cf2fc65e16cb9c49df2fa0dc3371fc4722 Mon Sep 17 00:00:00 2001 From: d-ryzhikov Date: Fri, 25 Oct 2019 19:44:07 +0300 Subject: [PATCH 03/17] Wrapper and profiler for 3.4 coroutines --- _aio_34.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 _aio_34.py diff --git a/_aio_34.py b/_aio_34.py new file mode 100644 index 0000000..874197a --- /dev/null +++ b/_aio_34.py @@ -0,0 +1,35 @@ +from asyncio import coroutine, iscoroutinefunction +from contextlib import contextmanager +from functools import wraps + +from memory_profiler import LineProfiler, show_results + + +class CoroLineProfiler(LineProfiler): + @contextmanager + def count_contextmgr(self): + self.enable_by_count() + try: + yield + finally: + self.disable_by_count() + + def wrap_function(self, func): + if iscoroutinefunction(func): + @coroutine + def f(*args, **kwargs): + with self.count_contextmgr(): + yield from func(*args, **kwargs) + return f + else: + return super(CoroLineProfiler, self).wrap_function(func) + + +def _get_coro_wrapper(coro, backend, stream, precision): + @wraps(coro) + @coroutine + def wrapper(*args, **kwargs): + prof = CoroLineProfiler(backend=backend) + val = yield from prof(coro)(*args, **kwargs) + show_results(prof, stream=stream, precision=precision) + return val From 891d56ea3c88c4a5b540ece4e2848885277f9564 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sun, 5 Jan 2020 22:19:06 +0300 Subject: [PATCH 04/17] Convert module to package --- .../__init__.py | 491 +----------------- memory_profiler/__main__.py | 115 ++++ memory_profiler/code_map.py | 69 +++ memory_profiler/common.py | 13 + memory_profiler/line_profiler.py | 160 ++++++ memory_profiler/utils.py | 183 +++++++ setup.py | 18 +- 7 files changed, 569 insertions(+), 480 deletions(-) rename memory_profiler.py => memory_profiler/__init__.py (62%) create mode 100644 memory_profiler/__main__.py create mode 100644 memory_profiler/code_map.py create mode 100644 memory_profiler/common.py create mode 100644 memory_profiler/line_profiler.py create mode 100644 memory_profiler/utils.py diff --git a/memory_profiler.py b/memory_profiler/__init__.py similarity index 62% rename from memory_profiler.py rename to memory_profiler/__init__.py index 5e77129..934ad15 100644 --- a/memory_profiler.py +++ b/memory_profiler/__init__.py @@ -5,14 +5,11 @@ __version__ = '0.55.0' -_CMD_USAGE = "python -m memory_profiler script_file.py" from functools import wraps import inspect -import linecache import logging import os -import pdb import subprocess import sys import time @@ -20,6 +17,16 @@ import warnings import contextlib +from .common import PY2, HAS_TRACEMALLOC +from .utils import ( + show_results, + choose_backend, + get_memory as _get_memory, + get_child_memory as _get_child_memory, +) +from .line_profiler import LineProfiler + + if sys.platform == "win32": # any value except signal.CTRL_C_EVENT and signal.CTRL_BREAK_EVENT # can be used to kill a process unconditionally in Windows @@ -29,16 +36,6 @@ import psutil -if sys.version_info > (3, 3): - if sys.version_info < (3, 5): - from asyncio import iscoroutinefunction - else: - from inspect import iscoroutinefunction -else: - def iscoroutinefunction(_): - return False - - # TODO: provide alternative when multiprocessing is not available try: from multiprocessing import Process, Pipe @@ -53,10 +50,6 @@ def iscoroutinefunction(_): line_cell_magic = lambda func: func magics_class = lambda cls: cls -PY2 = sys.version_info[0] == 2 - -_TWO_20 = float(2 ** 20) - if PY2: import __builtin__ as builtins to_str = lambda x: x @@ -65,14 +58,9 @@ def iscoroutinefunction(_): import builtins to_str = lambda x: str(x) -# .. get available packages .. -try: +if HAS_TRACEMALLOC: import tracemalloc - has_tracemalloc = True -except ImportError: - has_tracemalloc = False - class MemitResult(object): """memit magic run details. @@ -99,108 +87,6 @@ def _repr_pretty_(self, p, cycle): p.text(u'') -def _get_child_memory(process, meminfo_attr=None): - """ - Returns a generator that yields memory for all child processes. - """ - # Convert a pid to a process - if isinstance(process, int): - if process == -1: process = os.getpid() - process = psutil.Process(process) - - if not meminfo_attr: - # Use the psutil 2.0 attr if the older version isn't passed in. - meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') else 'get_memory_info' - - # Select the psutil function get the children similar to how we selected - # the memory_info attr (a change from excepting the AttributeError). - children_attr = 'children' if hasattr(process, 'children') else 'get_children' - - # Loop over the child processes and yield their memory - try: - for child in getattr(process, children_attr)(recursive=True): - yield getattr(child, meminfo_attr)()[0] / _TWO_20 - except (psutil.NoSuchProcess, psutil.AccessDenied): - # https://github.com/fabianp/memory_profiler/issues/71 - yield 0.0 - - -def _get_memory(pid, backend, timestamps=False, include_children=False, filename=None): - # .. low function to get memory consumption .. - if pid == -1: - pid = os.getpid() - - def tracemalloc_tool(): - # .. cross-platform but but requires Python 3.4 or higher .. - stat = next(filter(lambda item: str(item).startswith(filename), - tracemalloc.take_snapshot().statistics('filename'))) - mem = stat.size / _TWO_20 - if timestamps: - return mem, time.time() - else: - return mem - - def ps_util_tool(): - # .. cross-platform but but requires psutil .. - process = psutil.Process(pid) - try: - # avoid using get_memory_info since it does not exists - # in psutil > 2.0 and accessing it will cause exception. - meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') \ - else 'get_memory_info' - mem = getattr(process, meminfo_attr)()[0] / _TWO_20 - if include_children: - mem += sum(_get_child_memory(process, meminfo_attr)) - if timestamps: - return mem, time.time() - else: - return mem - except psutil.AccessDenied: - pass - # continue and try to get this from ps - - def posix_tool(): - # .. scary stuff .. - if include_children: - raise NotImplementedError(( - "The psutil module is required to monitor the " - "memory usage of child processes." - )) - - warnings.warn("psutil module not found. memory_profiler will be slow") - # .. - # .. memory usage in MiB .. - # .. this should work on both Mac and Linux .. - # .. subprocess.check_output appeared in 2.7, using Popen .. - # .. for backwards compatibility .. - out = subprocess.Popen(['ps', 'v', '-p', str(pid)], - stdout=subprocess.PIPE - ).communicate()[0].split(b'\n') - try: - vsz_index = out[0].split().index(b'RSS') - mem = float(out[1].split()[vsz_index]) / 1024 - if timestamps: - return mem, time.time() - else: - return mem - except: - if timestamps: - return -1, time.time() - else: - return -1 - - if backend == 'tracemalloc' and \ - (filename is None or filename == ''): - raise RuntimeError( - 'There is no access to source file of the profiled function' - ) - - tools = {'tracemalloc': tracemalloc_tool, - 'psutil': ps_util_tool, - 'posix': posix_tool} - return tools[backend]() - - class MemTimer(Process): """ Fetch memory consumption from over a time interval @@ -458,23 +344,6 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, # .. utility functions for line-by-line .. -def _find_script(script_name): - """ Find the script. - - If the input is not a file, then $PATH will be searched. - """ - if os.path.isfile(script_name): - return script_name - path = os.getenv('PATH', os.defpath).split(os.pathsep) - for folder in path: - if not folder: - continue - fn = os.path.join(folder, script_name) - if os.path.isfile(fn): - return fn - - sys.stderr.write('Could not find script {0}\n'.format(script_name)) - raise SystemExit(1) class _TimeStamperCM(object): @@ -603,234 +472,6 @@ def show_results(self, stream=None): (function_name,) + ts[0] + ts[1] + (level,))) -class CodeMap(dict): - def __init__(self, include_children, backend): - self.include_children = include_children - self._toplevel = [] - self.backend = backend - - def add(self, code, toplevel_code=None): - if code in self: - return - - if toplevel_code is None: - filename = code.co_filename - if filename.endswith((".pyc", ".pyo")): - filename = filename[:-1] - if not os.path.exists(filename): - print('ERROR: Could not find file ' + filename) - if filename.startswith(("ipython-input", " 0: - self.enable_count -= 1 - if self.enable_count == 0: - self.disable() - - def trace_memory_usage(self, frame, event, arg): - """Callback for sys.settrace""" - if frame.f_code in self.code_map: - if event == 'call': - # "call" event just saves the lineno but not the memory - self.prevlines.append(frame.f_lineno) - elif event == 'line': - # trace needs current line and previous line - self.code_map.trace(frame.f_code, self.prevlines[-1], self.prev_lineno) - # saving previous line - self.prev_lineno = self.prevlines[-1] - self.prevlines[-1] = frame.f_lineno - elif event == 'return': - lineno = self.prevlines.pop() - self.code_map.trace(frame.f_code, lineno, self.prev_lineno) - self.prev_lineno = lineno - - if self._original_trace_function is not None: - self._original_trace_function(frame, event, arg) - - return self.trace_memory_usage - - def trace_max_mem(self, frame, event, arg): - # run into PDB as soon as memory is higher than MAX_MEM - if event in ('line', 'return') and frame.f_code in self.code_map: - c = _get_memory(-1, self.backend, filename=frame.f_code.co_filename) - if c >= self.max_mem: - t = ('Current memory {0:.2f} MiB exceeded the ' - 'maximum of {1:.2f} MiB\n'.format(c, self.max_mem)) - sys.stdout.write(t) - sys.stdout.write('Stepping into the debugger \n') - frame.f_lineno -= 2 - p = pdb.Pdb() - p.quitting = False - p.stopframe = frame - p.returnframe = None - p.stoplineno = frame.f_lineno - 3 - p.botframe = None - return p.trace_dispatch - - if self._original_trace_function is not None: - (self._original_trace_function)(frame, event, arg) - - return self.trace_max_mem - - def __enter__(self): - self.enable_by_count() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.disable_by_count() - - def enable(self): - self._original_trace_function = sys.gettrace() - if self.max_mem is not None: - sys.settrace(self.trace_max_mem) - else: - sys.settrace(self.trace_memory_usage) - - def disable(self): - sys.settrace(self._original_trace_function) - - -def show_results(prof, stream=None, precision=1): - if stream is None: - stream = sys.stdout - template = '{0:>6} {1:>12} {2:>12} {3:<}' - - for (filename, lines) in prof.code_map.items(): - header = template.format('Line #', 'Mem usage', 'Increment', - 'Line Contents') - - stream.write(u'Filename: ' + filename + '\n\n') - stream.write(header + u'\n') - stream.write(u'=' * len(header) + '\n') - - all_lines = linecache.getlines(filename) - - float_format = u'{0}.{1}f'.format(precision + 4, precision) - template_mem = u'{0:' + float_format + '} MiB' - for (lineno, mem) in lines: - if mem: - inc = mem[0] - mem = mem[1] - mem = template_mem.format(mem) - inc = template_mem.format(inc) - else: - mem = u'' - inc = u'' - tmp = template.format(lineno, mem, inc, all_lines[lineno - 1]) - stream.write(to_str(tmp)) - stream.write(u'\n\n') - - def _func_exec(stmt, ns): # helper for magic_memit, just a function proxy for the exec # statement @@ -1113,7 +754,7 @@ def profile(func=None, stream=None, precision=1, backend='psutil'): Decorator that will run the function and print a line-by-line profile """ backend = choose_backend(backend) - if backend == 'tracemalloc' and has_tracemalloc: + if backend == 'tracemalloc' and HAS_TRACEMALLOC: if not tracemalloc.is_tracing(): tracemalloc.start() if func is not None: @@ -1133,58 +774,32 @@ def inner_wrapper(f): return inner_wrapper -def choose_backend(new_backend=None): - """ - Function that tries to setup backend, chosen by user, and if failed, - setup one of the allowable backends - """ - - _backend = 'no_backend' - all_backends = [ - ('psutil', True), - ('posix', os.name == 'posix'), - ('tracemalloc', has_tracemalloc), - ] - backends_indices = dict((b[0], i) for i, b in enumerate(all_backends)) - - if new_backend is not None: - all_backends.insert(0, all_backends.pop(backends_indices[new_backend])) - - for n_backend, is_available in all_backends: - if is_available: - _backend = n_backend - break - if _backend != new_backend and new_backend is not None: - warnings.warn('{0} can not be used, {1} used instead'.format( - new_backend, _backend)) - return _backend - - # Insert in the built-ins to have profile # globally defined (global variables is not enough # for all cases, e.g. a script that imports another # script where @profile is used) def exec_with_profiler(filename, profiler, backend, passed_args=[]): - from runpy import run_module builtins.__dict__['profile'] = profiler - ns = dict(_CLEAN_GLOBALS, - profile=profiler, - # Make sure the __file__ variable is usable - # by the script we're profiling - __file__=filename) + ns = dict( + _CLEAN_GLOBALS, + profile=profiler, + # Make sure the __file__ variable is usable + # by the script we're profiling + __file__=filename + ) # Make sure the script's directory in on sys.path # credit to line_profiler - sys.path.insert(0, os.path.dirname(script_filename)) + sys.path.insert(0, os.path.dirname(filename)) _backend = choose_backend(backend) sys.argv = [filename] + passed_args try: - if _backend == 'tracemalloc' and has_tracemalloc: + if _backend == 'tracemalloc' and HAS_TRACEMALLOC: tracemalloc.start() with open(filename) as f: exec(compile(f.read(), filename, 'exec'), ns, ns) finally: - if has_tracemalloc and tracemalloc.is_tracing(): + if HAS_TRACEMALLOC and tracemalloc.is_tracing(): tracemalloc.stop() @@ -1197,12 +812,12 @@ def run_module_with_profiler(module, profiler, backend, passed_args=[]): if PY2: run_module(module, run_name="__main__", init_globals=ns) else: - if _backend == 'tracemalloc' and has_tracemalloc: + if _backend == 'tracemalloc' and HAS_TRACEMALLOC: tracemalloc.start() try: run_module(module, run_name="__main__", init_globals=ns) finally: - if has_tracemalloc and tracemalloc.is_tracing(): + if HAS_TRACEMALLOC and tracemalloc.is_tracing(): tracemalloc.stop() @@ -1236,61 +851,3 @@ def write(self, msg, level=logging.INFO): def flush(self): for handler in self.logger.handlers: handler.flush() - - -if __name__ == '__main__': - from argparse import ArgumentParser, REMAINDER - - parser = ArgumentParser(usage=_CMD_USAGE) - parser.add_argument('--version', action='version', version=__version__) - parser.add_argument( - '--pdb-mmem', dest='max_mem', metavar='MAXMEM', - type=float, action='store', - help='step into the debugger when memory exceeds MAXMEM') - parser.add_argument( - '--precision', dest='precision', type=int, - action='store', default=3, - help='precision of memory output in number of significant digits') - parser.add_argument('-o', dest='out_filename', type=str, - action='store', default=None, - help='path to a file where results will be written') - parser.add_argument('--timestamp', dest='timestamp', default=False, - action='store_true', - help='''print timestamp instead of memory measurement for - decorated functions''') - parser.add_argument('--backend', dest='backend', type=str, action='store', - choices=['tracemalloc', 'psutil', 'posix'], default='psutil', - help='backend using for getting memory info ' - '(one of the {tracemalloc, psutil, posix})') - parser.add_argument("program", nargs=REMAINDER, - help='python script or module followed by command line arguements to run') - args = parser.parse_args() - - if len(args.program) == 0: - print("A program to run must be provided. Use -h for help") - sys.exit(1) - - target = args.program[0] - script_args = args.program[1:] - _backend = choose_backend(args.backend) - if args.timestamp: - prof = TimeStamper(_backend) - else: - prof = LineProfiler(max_mem=args.max_mem, backend=_backend) - - try: - if args.program[0].endswith('.py'): - script_filename = _find_script(args.program[0]) - exec_with_profiler(script_filename, prof, args.backend, script_args) - else: - run_module_with_profiler(target, prof, args.backend, script_args) - finally: - if args.out_filename is not None: - out_file = open(args.out_filename, "a") - else: - out_file = sys.stdout - - if args.timestamp: - prof.show_results(stream=out_file) - else: - show_results(prof, precision=args.precision, stream=out_file) diff --git a/memory_profiler/__main__.py b/memory_profiler/__main__.py new file mode 100644 index 0000000..4cca042 --- /dev/null +++ b/memory_profiler/__main__.py @@ -0,0 +1,115 @@ +import os +import sys +from argparse import ArgumentParser, REMAINDER + +from . import exec_with_profiler, run_module_with_profiler, TimeStamper, __version__ +from .line_profiler import LineProfiler +from .utils import choose_backend, show_results + + +_CMD_USAGE = "python -m memory_profiler script_file.py" + + +def _find_script(script_name): + """ Find the script. + + If the input is not a file, then $PATH will be searched. + """ + if os.path.isfile(script_name): + return script_name + path = os.getenv('PATH', os.defpath).split(os.pathsep) + for folder in path: + if not folder: + continue + fn = os.path.join(folder, script_name) + if os.path.isfile(fn): + return fn + + sys.stderr.write('Could not find script {0}\n'.format(script_name)) + raise SystemExit(1) + + +if __name__ == '__main__': + parser = ArgumentParser(usage=_CMD_USAGE) + parser.add_argument('--version', action='version', version=__version__) + parser.add_argument( + '--pdb-mmem', + dest='max_mem', + metavar='MAXMEM', + type=float, + action='store', + help='step into the debugger when memory exceeds MAXMEM' + ) + parser.add_argument( + '--precision', + dest='precision', + type=int, + action='store', + default=3, + help='precision of memory output in number of significant digits' + ) + parser.add_argument( + '-o', + dest='out_filename', + type=str, + action='store', + default=None, + help='path to a file where results will be written' + ) + parser.add_argument( + '--timestamp', + dest='timestamp', + default=False, + action='store_true', + help='''print timestamp instead of memory measurement for + decorated functions''' + ) + parser.add_argument( + '--backend', + dest='backend', + type=str, + action='store', + choices=['tracemalloc', 'psutil', 'posix'], + default='psutil', + help='backend using for getting memory info ' + '(one of the {tracemalloc, psutil, posix})' + ) + parser.add_argument( + "program", + nargs=REMAINDER, + help='python script or module followed by ' + 'command line arguments to run' + ) + args = parser.parse_args() + + if len(args.program) == 0: + print("A program to run must be provided. Use -h for help") + sys.exit(1) + + target = args.program[0] + script_args = args.program[1:] + + _backend = choose_backend(args.backend) + if args.timestamp: + prof = TimeStamper(_backend) + else: + prof = LineProfiler(max_mem=args.max_mem, backend=_backend) + + try: + if args.program[0].endswith('.py'): + script_filename = _find_script(args.program[0]) + exec_with_profiler( + script_filename, prof, args.backend, script_args + ) + else: + run_module_with_profiler(target, prof, args.backend, script_args) + finally: + if args.out_filename is not None: + out_file = open(args.out_filename, "a") + else: + out_file = sys.stdout + + if args.timestamp: + prof.show_results(stream=out_file) + else: + show_results(prof, precision=args.precision, stream=out_file) diff --git a/memory_profiler/code_map.py b/memory_profiler/code_map.py new file mode 100644 index 0000000..c5826fc --- /dev/null +++ b/memory_profiler/code_map.py @@ -0,0 +1,69 @@ +import inspect +import os.path + +from .utils import get_memory + + +class CodeMap(dict): + def __init__(self, include_children, backend): + self.include_children = include_children + self._toplevel = [] + self.backend = backend + + def add(self, code, toplevel_code=None): + if code in self: + return + + if toplevel_code is None: + filename = code.co_filename + if filename.endswith((".pyc", ".pyo")): + filename = filename[:-1] + if not os.path.exists(filename): + print('ERROR: Could not find file ' + filename) + if filename.startswith(("ipython-input", " 0: + self.enable_count -= 1 + if self.enable_count == 0: + self.disable() + + def trace_memory_usage(self, frame, event, arg): + """Callback for sys.settrace""" + if frame.f_code in self.code_map: + if event == 'call': + # "call" event just saves the lineno but not the memory + self.prevlines.append(frame.f_lineno) + elif event == 'line': + # trace needs current line and previous line + self.code_map.trace(frame.f_code, self.prevlines[-1], self.prev_lineno) + # saving previous line + self.prev_lineno = self.prevlines[-1] + self.prevlines[-1] = frame.f_lineno + elif event == 'return': + lineno = self.prevlines.pop() + self.code_map.trace(frame.f_code, lineno, self.prev_lineno) + self.prev_lineno = lineno + + if self._original_trace_function is not None: + self._original_trace_function(frame, event, arg) + + return self.trace_memory_usage + + def trace_max_mem(self, frame, event, arg): + # run into PDB as soon as memory is higher than MAX_MEM + if event in ('line', 'return') and frame.f_code in self.code_map: + c = get_memory(-1, self.backend, filename=frame.f_code.co_filename) + if c >= self.max_mem: + t = ('Current memory {0:.2f} MiB exceeded the ' + 'maximum of {1:.2f} MiB\n'.format(c, self.max_mem)) + sys.stdout.write(t) + sys.stdout.write('Stepping into the debugger \n') + frame.f_lineno -= 2 + p = pdb.Pdb() + p.quitting = False + p.stopframe = frame + p.returnframe = None + p.stoplineno = frame.f_lineno - 3 + p.botframe = None + return p.trace_dispatch + + if self._original_trace_function is not None: + (self._original_trace_function)(frame, event, arg) + + return self.trace_max_mem + + def __enter__(self): + self.enable_by_count() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.disable_by_count() + + def enable(self): + self._original_trace_function = sys.gettrace() + if self.max_mem is not None: + sys.settrace(self.trace_max_mem) + else: + sys.settrace(self.trace_memory_usage) + + def disable(self): + sys.settrace(self._original_trace_function) + + +def get_profile_wrapper(func, precision, backend, stream): + @wraps(func) + def wrapper(*args, **kwargs): + prof = LineProfiler(backend=backend) + val = prof(func)(*args, **kwargs) + show_results(prof, stream=stream, precision=precision) + return val + + return wrapper diff --git a/memory_profiler/utils.py b/memory_profiler/utils.py new file mode 100644 index 0000000..27b8861 --- /dev/null +++ b/memory_profiler/utils.py @@ -0,0 +1,183 @@ +import linecache +import os +import sys +import warnings +import subprocess +import time + +import psutil + +from .common import HAS_TRACEMALLOC, PY2, TWO_20 + +if HAS_TRACEMALLOC: + import tracemalloc + + +if PY2: + def to_str(x): + return x + + from future_builtins import filter +else: + def to_str(x): + return str(x) + + +def show_results(prof, stream=None, precision=1): + if stream is None: + stream = sys.stdout + template = '{0:>6} {1:>12} {2:>12} {3:<}' + + for (filename, lines) in prof.code_map.items(): + header = template.format('Line #', 'Mem usage', 'Increment', + 'Line Contents') + + stream.write(u'Filename: ' + filename + '\n\n') + stream.write(header + u'\n') + stream.write(u'=' * len(header) + '\n') + + all_lines = linecache.getlines(filename) + + float_format = u'{0}.{1}f'.format(precision + 4, precision) + template_mem = u'{0:' + float_format + '} MiB' + for (lineno, mem) in lines: + if mem: + inc = mem[0] + mem = mem[1] + mem = template_mem.format(mem) + inc = template_mem.format(inc) + else: + mem = u'' + inc = u'' + tmp = template.format(lineno, mem, inc, all_lines[lineno - 1]) + stream.write(to_str(tmp)) + stream.write(u'\n\n') + + +def choose_backend(new_backend=None): + """ + Function that tries to setup backend, chosen by user, and if failed, + setup one of the allowable backends + """ + + _backend = 'no_backend' + all_backends = [ + ('psutil', True), + ('posix', os.name == 'posix'), + ('tracemalloc', HAS_TRACEMALLOC), + ] + backends_indices = dict((b[0], i) for i, b in enumerate(all_backends)) + + if new_backend is not None: + all_backends.insert(0, all_backends.pop(backends_indices[new_backend])) + + for n_backend, is_available in all_backends: + if is_available: + _backend = n_backend + break + if _backend != new_backend and new_backend is not None: + warnings.warn('{0} can not be used, {1} used instead'.format( + new_backend, _backend)) + return _backend + + +def get_child_memory(process, meminfo_attr=None): + """ + Returns a generator that yields memory for all child processes. + """ + # Convert a pid to a process + if isinstance(process, int): + if process == -1: process = os.getpid() + process = psutil.Process(process) + + if not meminfo_attr: + # Use the psutil 2.0 attr if the older version isn't passed in. + meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') else 'get_memory_info' + + # Select the psutil function get the children similar to how we selected + # the memory_info attr (a change from excepting the AttributeError). + children_attr = 'children' if hasattr(process, 'children') else 'get_children' + + # Loop over the child processes and yield their memory + try: + for child in getattr(process, children_attr)(recursive=True): + yield getattr(child, meminfo_attr)()[0] / TWO_20 + except (psutil.NoSuchProcess, psutil.AccessDenied): + # https://github.com/fabianp/memory_profiler/issues/71 + yield 0.0 + + +def get_memory(pid, backend, timestamps=False, include_children=False, filename=None): + # .. low function to get memory consumption .. + if pid == -1: + pid = os.getpid() + + def tracemalloc_tool(): + # .. cross-platform but but requires Python 3.4 or higher .. + stat = next(filter(lambda item: str(item).startswith(filename), + tracemalloc.take_snapshot().statistics('filename'))) + mem = stat.size / TWO_20 + if timestamps: + return mem, time.time() + else: + return mem + + def ps_util_tool(): + # .. cross-platform but but requires psutil .. + process = psutil.Process(pid) + try: + # avoid using get_memory_info since it does not exists + # in psutil > 2.0 and accessing it will cause exception. + meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') \ + else 'get_memory_info' + mem = getattr(process, meminfo_attr)()[0] / TWO_20 + if include_children: + mem += sum(get_child_memory(process, meminfo_attr)) + if timestamps: + return mem, time.time() + else: + return mem + except psutil.AccessDenied: + pass + # continue and try to get this from ps + + def posix_tool(): + # .. scary stuff .. + if include_children: + raise NotImplementedError(( + "The psutil module is required to monitor the " + "memory usage of child processes." + )) + + warnings.warn("psutil module not found. memory_profiler will be slow") + # .. + # .. memory usage in MiB .. + # .. this should work on both Mac and Linux .. + # .. subprocess.check_output appeared in 2.7, using Popen .. + # .. for backwards compatibility .. + out = subprocess.Popen(['ps', 'v', '-p', str(pid)], + stdout=subprocess.PIPE + ).communicate()[0].split(b'\n') + try: + vsz_index = out[0].split().index(b'RSS') + mem = float(out[1].split()[vsz_index]) / 1024 + if timestamps: + return mem, time.time() + else: + return mem + except: + if timestamps: + return -1, time.time() + else: + return -1 + + if backend == 'tracemalloc' and \ + (filename is None or filename == ''): + raise RuntimeError( + 'There is no access to source file of the profiled function' + ) + + tools = {'tracemalloc': tracemalloc_tool, + 'psutil': ps_util_tool, + 'posix': posix_tool} + return tools[backend]() diff --git a/setup.py b/setup.py index 4050cd4..b529063 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,7 @@ import os import io import re -import sys -from setuptools import setup +from setuptools import find_packages, setup # https://packaging.python.org/guides/single-sourcing-package-version/ @@ -22,14 +21,6 @@ def find_version(*file_paths): return version_match.group(1) -async_modules = [] -if sys.version_info > (3, 3): - if sys.version_info < (3, 5): - async_modules = ['_aio_34'] - else: - async_modules = ['_aio_35'] - - CLASSIFIERS = """\ Development Status :: 5 - Production/Stable Intended Audience :: Science/Research @@ -53,13 +44,14 @@ def find_version(*file_paths): name='memory_profiler', description='A module for monitoring memory usage of a python program', long_description=open('README.rst').read(), - version=find_version("memory_profiler.py"), + version=find_version("memory_profiler", "__init__.py"), author='Fabian Pedregosa', author_email='f@bianp.net', url='https://github.com/pythonprofilers/memory_profiler', - py_modules=['memory_profiler', 'mprof'] + async_modules, + py_modules=['mprof'], + packages=find_packages(include='memory_profiler'), entry_points={ - 'console_scripts' : ['mprof = mprof:main'], + 'console_scripts': ['mprof = mprof:main'], }, install_requires=['psutil'], classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], From c33afb0371950b3218bbfefcc743e67208458df1 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sun, 5 Jan 2020 23:06:07 +0300 Subject: [PATCH 05/17] Add coroutine support --- _aio_34.py | 35 ----------------------------------- memory_profiler/__init__.py | 18 ++++++++---------- memory_profiler/_aio_34.py | 35 +++++++++++++++++++++++++++++++++++ memory_profiler/_aio_35.py | 33 +++++++++++++++++++++++++++++++++ memory_profiler/common.py | 3 +++ 5 files changed, 79 insertions(+), 45 deletions(-) delete mode 100644 _aio_34.py create mode 100644 memory_profiler/_aio_34.py create mode 100644 memory_profiler/_aio_35.py diff --git a/_aio_34.py b/_aio_34.py deleted file mode 100644 index 874197a..0000000 --- a/_aio_34.py +++ /dev/null @@ -1,35 +0,0 @@ -from asyncio import coroutine, iscoroutinefunction -from contextlib import contextmanager -from functools import wraps - -from memory_profiler import LineProfiler, show_results - - -class CoroLineProfiler(LineProfiler): - @contextmanager - def count_contextmgr(self): - self.enable_by_count() - try: - yield - finally: - self.disable_by_count() - - def wrap_function(self, func): - if iscoroutinefunction(func): - @coroutine - def f(*args, **kwargs): - with self.count_contextmgr(): - yield from func(*args, **kwargs) - return f - else: - return super(CoroLineProfiler, self).wrap_function(func) - - -def _get_coro_wrapper(coro, backend, stream, precision): - @wraps(coro) - @coroutine - def wrapper(*args, **kwargs): - prof = CoroLineProfiler(backend=backend) - val = yield from prof(coro)(*args, **kwargs) - show_results(prof, stream=stream, precision=precision) - return val diff --git a/memory_profiler/__init__.py b/memory_profiler/__init__.py index 934ad15..feac67c 100644 --- a/memory_profiler/__init__.py +++ b/memory_profiler/__init__.py @@ -17,14 +17,19 @@ import warnings import contextlib -from .common import PY2, HAS_TRACEMALLOC +from .common import PY2, PY34, PY35, HAS_TRACEMALLOC from .utils import ( show_results, choose_backend, get_memory as _get_memory, get_child_memory as _get_child_memory, ) -from .line_profiler import LineProfiler +from .line_profiler import LineProfiler, get_profile_wrapper + +if PY34: + from ._aio_34 import get_profile_wrapper +elif PY35: + from ._aio_35 import get_profile_wrapper if sys.platform == "win32": @@ -758,14 +763,7 @@ def profile(func=None, stream=None, precision=1, backend='psutil'): if not tracemalloc.is_tracing(): tracemalloc.start() if func is not None: - @wraps(func) - def wrapper(*args, **kwargs): - prof = LineProfiler(backend=backend) - val = prof(func)(*args, **kwargs) - show_results(prof, stream=stream, precision=precision) - return val - - return wrapper + return get_profile_wrapper(func, precision, backend, stream) else: def inner_wrapper(f): return profile(f, stream=stream, precision=precision, diff --git a/memory_profiler/_aio_34.py b/memory_profiler/_aio_34.py new file mode 100644 index 0000000..86c2887 --- /dev/null +++ b/memory_profiler/_aio_34.py @@ -0,0 +1,35 @@ +from asyncio import coroutine, iscoroutinefunction +from functools import wraps + +from .line_profiler import ( + get_profile_wrapper as default_profile_wrapper, + LineProfiler, +) +from .utils import show_results + + +class CoroLineProfiler(LineProfiler): + def wrap_function(self, func): + if iscoroutinefunction(func): + @coroutine + def f(*args, **kwargs): + with self.count_ctxmgr(): + yield from func(*args, **kwargs) + return f + else: + return super().wrap_function(func) + + +def get_profile_wrapper(func, precision, backend, stream): + if iscoroutinefunction(func): + @wraps(func) + @coroutine + def wrapper(*args, **kwargs): + prof = CoroLineProfiler(backend=backend) + val = yield from prof(func)(*args, **kwargs) + show_results(prof, stream=stream, precision=precision) + return val + else: + wrapper = default_profile_wrapper(func, precision, backend, stream) + + return wrapper diff --git a/memory_profiler/_aio_35.py b/memory_profiler/_aio_35.py new file mode 100644 index 0000000..b14d093 --- /dev/null +++ b/memory_profiler/_aio_35.py @@ -0,0 +1,33 @@ +from functools import wraps +from inspect import iscoroutinefunction + +from .line_profiler import ( + get_profile_wrapper as default_profile_wrapper, + LineProfiler, +) +from .utils import show_results + + +class CoroLineProfiler(LineProfiler): + def wrap_function(self, func): + if iscoroutinefunction(func): + async def f(*args, **kwargs): + with self.count_ctxmgr(): + return await func(*args, **kwargs) + return f + else: + return super().wrap_function(func) + + +def get_profile_wrapper(func, precision, backend, stream): + if iscoroutinefunction(func): + @wraps(func) + async def wrapper(*args, **kwargs): + prof = CoroLineProfiler(backend=backend) + val = await prof(func)(*args, **kwargs) + show_results(prof, stream=stream, precision=precision) + return val + else: + wrapper = default_profile_wrapper(func, precision, backend, stream) + + return wrapper diff --git a/memory_profiler/common.py b/memory_profiler/common.py index c5f05f8..e6ed9fa 100644 --- a/memory_profiler/common.py +++ b/memory_profiler/common.py @@ -3,6 +3,9 @@ PY2 = sys.version_info[0] == 2 +PY34 = (3, 4) < sys.version_info < (3, 5) +PY35 = (3, 5) < sys.version_info + try: import tracemalloc # noqa except ImportError: From 76a433544c758a4ce92c08494d95e9d890da6314 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sun, 5 Jan 2020 23:21:32 +0300 Subject: [PATCH 06/17] Fix setup package lookup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b529063..effd3f3 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def find_version(*file_paths): author_email='f@bianp.net', url='https://github.com/pythonprofilers/memory_profiler', py_modules=['mprof'], - packages=find_packages(include='memory_profiler'), + packages=find_packages(include=('memory_profiler',)), entry_points={ 'console_scripts': ['mprof = mprof:main'], }, From 79abbb617782d92848f9a802577712d9ae80bc14 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sun, 5 Jan 2020 23:28:37 +0300 Subject: [PATCH 07/17] Import CodeMap in __init__ for backward compatibility --- memory_profiler/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/memory_profiler/__init__.py b/memory_profiler/__init__.py index ec337ff..403bc84 100644 --- a/memory_profiler/__init__.py +++ b/memory_profiler/__init__.py @@ -17,6 +17,7 @@ import warnings import contextlib +from .code_map import CodeMap from .common import PY2, PY34, PY35, HAS_TRACEMALLOC from .utils import ( show_results, From cb7ecdada7305019277a9d38f1bf91c578ede093 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 6 Jan 2020 01:05:44 +0300 Subject: [PATCH 08/17] Add count_ctxmgr --- memory_profiler/line_profiler.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/memory_profiler/line_profiler.py b/memory_profiler/line_profiler.py index 7806e82..40d5b7c 100644 --- a/memory_profiler/line_profiler.py +++ b/memory_profiler/line_profiler.py @@ -1,6 +1,7 @@ import pdb import sys import warnings +from contextlib import contextmanager from functools import wraps from .code_map import CodeMap @@ -48,27 +49,29 @@ def add_function(self, func): else: self.code_map.add(code) + @contextmanager + def count_ctxmgr(self): + self.enable_by_count() + try: + yield + finally: + self.disable_by_count() + def wrap_function(self, func): """ Wrap a function to profile it. """ def f(*args, **kwds): - self.enable_by_count() - try: + with self.count_ctxmgr(): return func(*args, **kwds) - finally: - self.disable_by_count() return f def runctx(self, cmd, globals, locals): """ Profile a single executable statement in the given namespaces. """ - self.enable_by_count() - try: + with self.count_ctxmgr(): exec(cmd, globals, locals) - finally: - self.disable_by_count() return self def enable_by_count(self): From 300b89dbcb1a3718279303a1c3b7e577c0543734 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Tue, 7 Jan 2020 19:30:25 +0300 Subject: [PATCH 09/17] Remove redundant implementation in new syntax --- memory_profiler/__init__.py | 6 ++--- memory_profiler/_aio_35.py | 33 ----------------------- memory_profiler/{_aio_34.py => _async.py} | 0 memory_profiler/common.py | 3 +-- 4 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 memory_profiler/_aio_35.py rename memory_profiler/{_aio_34.py => _async.py} (100%) diff --git a/memory_profiler/__init__.py b/memory_profiler/__init__.py index 403bc84..c334c4e 100644 --- a/memory_profiler/__init__.py +++ b/memory_profiler/__init__.py @@ -18,7 +18,7 @@ import contextlib from .code_map import CodeMap -from .common import PY2, PY34, PY35, HAS_TRACEMALLOC +from .common import PY2, PY34, HAS_TRACEMALLOC from .utils import ( show_results, choose_backend, @@ -28,9 +28,7 @@ from .line_profiler import LineProfiler, get_profile_wrapper if PY34: - from ._aio_34 import get_profile_wrapper -elif PY35: - from ._aio_35 import get_profile_wrapper + from ._async import get_profile_wrapper if sys.platform == "win32": diff --git a/memory_profiler/_aio_35.py b/memory_profiler/_aio_35.py deleted file mode 100644 index b14d093..0000000 --- a/memory_profiler/_aio_35.py +++ /dev/null @@ -1,33 +0,0 @@ -from functools import wraps -from inspect import iscoroutinefunction - -from .line_profiler import ( - get_profile_wrapper as default_profile_wrapper, - LineProfiler, -) -from .utils import show_results - - -class CoroLineProfiler(LineProfiler): - def wrap_function(self, func): - if iscoroutinefunction(func): - async def f(*args, **kwargs): - with self.count_ctxmgr(): - return await func(*args, **kwargs) - return f - else: - return super().wrap_function(func) - - -def get_profile_wrapper(func, precision, backend, stream): - if iscoroutinefunction(func): - @wraps(func) - async def wrapper(*args, **kwargs): - prof = CoroLineProfiler(backend=backend) - val = await prof(func)(*args, **kwargs) - show_results(prof, stream=stream, precision=precision) - return val - else: - wrapper = default_profile_wrapper(func, precision, backend, stream) - - return wrapper diff --git a/memory_profiler/_aio_34.py b/memory_profiler/_async.py similarity index 100% rename from memory_profiler/_aio_34.py rename to memory_profiler/_async.py diff --git a/memory_profiler/common.py b/memory_profiler/common.py index e6ed9fa..55ae413 100644 --- a/memory_profiler/common.py +++ b/memory_profiler/common.py @@ -3,8 +3,7 @@ PY2 = sys.version_info[0] == 2 -PY34 = (3, 4) < sys.version_info < (3, 5) -PY35 = (3, 5) < sys.version_info +PY34 = (3, 4) < sys.version_info try: import tracemalloc # noqa From 1ee8e036d1f0eebd1077d952985bbe77baa42e0c Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Tue, 7 Jan 2020 19:33:31 +0300 Subject: [PATCH 10/17] Add test_async.py --- Makefile | 7 ++++++- test/test_async.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 test/test_async.py diff --git a/Makefile b/Makefile index e95a767..8943f84 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ PYTHON ?= python +PY_34 ?= $(shell ! python -c \ + 'import sys; print((3, 4) < sys.version_info)') -.PHONY: test +.PHONY: test develop test: $(PYTHON) -m memory_profiler test/test_func.py @@ -18,6 +20,9 @@ test: $(PYTHON) test/test_exception.py $(PYTHON) test/test_exit_code.py $(PYTHON) test/test_mprof.py + if [ $(PY_34) = True ]; then \ + $(PYTHON) test/test_aio_34.py; \ + fi develop: pip install -e . diff --git a/test/test_async.py b/test/test_async.py new file mode 100644 index 0000000..a2301e2 --- /dev/null +++ b/test/test_async.py @@ -0,0 +1,18 @@ +import asyncio + +from memory_profiler import profile + + +@profile +@asyncio.coroutine +def my_func(): + a = [1] * (10 ** 6) + b = [2] * (2 * 10 ** 7) + print('kek') + yield from asyncio.sleep(1e-2) + del b + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(my_func()) From 1648c6b8423fd3efba86101fc721d4128f06d9ec Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Tue, 7 Jan 2020 19:36:57 +0300 Subject: [PATCH 11/17] Fix async test filename --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8943f84..163cac0 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test: $(PYTHON) test/test_exit_code.py $(PYTHON) test/test_mprof.py if [ $(PY_34) = True ]; then \ - $(PYTHON) test/test_aio_34.py; \ + $(PYTHON) test/test_async.py; \ fi develop: From 625021f72db03c02f0d8ab9c74749e19cce7f09e Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 13 Jan 2020 03:01:01 +0300 Subject: [PATCH 12/17] Get rid of unnecessary print --- test/test_async.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_async.py b/test/test_async.py index a2301e2..bfafa79 100644 --- a/test/test_async.py +++ b/test/test_async.py @@ -8,7 +8,6 @@ def my_func(): a = [1] * (10 ** 6) b = [2] * (2 * 10 ** 7) - print('kek') yield from asyncio.sleep(1e-2) del b From 205f901c1768cac001635d4c8aad396b03852f1a Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sat, 13 Jun 2020 20:01:38 +0300 Subject: [PATCH 13/17] Ignore pyenv version file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b0ec204..9c42d3f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ mprofile_*.dat # virtual environment venv/ +.python-version From e8d6bf7e8326cf9688703ad1243e9cea9a2c132d Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sat, 13 Jun 2020 21:02:03 +0300 Subject: [PATCH 14/17] Keep memory_profiler in single file, drop support of python <3.4 --- Makefile | 4 +- .../__init__.py => memory_profiler.py | 573 ++++++++++++++++-- memory_profiler/__main__.py | 115 ---- memory_profiler/_async.py | 35 -- memory_profiler/code_map.py | 69 --- memory_profiler/common.py | 15 - memory_profiler/line_profiler.py | 163 ----- memory_profiler/utils.py | 183 ------ 8 files changed, 520 insertions(+), 637 deletions(-) rename memory_profiler/__init__.py => memory_profiler.py (59%) delete mode 100644 memory_profiler/__main__.py delete mode 100644 memory_profiler/_async.py delete mode 100644 memory_profiler/code_map.py delete mode 100644 memory_profiler/common.py delete mode 100644 memory_profiler/line_profiler.py delete mode 100644 memory_profiler/utils.py diff --git a/Makefile b/Makefile index 163cac0..ecf4498 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,7 @@ test: $(PYTHON) test/test_exception.py $(PYTHON) test/test_exit_code.py $(PYTHON) test/test_mprof.py - if [ $(PY_34) = True ]; then \ - $(PYTHON) test/test_async.py; \ - fi + $(PYTHON) test/test_async.py develop: pip install -e . diff --git a/memory_profiler/__init__.py b/memory_profiler.py similarity index 59% rename from memory_profiler/__init__.py rename to memory_profiler.py index c527fc4..e13786b 100644 --- a/memory_profiler/__init__.py +++ b/memory_profiler.py @@ -5,31 +5,22 @@ __version__ = '0.57.0' +_CMD_USAGE = "python -m memory_profiler script_file.py" -from functools import wraps +from asyncio import coroutine, iscoroutinefunction +from contextlib import contextmanager +from functools import partial, wraps +import builtins import inspect +import linecache import logging import os +import pdb import subprocess import sys import time import traceback import warnings -import contextlib - -from .code_map import CodeMap -from .common import PY2, PY34, HAS_TRACEMALLOC -from .utils import ( - show_results, - choose_backend, - get_memory as _get_memory, - get_child_memory as _get_child_memory, -) -from .line_profiler import LineProfiler, get_profile_wrapper - -if PY34: - from ._async import get_profile_wrapper - if sys.platform == "win32": # any value except signal.CTRL_C_EVENT and signal.CTRL_BREAK_EVENT @@ -54,17 +45,17 @@ line_cell_magic = lambda func: func magics_class = lambda cls: cls -if PY2: - import __builtin__ as builtins - to_str = lambda x: x - from future_builtins import filter -else: - import builtins - to_str = lambda x: str(x) +_TWO_20 = float(2 ** 20) -if HAS_TRACEMALLOC: + +# .. get available packages .. +try: import tracemalloc + has_tracemalloc = True +except ImportError: + has_tracemalloc = False + class MemitResult(object): """memit magic run details. @@ -91,6 +82,108 @@ def _repr_pretty_(self, p, cycle): p.text(u'') +def _get_child_memory(process, meminfo_attr=None): + """ + Returns a generator that yields memory for all child processes. + """ + # Convert a pid to a process + if isinstance(process, int): + if process == -1: process = os.getpid() + process = psutil.Process(process) + + if not meminfo_attr: + # Use the psutil 2.0 attr if the older version isn't passed in. + meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') else 'get_memory_info' + + # Select the psutil function get the children similar to how we selected + # the memory_info attr (a change from excepting the AttributeError). + children_attr = 'children' if hasattr(process, 'children') else 'get_children' + + # Loop over the child processes and yield their memory + try: + for child in getattr(process, children_attr)(recursive=True): + yield getattr(child, meminfo_attr)()[0] / _TWO_20 + except (psutil.NoSuchProcess, psutil.AccessDenied): + # https://github.com/fabianp/memory_profiler/issues/71 + yield 0.0 + + +def _get_memory(pid, backend, timestamps=False, include_children=False, filename=None): + # .. low function to get memory consumption .. + if pid == -1: + pid = os.getpid() + + def tracemalloc_tool(): + # .. cross-platform but but requires Python 3.4 or higher .. + stat = next(filter(lambda item: str(item).startswith(filename), + tracemalloc.take_snapshot().statistics('filename'))) + mem = stat.size / _TWO_20 + if timestamps: + return mem, time.time() + else: + return mem + + def ps_util_tool(): + # .. cross-platform but but requires psutil .. + process = psutil.Process(pid) + try: + # avoid using get_memory_info since it does not exists + # in psutil > 2.0 and accessing it will cause exception. + meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') \ + else 'get_memory_info' + mem = getattr(process, meminfo_attr)()[0] / _TWO_20 + if include_children: + mem += sum(_get_child_memory(process, meminfo_attr)) + if timestamps: + return mem, time.time() + else: + return mem + except psutil.AccessDenied: + pass + # continue and try to get this from ps + + def posix_tool(): + # .. scary stuff .. + if include_children: + raise NotImplementedError(( + "The psutil module is required to monitor the " + "memory usage of child processes." + )) + + warnings.warn("psutil module not found. memory_profiler will be slow") + # .. + # .. memory usage in MiB .. + # .. this should work on both Mac and Linux .. + # .. subprocess.check_output appeared in 2.7, using Popen .. + # .. for backwards compatibility .. + out = subprocess.Popen(['ps', 'v', '-p', str(pid)], + stdout=subprocess.PIPE + ).communicate()[0].split(b'\n') + try: + vsz_index = out[0].split().index(b'RSS') + mem = float(out[1].split()[vsz_index]) / 1024 + if timestamps: + return mem, time.time() + else: + return mem + except: + if timestamps: + return -1, time.time() + else: + return -1 + + if backend == 'tracemalloc' and \ + (filename is None or filename == ''): + raise RuntimeError( + 'There is no access to source file of the profiled function' + ) + + tools = {'tracemalloc': tracemalloc_tool, + 'psutil': ps_util_tool, + 'posix': posix_tool} + return tools[backend]() + + class MemTimer(Process): """ Fetch memory consumption from over a time interval @@ -180,10 +273,10 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, to this file instead of stored in memory and returned at the end of the subprocess. Useful for long-running processes. Implies timestamps=True. - + max_iterations : int Limits the number of iterations (calls to the process being monitored). Relevent - when the process is a python function. + when the process is a python function. Returns ------- @@ -257,7 +350,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, raise p.join(5 * interval) - + if (n_measurements > 4) or (current_iter == max_iter) or (interval < 1e-6): break interval /= 10. @@ -348,17 +441,36 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, # .. utility functions for line-by-line .. +def _find_script(script_name): + """ Find the script. + + If the input is not a file, then $PATH will be searched. + """ + if os.path.isfile(script_name): + return script_name + path = os.getenv('PATH', os.defpath).split(os.pathsep) + for folder in path: + if not folder: + continue + fn = os.path.join(folder, script_name) + if os.path.isfile(fn): + return fn + + sys.stderr.write('Could not find script {0}\n'.format(script_name)) + raise SystemExit(1) class _TimeStamperCM(object): """Time-stamping context manager.""" - def __init__(self, timestamps, filename, backend, timestamper=None, func=None): + def __init__(self, timestamps, filename, backend, timestamper=None, func=None, + include_children=False): self.timestamps = timestamps self.filename = filename self.backend = backend self.ts = timestamper self.func = func + self.include_children = include_children def __enter__(self): if self.ts is not None: @@ -366,14 +478,16 @@ def __enter__(self): self.ts.stack[self.func].append(self.ts.current_stack_level) self.timestamps.append( - _get_memory(os.getpid(), self.backend, timestamps=True, filename=self.filename)) + _get_memory(os.getpid(), self.backend, timestamps=True, + include_children=self.include_children, filename=self.filename)) def __exit__(self, *args): if self.ts is not None: self.ts.current_stack_level -= 1 self.timestamps.append( - _get_memory(os.getpid(), self.backend, timestamps=True, filename=self.filename)) + _get_memory(os.getpid(), self.backend, timestamps=True, + include_children=self.include_children, filename=self.filename)) class TimeStamper: @@ -381,9 +495,10 @@ class TimeStamper: any decorated function. """ - def __init__(self, backend): + def __init__(self, backend, include_children=False): self.functions = {} self.backend = backend + self.include_children = include_children self.current_stack_level = -1 self.stack = {} @@ -444,7 +559,8 @@ def f(*args, **kwds): except TypeError: filename = '' timestamps = [ - _get_memory(os.getpid(), self.backend, timestamps=True, filename=filename)] + _get_memory(os.getpid(), self.backend, timestamps=True, + include_children=self.include_children, filename=filename)] self.functions[func].append(timestamps) try: with self.call_on_stack(func, *args, **kwds) as result: @@ -452,11 +568,12 @@ def f(*args, **kwds): finally: # end time timestamps.append(_get_memory(os.getpid(), self.backend, timestamps=True, + include_children=self.include_children, filename=filename)) return f - @contextlib.contextmanager + @contextmanager def call_on_stack(self, func, *args, **kwds): self.current_stack_level += 1 self.stack[func].append(self.current_stack_level) @@ -476,6 +593,252 @@ def show_results(self, stream=None): (function_name,) + ts[0] + ts[1] + (level,))) +class CodeMap(dict): + def __init__(self, include_children, backend): + self.include_children = include_children + self._toplevel = [] + self.backend = backend + + def add(self, code, toplevel_code=None): + if code in self: + return + + if toplevel_code is None: + filename = code.co_filename + if filename.endswith((".pyc", ".pyo")): + filename = filename[:-1] + if not os.path.exists(filename): + print('ERROR: Could not find file ' + filename) + if filename.startswith(("ipython-input", " 0: + self.enable_count -= 1 + if self.enable_count == 0: + self.disable() + + def trace_memory_usage(self, frame, event, arg): + """Callback for sys.settrace""" + if frame.f_code in self.code_map: + if event == 'call': + # "call" event just saves the lineno but not the memory + self.prevlines.append(frame.f_lineno) + elif event == 'line': + # trace needs current line and previous line + self.code_map.trace(frame.f_code, self.prevlines[-1], self.prev_lineno) + # saving previous line + self.prev_lineno = self.prevlines[-1] + self.prevlines[-1] = frame.f_lineno + elif event == 'return': + lineno = self.prevlines.pop() + self.code_map.trace(frame.f_code, lineno, self.prev_lineno) + self.prev_lineno = lineno + + if self._original_trace_function is not None: + self._original_trace_function(frame, event, arg) + + return self.trace_memory_usage + + def trace_max_mem(self, frame, event, arg): + # run into PDB as soon as memory is higher than MAX_MEM + if event in ('line', 'return') and frame.f_code in self.code_map: + c = _get_memory(-1, self.backend, filename=frame.f_code.co_filename) + if c >= self.max_mem: + t = ('Current memory {0:.2f} MiB exceeded the ' + 'maximum of {1:.2f} MiB\n'.format(c, self.max_mem)) + sys.stdout.write(t) + sys.stdout.write('Stepping into the debugger \n') + frame.f_lineno -= 2 + p = pdb.Pdb() + p.quitting = False + p.stopframe = frame + p.returnframe = None + p.stoplineno = frame.f_lineno - 3 + p.botframe = None + return p.trace_dispatch + + if self._original_trace_function is not None: + (self._original_trace_function)(frame, event, arg) + + return self.trace_max_mem + + def __enter__(self): + self.enable_by_count() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.disable_by_count() + + def enable(self): + self._original_trace_function = sys.gettrace() + if self.max_mem is not None: + sys.settrace(self.trace_max_mem) + else: + sys.settrace(self.trace_memory_usage) + + def disable(self): + sys.settrace(self._original_trace_function) + + +def show_results(prof, stream=None, precision=1): + if stream is None: + stream = sys.stdout + template = '{0:>6} {1:>12} {2:>12} {3:>10} {4:<}' + + for (filename, lines) in prof.code_map.items(): + header = template.format('Line #', 'Mem usage', 'Increment', 'Occurences', + 'Line Contents') + + stream.write(u'Filename: ' + filename + '\n\n') + stream.write(header + u'\n') + stream.write(u'=' * len(header) + '\n') + + all_lines = linecache.getlines(filename) + + float_format = u'{0}.{1}f'.format(precision + 4, precision) + template_mem = u'{0:' + float_format + '} MiB' + for (lineno, mem) in lines: + if mem: + inc = mem[0] + total_mem = mem[1] + total_mem = template_mem.format(total_mem) + occurences = mem[2] + inc = template_mem.format(inc) + else: + total_mem = u'' + inc = u'' + occurences = u'' + tmp = template.format(lineno, total_mem, inc, occurences, all_lines[lineno - 1]) + stream.write(tmp) + stream.write(u'\n\n') + + def _func_exec(stmt, ns): # helper for magic_memit, just a function proxy for the exec # statement @@ -758,11 +1121,29 @@ def profile(func=None, stream=None, precision=1, backend='psutil'): Decorator that will run the function and print a line-by-line profile """ backend = choose_backend(backend) - if backend == 'tracemalloc' and HAS_TRACEMALLOC: + if backend == 'tracemalloc' and has_tracemalloc: if not tracemalloc.is_tracing(): tracemalloc.start() if func is not None: - return get_profile_wrapper(func, precision, backend, stream) + get_prof = partial(LineProfiler, backend=backend) + show_results_bound = partial( + show_results, stream=stream, precision=precision + ) + if iscoroutinefunction(func): + @coroutine + def wrapper(*args, **kwargs): + prof = get_prof() + val = yield from prof(func)(*args, **kwargs) + show_results_bound(prof) + return val + else: + def wrapper(*args, **kwargs): + prof = get_prof() + val = prof(func)(*args, **kwargs) + show_results_bound(prof) + return val + + return wrapper else: def inner_wrapper(f): return profile(f, stream=stream, precision=precision, @@ -771,32 +1152,58 @@ def inner_wrapper(f): return inner_wrapper +def choose_backend(new_backend=None): + """ + Function that tries to setup backend, chosen by user, and if failed, + setup one of the allowable backends + """ + + _backend = 'no_backend' + all_backends = [ + ('psutil', True), + ('posix', os.name == 'posix'), + ('tracemalloc', has_tracemalloc), + ] + backends_indices = dict((b[0], i) for i, b in enumerate(all_backends)) + + if new_backend is not None: + all_backends.insert(0, all_backends.pop(backends_indices[new_backend])) + + for n_backend, is_available in all_backends: + if is_available: + _backend = n_backend + break + if _backend != new_backend and new_backend is not None: + warnings.warn('{0} can not be used, {1} used instead'.format( + new_backend, _backend)) + return _backend + + # Insert in the built-ins to have profile # globally defined (global variables is not enough # for all cases, e.g. a script that imports another # script where @profile is used) def exec_with_profiler(filename, profiler, backend, passed_args=[]): + from runpy import run_module builtins.__dict__['profile'] = profiler - ns = dict( - _CLEAN_GLOBALS, - profile=profiler, - # Make sure the __file__ variable is usable - # by the script we're profiling - __file__=filename - ) + ns = dict(_CLEAN_GLOBALS, + profile=profiler, + # Make sure the __file__ variable is usable + # by the script we're profiling + __file__=filename) # Make sure the script's directory in on sys.path # credit to line_profiler - sys.path.insert(0, os.path.dirname(filename)) + sys.path.insert(0, os.path.dirname(script_filename)) _backend = choose_backend(backend) sys.argv = [filename] + passed_args try: - if _backend == 'tracemalloc' and HAS_TRACEMALLOC: + if _backend == 'tracemalloc' and has_tracemalloc: tracemalloc.start() with open(filename) as f: exec(compile(f.read(), filename, 'exec'), ns, ns) finally: - if HAS_TRACEMALLOC and tracemalloc.is_tracing(): + if has_tracemalloc and tracemalloc.is_tracing(): tracemalloc.stop() @@ -806,16 +1213,13 @@ def run_module_with_profiler(module, profiler, backend, passed_args=[]): ns = dict(_CLEAN_GLOBALS, profile=profiler) _backend = choose_backend(backend) sys.argv = [module] + passed_args - if PY2: + if _backend == 'tracemalloc' and has_tracemalloc: + tracemalloc.start() + try: run_module(module, run_name="__main__", init_globals=ns) - else: - if _backend == 'tracemalloc' and HAS_TRACEMALLOC: - tracemalloc.start() - try: - run_module(module, run_name="__main__", init_globals=ns) - finally: - if HAS_TRACEMALLOC and tracemalloc.is_tracing(): - tracemalloc.stop() + finally: + if has_tracemalloc and tracemalloc.is_tracing(): + tracemalloc.stop() class LogFile(object): @@ -848,3 +1252,64 @@ def write(self, msg, level=logging.INFO): def flush(self): for handler in self.logger.handlers: handler.flush() + + +if __name__ == '__main__': + from argparse import ArgumentParser, REMAINDER + + parser = ArgumentParser(usage=_CMD_USAGE) + parser.add_argument('--version', action='version', version=__version__) + parser.add_argument( + '--pdb-mmem', dest='max_mem', metavar='MAXMEM', + type=float, action='store', + help='step into the debugger when memory exceeds MAXMEM') + parser.add_argument( + '--precision', dest='precision', type=int, + action='store', default=3, + help='precision of memory output in number of significant digits') + parser.add_argument('-o', dest='out_filename', type=str, + action='store', default=None, + help='path to a file where results will be written') + parser.add_argument('--timestamp', dest='timestamp', default=False, + action='store_true', + help='''print timestamp instead of memory measurement for + decorated functions''') + parser.add_argument('--include-children', dest='include_children', + default=False, action='store_true', + help='also include memory used by child processes') + parser.add_argument('--backend', dest='backend', type=str, action='store', + choices=['tracemalloc', 'psutil', 'posix'], default='psutil', + help='backend using for getting memory info ' + '(one of the {tracemalloc, psutil, posix})') + parser.add_argument("program", nargs=REMAINDER, + help='python script or module followed by command line arguements to run') + args = parser.parse_args() + + if len(args.program) == 0: + print("A program to run must be provided. Use -h for help") + sys.exit(1) + + target = args.program[0] + script_args = args.program[1:] + _backend = choose_backend(args.backend) + if args.timestamp: + prof = TimeStamper(_backend, include_children=args.include_children) + else: + prof = LineProfiler(max_mem=args.max_mem, backend=_backend) + + try: + if args.program[0].endswith('.py'): + script_filename = _find_script(args.program[0]) + exec_with_profiler(script_filename, prof, args.backend, script_args) + else: + run_module_with_profiler(target, prof, args.backend, script_args) + finally: + if args.out_filename is not None: + out_file = open(args.out_filename, "a") + else: + out_file = sys.stdout + + if args.timestamp: + prof.show_results(stream=out_file) + else: + show_results(prof, precision=args.precision, stream=out_file) diff --git a/memory_profiler/__main__.py b/memory_profiler/__main__.py deleted file mode 100644 index 4cca042..0000000 --- a/memory_profiler/__main__.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import sys -from argparse import ArgumentParser, REMAINDER - -from . import exec_with_profiler, run_module_with_profiler, TimeStamper, __version__ -from .line_profiler import LineProfiler -from .utils import choose_backend, show_results - - -_CMD_USAGE = "python -m memory_profiler script_file.py" - - -def _find_script(script_name): - """ Find the script. - - If the input is not a file, then $PATH will be searched. - """ - if os.path.isfile(script_name): - return script_name - path = os.getenv('PATH', os.defpath).split(os.pathsep) - for folder in path: - if not folder: - continue - fn = os.path.join(folder, script_name) - if os.path.isfile(fn): - return fn - - sys.stderr.write('Could not find script {0}\n'.format(script_name)) - raise SystemExit(1) - - -if __name__ == '__main__': - parser = ArgumentParser(usage=_CMD_USAGE) - parser.add_argument('--version', action='version', version=__version__) - parser.add_argument( - '--pdb-mmem', - dest='max_mem', - metavar='MAXMEM', - type=float, - action='store', - help='step into the debugger when memory exceeds MAXMEM' - ) - parser.add_argument( - '--precision', - dest='precision', - type=int, - action='store', - default=3, - help='precision of memory output in number of significant digits' - ) - parser.add_argument( - '-o', - dest='out_filename', - type=str, - action='store', - default=None, - help='path to a file where results will be written' - ) - parser.add_argument( - '--timestamp', - dest='timestamp', - default=False, - action='store_true', - help='''print timestamp instead of memory measurement for - decorated functions''' - ) - parser.add_argument( - '--backend', - dest='backend', - type=str, - action='store', - choices=['tracemalloc', 'psutil', 'posix'], - default='psutil', - help='backend using for getting memory info ' - '(one of the {tracemalloc, psutil, posix})' - ) - parser.add_argument( - "program", - nargs=REMAINDER, - help='python script or module followed by ' - 'command line arguments to run' - ) - args = parser.parse_args() - - if len(args.program) == 0: - print("A program to run must be provided. Use -h for help") - sys.exit(1) - - target = args.program[0] - script_args = args.program[1:] - - _backend = choose_backend(args.backend) - if args.timestamp: - prof = TimeStamper(_backend) - else: - prof = LineProfiler(max_mem=args.max_mem, backend=_backend) - - try: - if args.program[0].endswith('.py'): - script_filename = _find_script(args.program[0]) - exec_with_profiler( - script_filename, prof, args.backend, script_args - ) - else: - run_module_with_profiler(target, prof, args.backend, script_args) - finally: - if args.out_filename is not None: - out_file = open(args.out_filename, "a") - else: - out_file = sys.stdout - - if args.timestamp: - prof.show_results(stream=out_file) - else: - show_results(prof, precision=args.precision, stream=out_file) diff --git a/memory_profiler/_async.py b/memory_profiler/_async.py deleted file mode 100644 index 86c2887..0000000 --- a/memory_profiler/_async.py +++ /dev/null @@ -1,35 +0,0 @@ -from asyncio import coroutine, iscoroutinefunction -from functools import wraps - -from .line_profiler import ( - get_profile_wrapper as default_profile_wrapper, - LineProfiler, -) -from .utils import show_results - - -class CoroLineProfiler(LineProfiler): - def wrap_function(self, func): - if iscoroutinefunction(func): - @coroutine - def f(*args, **kwargs): - with self.count_ctxmgr(): - yield from func(*args, **kwargs) - return f - else: - return super().wrap_function(func) - - -def get_profile_wrapper(func, precision, backend, stream): - if iscoroutinefunction(func): - @wraps(func) - @coroutine - def wrapper(*args, **kwargs): - prof = CoroLineProfiler(backend=backend) - val = yield from prof(func)(*args, **kwargs) - show_results(prof, stream=stream, precision=precision) - return val - else: - wrapper = default_profile_wrapper(func, precision, backend, stream) - - return wrapper diff --git a/memory_profiler/code_map.py b/memory_profiler/code_map.py deleted file mode 100644 index c5826fc..0000000 --- a/memory_profiler/code_map.py +++ /dev/null @@ -1,69 +0,0 @@ -import inspect -import os.path - -from .utils import get_memory - - -class CodeMap(dict): - def __init__(self, include_children, backend): - self.include_children = include_children - self._toplevel = [] - self.backend = backend - - def add(self, code, toplevel_code=None): - if code in self: - return - - if toplevel_code is None: - filename = code.co_filename - if filename.endswith((".pyc", ".pyo")): - filename = filename[:-1] - if not os.path.exists(filename): - print('ERROR: Could not find file ' + filename) - if filename.startswith(("ipython-input", " 0: - self.enable_count -= 1 - if self.enable_count == 0: - self.disable() - - def trace_memory_usage(self, frame, event, arg): - """Callback for sys.settrace""" - if frame.f_code in self.code_map: - if event == 'call': - # "call" event just saves the lineno but not the memory - self.prevlines.append(frame.f_lineno) - elif event == 'line': - # trace needs current line and previous line - self.code_map.trace(frame.f_code, self.prevlines[-1], self.prev_lineno) - # saving previous line - self.prev_lineno = self.prevlines[-1] - self.prevlines[-1] = frame.f_lineno - elif event == 'return': - lineno = self.prevlines.pop() - self.code_map.trace(frame.f_code, lineno, self.prev_lineno) - self.prev_lineno = lineno - - if self._original_trace_function is not None: - self._original_trace_function(frame, event, arg) - - return self.trace_memory_usage - - def trace_max_mem(self, frame, event, arg): - # run into PDB as soon as memory is higher than MAX_MEM - if event in ('line', 'return') and frame.f_code in self.code_map: - c = get_memory(-1, self.backend, filename=frame.f_code.co_filename) - if c >= self.max_mem: - t = ('Current memory {0:.2f} MiB exceeded the ' - 'maximum of {1:.2f} MiB\n'.format(c, self.max_mem)) - sys.stdout.write(t) - sys.stdout.write('Stepping into the debugger \n') - frame.f_lineno -= 2 - p = pdb.Pdb() - p.quitting = False - p.stopframe = frame - p.returnframe = None - p.stoplineno = frame.f_lineno - 3 - p.botframe = None - return p.trace_dispatch - - if self._original_trace_function is not None: - (self._original_trace_function)(frame, event, arg) - - return self.trace_max_mem - - def __enter__(self): - self.enable_by_count() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.disable_by_count() - - def enable(self): - self._original_trace_function = sys.gettrace() - if self.max_mem is not None: - sys.settrace(self.trace_max_mem) - else: - sys.settrace(self.trace_memory_usage) - - def disable(self): - sys.settrace(self._original_trace_function) - - -def get_profile_wrapper(func, precision, backend, stream): - @wraps(func) - def wrapper(*args, **kwargs): - prof = LineProfiler(backend=backend) - val = prof(func)(*args, **kwargs) - show_results(prof, stream=stream, precision=precision) - return val - - return wrapper diff --git a/memory_profiler/utils.py b/memory_profiler/utils.py deleted file mode 100644 index 27b8861..0000000 --- a/memory_profiler/utils.py +++ /dev/null @@ -1,183 +0,0 @@ -import linecache -import os -import sys -import warnings -import subprocess -import time - -import psutil - -from .common import HAS_TRACEMALLOC, PY2, TWO_20 - -if HAS_TRACEMALLOC: - import tracemalloc - - -if PY2: - def to_str(x): - return x - - from future_builtins import filter -else: - def to_str(x): - return str(x) - - -def show_results(prof, stream=None, precision=1): - if stream is None: - stream = sys.stdout - template = '{0:>6} {1:>12} {2:>12} {3:<}' - - for (filename, lines) in prof.code_map.items(): - header = template.format('Line #', 'Mem usage', 'Increment', - 'Line Contents') - - stream.write(u'Filename: ' + filename + '\n\n') - stream.write(header + u'\n') - stream.write(u'=' * len(header) + '\n') - - all_lines = linecache.getlines(filename) - - float_format = u'{0}.{1}f'.format(precision + 4, precision) - template_mem = u'{0:' + float_format + '} MiB' - for (lineno, mem) in lines: - if mem: - inc = mem[0] - mem = mem[1] - mem = template_mem.format(mem) - inc = template_mem.format(inc) - else: - mem = u'' - inc = u'' - tmp = template.format(lineno, mem, inc, all_lines[lineno - 1]) - stream.write(to_str(tmp)) - stream.write(u'\n\n') - - -def choose_backend(new_backend=None): - """ - Function that tries to setup backend, chosen by user, and if failed, - setup one of the allowable backends - """ - - _backend = 'no_backend' - all_backends = [ - ('psutil', True), - ('posix', os.name == 'posix'), - ('tracemalloc', HAS_TRACEMALLOC), - ] - backends_indices = dict((b[0], i) for i, b in enumerate(all_backends)) - - if new_backend is not None: - all_backends.insert(0, all_backends.pop(backends_indices[new_backend])) - - for n_backend, is_available in all_backends: - if is_available: - _backend = n_backend - break - if _backend != new_backend and new_backend is not None: - warnings.warn('{0} can not be used, {1} used instead'.format( - new_backend, _backend)) - return _backend - - -def get_child_memory(process, meminfo_attr=None): - """ - Returns a generator that yields memory for all child processes. - """ - # Convert a pid to a process - if isinstance(process, int): - if process == -1: process = os.getpid() - process = psutil.Process(process) - - if not meminfo_attr: - # Use the psutil 2.0 attr if the older version isn't passed in. - meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') else 'get_memory_info' - - # Select the psutil function get the children similar to how we selected - # the memory_info attr (a change from excepting the AttributeError). - children_attr = 'children' if hasattr(process, 'children') else 'get_children' - - # Loop over the child processes and yield their memory - try: - for child in getattr(process, children_attr)(recursive=True): - yield getattr(child, meminfo_attr)()[0] / TWO_20 - except (psutil.NoSuchProcess, psutil.AccessDenied): - # https://github.com/fabianp/memory_profiler/issues/71 - yield 0.0 - - -def get_memory(pid, backend, timestamps=False, include_children=False, filename=None): - # .. low function to get memory consumption .. - if pid == -1: - pid = os.getpid() - - def tracemalloc_tool(): - # .. cross-platform but but requires Python 3.4 or higher .. - stat = next(filter(lambda item: str(item).startswith(filename), - tracemalloc.take_snapshot().statistics('filename'))) - mem = stat.size / TWO_20 - if timestamps: - return mem, time.time() - else: - return mem - - def ps_util_tool(): - # .. cross-platform but but requires psutil .. - process = psutil.Process(pid) - try: - # avoid using get_memory_info since it does not exists - # in psutil > 2.0 and accessing it will cause exception. - meminfo_attr = 'memory_info' if hasattr(process, 'memory_info') \ - else 'get_memory_info' - mem = getattr(process, meminfo_attr)()[0] / TWO_20 - if include_children: - mem += sum(get_child_memory(process, meminfo_attr)) - if timestamps: - return mem, time.time() - else: - return mem - except psutil.AccessDenied: - pass - # continue and try to get this from ps - - def posix_tool(): - # .. scary stuff .. - if include_children: - raise NotImplementedError(( - "The psutil module is required to monitor the " - "memory usage of child processes." - )) - - warnings.warn("psutil module not found. memory_profiler will be slow") - # .. - # .. memory usage in MiB .. - # .. this should work on both Mac and Linux .. - # .. subprocess.check_output appeared in 2.7, using Popen .. - # .. for backwards compatibility .. - out = subprocess.Popen(['ps', 'v', '-p', str(pid)], - stdout=subprocess.PIPE - ).communicate()[0].split(b'\n') - try: - vsz_index = out[0].split().index(b'RSS') - mem = float(out[1].split()[vsz_index]) / 1024 - if timestamps: - return mem, time.time() - else: - return mem - except: - if timestamps: - return -1, time.time() - else: - return -1 - - if backend == 'tracemalloc' and \ - (filename is None or filename == ''): - raise RuntimeError( - 'There is no access to source file of the profiled function' - ) - - tools = {'tracemalloc': tracemalloc_tool, - 'psutil': ps_util_tool, - 'posix': posix_tool} - return tools[backend]() From f62614f26dbd8512cb4cf41942ce990f9dd05561 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sat, 13 Jun 2020 21:13:11 +0300 Subject: [PATCH 15/17] Remove python2.7 from ci --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd260b0..11d4ac0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.7" - "3.4" - "3.5" - "3.6" From 7c52327b024c8d954b1db5076ce7821f4be29ba6 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sat, 13 Jun 2020 21:15:56 +0300 Subject: [PATCH 16/17] Remove python < 3.4 detection from Makefile --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index ecf4498..59d4950 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,4 @@ PYTHON ?= python -PY_34 ?= $(shell ! python -c \ - 'import sys; print((3, 4) < sys.version_info)') .PHONY: test develop From f9d95ef18168d4254ee8f6918b93cb21f0877791 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Sat, 13 Jun 2020 21:30:08 +0300 Subject: [PATCH 17/17] setup.py cleanup Minor refactoring of `find_version` function. Remove old python tags. Revert package module list. Specify required python version. --- setup.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index effd3f3..d53db18 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,14 @@ -import os -import io import re -from setuptools import find_packages, setup +from setuptools import setup # https://packaging.python.org/guides/single-sourcing-package-version/ -def read(*names, **kwargs): - with io.open( - os.path.join(os.path.dirname(__file__), *names), - encoding=kwargs.get("encoding", "utf8") - ) as fp: - return fp.read() - - -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, re.M) +def find_version(file_paths): + with open(file_paths) as f: + version_file = f.read() + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M + ) if version_match: return version_match.group(1) @@ -27,12 +19,7 @@ def find_version(*file_paths): Intended Audience :: Developers License :: OSI Approved :: BSD License Programming Language :: Python -Programming Language :: Python :: 2 -Programming Language :: Python :: 2.6 -Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 -Programming Language :: Python :: 3.2 -Programming Language :: Python :: 3.3 Topic :: Software Development Operating System :: POSIX Operating System :: Unix @@ -44,16 +31,16 @@ def find_version(*file_paths): name='memory_profiler', description='A module for monitoring memory usage of a python program', long_description=open('README.rst').read(), - version=find_version("memory_profiler", "__init__.py"), + version=find_version("memory_profiler.py"), author='Fabian Pedregosa', author_email='f@bianp.net', url='https://github.com/pythonprofilers/memory_profiler', - py_modules=['mprof'], - packages=find_packages(include=('memory_profiler',)), + py_modules=['memory_profiler', 'mprof'], entry_points={ 'console_scripts': ['mprof = mprof:main'], }, install_requires=['psutil'], + python_requires='>=3.4', classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], license='BSD' )