From 0da7579828f00c267915303481567174a4ae00be Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sun, 10 Jan 2021 23:20:15 +0100 Subject: [PATCH 1/5] More speedup via mtime-base caching. Caching based on mtime is similar to the one done on importlib's FileFinder. Locally, on a large-ish environment, this speeds up repeated calls to `distribution("pip")` ~10x. --- importlib_metadata/__init__.py | 90 +++++++++++++++++++--------------- tests/test_api.py | 7 +++ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c420a55..fb91957c 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -464,9 +464,15 @@ class FastPath: children. """ - def __init__(self, root): + @functools.lru_cache() # type: ignore + def __new__(cls, root): + self = object().__new__(cls) self.root = str(root) self.base = os.path.basename(self.root).lower() + self.last_mtime = -1 + self.infos = {} + self.eggs = {} + return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -482,15 +488,47 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def search(self, name): - return ( - self.joinpath(child) - for child in self.children() - if name.matches(child, self.base) - ) + def update_cache(self): + root = self.root or "." + try: + mtime = os.stat(root).st_mtime + except OSError: + self.infos.clear() + self.eggs.clear() + self.last_mtime = -1 + return + if mtime == self.last_mtime: + return + self.infos.clear() + self.eggs.clear() + base_is_egg = self.base.endswith(".egg") + for child in self.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos.setdefault(normalized, []).append(child) + elif base_is_egg and low == "egg-info": + name = self.base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs.setdefault(legacy_normalized, []).append(child) + self.last_mtime = mtime + + def search(self, prepared): + self.update_cache() + if prepared.name: + infos = self.infos.get(prepared.normalized, []) + yield from map(self.joinpath, infos) + eggs = self.eggs.get(prepared.legacy_normalized, []) + yield from map(self.joinpath, eggs) + else: + for infos in self.infos.values(): + yield from map(self.joinpath, infos) + for eggs in self.eggs.values(): + yield from map(self.joinpath, eggs) class Prepared: @@ -499,22 +537,14 @@ class Prepared: """ normalized = None - suffixes = 'dist-info', 'egg-info' - exact_matches = [''][:0] - egg_prefix = '' - versionless_egg_name = '' + legacy_normalized = None def __init__(self, name): self.name = name if name is None: return self.normalized = self.normalize(name) - self.exact_matches = [ - self.normalized + '.' + suffix for suffix in self.suffixes - ] - legacy_normalized = self.legacy_normalize(self.name) - self.egg_prefix = legacy_normalized + '-' - self.versionless_egg_name = legacy_normalized + '.egg' + self.legacy_normalized = self.legacy_normalize(name) @staticmethod def normalize(name): @@ -531,27 +561,6 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') - def matches(self, cand, base): - low = cand.lower() - # rpartition is faster than splitext and suitable for this purpose. - pre, _, ext = low.rpartition('.') - name, _, rest = pre.partition('-') - return ( - low in self.exact_matches - or ext in self.suffixes - and (not self.normalized or name.replace('.', '_') == self.normalized) - # legacy case: - or self.is_egg(base) - and low == 'egg-info' - ) - - def is_egg(self, base): - return ( - base == self.versionless_egg_name - or base.startswith(self.egg_prefix) - and base.endswith('.egg') - ) - @install class MetadataPathFinder(NullFinder, DistributionFinder): @@ -581,6 +590,9 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + class PathDistribution(Distribution): def __init__(self, path): diff --git a/tests/test_api.py b/tests/test_api.py index 04a5b9d3..134acf60 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import re import textwrap import unittest +import importlib from . import fixtures from importlib_metadata import ( @@ -224,3 +225,9 @@ def test_distribution_at_str(self): dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' dist = Distribution.at(str(dist_info_path)) assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() From 12e33bb37bc8517ff89f3e7b56a1b3bd585482ed Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 21:17:36 -0500 Subject: [PATCH 2/5] Extract Lookup class out of directory lookup behavior. Separate it from mtime caching behavior. --- importlib_metadata/__init__.py | 80 +++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 67c925d4..349de2e0 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,6 +12,7 @@ import functools import itertools import posixpath +import contextlib import collections.abc from ._compat import ( @@ -602,13 +603,10 @@ class FastPath: @functools.lru_cache() # type: ignore def __new__(cls, root): - self = object().__new__(cls) + return super().__new__(cls) + + def __init__(self, root): self.root = str(root) - self.base = os.path.basename(self.root).lower() - self.last_mtime = -1 - self.infos = {} - self.eggs = {} - return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -624,47 +622,54 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def update_cache(self): - root = self.root or "." - try: - mtime = os.stat(root).st_mtime - except OSError: - self.infos.clear() - self.eggs.clear() - self.last_mtime = -1 - return - if mtime == self.last_mtime: - return - self.infos.clear() - self.eggs.clear() - base_is_egg = self.base.endswith(".egg") - for child in self.children(): + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with contextlib.suppress(OSError): + return os.stat(self.root).st_mtime + FastPath.lookup.cache_clear() + + @functools.lru_cache() + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = collections.defaultdict(list) + self.eggs = collections.defaultdict(list) + + for child in path.children(): low = child.lower() if low.endswith((".dist-info", ".egg-info")): # rpartition is faster than splitext and suitable for this purpose. name = low.rpartition(".")[0].partition("-")[0] normalized = Prepared.normalize(name) - self.infos.setdefault(normalized, []).append(child) + self.infos[normalized].append(path.joinpath(child)) elif base_is_egg and low == "egg-info": - name = self.base.rpartition(".")[0].partition("-")[0] + name = base.rpartition(".")[0].partition("-")[0] legacy_normalized = Prepared.legacy_normalize(name) - self.eggs.setdefault(legacy_normalized, []).append(child) - self.last_mtime = mtime + self.eggs[legacy_normalized].append(path.joinpath(child)) def search(self, prepared): - self.update_cache() - if prepared.name: - infos = self.infos.get(prepared.normalized, []) - yield from map(self.joinpath, infos) - eggs = self.eggs.get(prepared.legacy_normalized, []) - yield from map(self.joinpath, eggs) - else: - for infos in self.infos.values(): - yield from map(self.joinpath, infos) - for eggs in self.eggs.values(): - yield from map(self.joinpath, eggs) + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) class Prepared: @@ -697,6 +702,9 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') + def __bool__(self): + return bool(self.name) + @install class MetadataPathFinder(NullFinder, DistributionFinder): From 6e2740ec6e1166f2b60719b47920eb66e87019ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:10:18 -0400 Subject: [PATCH 3/5] Remove 'cache clear' operation. Unlocks caching. --- importlib_metadata/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 349de2e0..5d34bbd2 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -632,7 +632,6 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime - FastPath.lookup.cache_clear() @functools.lru_cache() def lookup(self, mtime): From db55bc647bf1778c17339d57aefd2a90f074f264 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:41:32 -0400 Subject: [PATCH 4/5] Restore cache-clear behavior on a per-path basis. --- importlib_metadata/__init__.py | 4 +- importlib_metadata/_functools.py | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 importlib_metadata/_functools.py diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5d34bbd2..42259f6f 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -22,6 +22,7 @@ Protocol, ) +from ._functools import method_cache from ._itertools import unique_everseen from configparser import ConfigParser @@ -632,8 +633,9 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime + self.lookup.cache_clear() - @functools.lru_cache() + @method_cache def lookup(self, mtime): return Lookup(self) diff --git a/importlib_metadata/_functools.py b/importlib_metadata/_functools.py new file mode 100644 index 00000000..73f50d00 --- /dev/null +++ b/importlib_metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper From 5ef4f529308db4e0778e71fb20522e004d057dcb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:54:45 -0400 Subject: [PATCH 5/5] Update changelog. --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2792caf8..1bc91e5c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v3.8.0 +====== + +* #290: Add mtime-based caching for ``FastPath`` and its + lookups, dramatically increasing performance for repeated + distribution lookups. + v3.7.2 ======