diff --git a/docs/source/index.rst b/docs/source/index.rst index 54dc6e69220..050ecf4dff8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,9 +60,9 @@ Technical documentation is available at `docs.monai.io `_ .. toctree:: :maxdepth: 1 - :caption: Precision and Performance + :caption: Precision and Accelerating - precision_performance + precision_accelerating .. toctree:: :maxdepth: 1 diff --git a/docs/source/installation.md b/docs/source/installation.md index 6d63fbf08fb..6580ce67172 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -254,11 +254,11 @@ Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is - The options are ``` -[nibabel, skimage, scipy, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, clearml, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna, onnx, onnxruntime, zarr, lpips] +[nibabel, skimage, scipy, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, clearml, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna, onnx, onnxruntime, zarr, lpips, pynvml] ``` which correspond to `nibabel`, `scikit-image`,`scipy`, `pillow`, `tensorboard`, -`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `clearml`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, `onnx`, `onnxruntime`, `zarr` and `lpips` respectively. +`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `clearml`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, `onnx`, `onnxruntime`, `zarr`, `lpips` and `nvidia-ml-py` respectively. - `pip install 'monai[all]'` installs all the optional dependencies. diff --git a/docs/source/precision_performance.md b/docs/source/precision_accelerating.md similarity index 87% rename from docs/source/precision_performance.md rename to docs/source/precision_accelerating.md index 6e6c51d8c15..d6c6f4959a7 100644 --- a/docs/source/precision_performance.md +++ b/docs/source/precision_accelerating.md @@ -29,11 +29,11 @@ by TF32 mode so the impact is very wide. torch.backends.cuda.matmul.allow_tf32 = False # in PyTorch 1.12 and later. torch.backends.cudnn.allow_tf32 = True ``` -Please note that there are environment variables that can override the flags above. For example, the environment variables mentioned in [Accelerating AI Training with NVIDIA TF32 Tensor Cores](https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/) and `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE` used by PyTorch. Thus, in some cases, the flags may be accidentally changed or overridden. - -We recommend that users print out these two flags for confirmation when unsure. +Please note that there are environment variables that can override the flags above. For example, the environment variable `NVIDIA_TF32_OVERRIDE` mentioned in [Accelerating AI Training with NVIDIA TF32 Tensor Cores](https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/) and `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE` used by PyTorch. Thus, in some cases, the flags may be accidentally changed or overridden. If you are using an [NGC PyTorch container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch), the container includes a layer `ENV TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1`. The default value `torch.backends.cuda.matmul.allow_tf32` will be overridden to `True`. +We recommend that users print out these two flags for confirmation when unsure. + If you can confirm through experiments that your model has no accuracy or convergence issues in TF32 mode and you have NVIDIA Ampere GPUs or above, you can set the two flags above to `True` to speed up your model. diff --git a/monai/__init__.py b/monai/__init__.py index 9f542dd25c4..638220f6df0 100644 --- a/monai/__init__.py +++ b/monai/__init__.py @@ -78,3 +78,13 @@ "utils", "visualize", ] + +try: + from .utils.tf32 import detect_default_tf32 + + detect_default_tf32() +except BaseException: + from .utils.misc import MONAIEnvVars + + if MONAIEnvVars.debug(): + raise diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py index 58bde3f5e8f..c973d4bfa18 100644 --- a/monai/utils/__init__.py +++ b/monai/utils/__init__.py @@ -115,6 +115,7 @@ require_pkg, run_debug, run_eval, + version_geq, version_leq, ) from .nvtx import Range @@ -128,6 +129,7 @@ torch_profiler_time_end_to_end, ) from .state_cacher import StateCacher +from .tf32 import detect_default_tf32, has_ampere_or_later from .type_conversion import ( convert_data_type, convert_to_cupy, diff --git a/monai/utils/module.py b/monai/utils/module.py index fcd5e04145f..f46ba7c1b31 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -25,7 +25,7 @@ from pydoc import locate from re import match from types import FunctionType, ModuleType -from typing import Any, cast +from typing import Any, Iterable, cast import torch @@ -55,6 +55,7 @@ "get_package_version", "get_torch_version_tuple", "version_leq", + "version_geq", "pytorch_after", ] @@ -518,24 +519,11 @@ def get_torch_version_tuple(): return tuple(int(x) for x in torch.__version__.split(".")[:2]) -def version_leq(lhs: str, rhs: str) -> bool: +def parse_version_strs(lhs: str, rhs: str) -> tuple[Iterable[int | str], Iterable[int | str]]: """ - Returns True if version `lhs` is earlier or equal to `rhs`. - - Args: - lhs: version name to compare with `rhs`, return True if earlier or equal to `rhs`. - rhs: version name to compare with `lhs`, return True if later or equal to `lhs`. - + Parse the version strings. """ - lhs, rhs = str(lhs), str(rhs) - pkging, has_ver = optional_import("pkg_resources", name="packaging") - if has_ver: - try: - return cast(bool, pkging.version.Version(lhs) <= pkging.version.Version(rhs)) - except pkging.version.InvalidVersion: - return True - def _try_cast(val: str) -> int | str: val = val.strip() try: @@ -554,7 +542,28 @@ def _try_cast(val: str) -> int | str: # parse the version strings in this basic way without `packaging` package lhs_ = map(_try_cast, lhs.split(".")) rhs_ = map(_try_cast, rhs.split(".")) + return lhs_, rhs_ + +def version_leq(lhs: str, rhs: str) -> bool: + """ + Returns True if version `lhs` is earlier or equal to `rhs`. + + Args: + lhs: version name to compare with `rhs`, return True if earlier or equal to `rhs`. + rhs: version name to compare with `lhs`, return True if later or equal to `lhs`. + + """ + + lhs, rhs = str(lhs), str(rhs) + pkging, has_ver = optional_import("pkg_resources", name="packaging") + if has_ver: + try: + return cast(bool, pkging.version.Version(lhs) <= pkging.version.Version(rhs)) + except pkging.version.InvalidVersion: + return True + + lhs_, rhs_ = parse_version_strs(lhs, rhs) for l, r in zip(lhs_, rhs_): if l != r: if isinstance(l, int) and isinstance(r, int): @@ -564,6 +573,33 @@ def _try_cast(val: str) -> int | str: return True +def version_geq(lhs: str, rhs: str) -> bool: + """ + Returns True if version `lhs` is later or equal to `rhs`. + + Args: + lhs: version name to compare with `rhs`, return True if later or equal to `rhs`. + rhs: version name to compare with `lhs`, return True if earlier or equal to `lhs`. + + """ + lhs, rhs = str(lhs), str(rhs) + pkging, has_ver = optional_import("pkg_resources", name="packaging") + if has_ver: + try: + return cast(bool, pkging.version.Version(lhs) >= pkging.version.Version(rhs)) + except pkging.version.InvalidVersion: + return True + + lhs_, rhs_ = parse_version_strs(lhs, rhs) + for l, r in zip(lhs_, rhs_): + if l != r: + if isinstance(l, int) and isinstance(r, int): + return l > r + return f"{l}" > f"{r}" + + return True + + @functools.lru_cache(None) def pytorch_after(major: int, minor: int, patch: int = 0, current_ver_string: str | None = None) -> bool: """ diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py new file mode 100644 index 00000000000..9ef425ab8bd --- /dev/null +++ b/monai/utils/tf32.py @@ -0,0 +1,89 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import functools +import os +import warnings + +__all__ = ["has_ampere_or_later", "detect_default_tf32"] + + +@functools.lru_cache(None) +def has_ampere_or_later() -> bool: + """ + Check if there is any Ampere and later GPU. + """ + import torch + + from monai.utils.module import optional_import, version_geq + + if not (torch.version.cuda and version_geq(f"{torch.version.cuda}", "11.0")): + return False + + pynvml, has_pynvml = optional_import("pynvml") + if not has_pynvml: # assuming that the user has Ampere and later GPU + return True + + try: + pynvml.nvmlInit() + for i in range(pynvml.nvmlDeviceGetCount()): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + major, _ = pynvml.nvmlDeviceGetCudaComputeCapability(handle) + if major >= 8: + return True + except BaseException: + pass + finally: + pynvml.nvmlShutdown() + + return False + + +@functools.lru_cache(None) +def detect_default_tf32() -> bool: + """ + Dectect if there is anything that may enable TF32 mode by default. + If any, show a warning message. + """ + may_enable_tf32 = False + try: + if not has_ampere_or_later(): + return False + + from monai.utils.module import pytorch_after + + if pytorch_after(1, 7, 0) and not pytorch_after(1, 12, 0): + warnings.warn( + "torch.backends.cuda.matmul.allow_tf32 = True by default.\n" + " This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision.\n" + " See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" + ) + may_enable_tf32 = True + + override_tf32_env_vars = {"NVIDIA_TF32_OVERRIDE": "1", "TORCH_ALLOW_TF32_CUBLAS_OVERRIDE": "1"} + for name, override_val in override_tf32_env_vars.items(): + if os.environ.get(name) == override_val: + warnings.warn( + f"Environment variable `{name} = {override_val}` is set.\n" + f" This environment variable may enable TF32 mode accidentally and affect precision.\n" + f" See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" + ) + may_enable_tf32 = True + + return may_enable_tf32 + except BaseException: + from monai.utils.misc import MONAIEnvVars + + if MONAIEnvVars.debug(): + raise + return False diff --git a/requirements-dev.txt b/requirements-dev.txt index f0e3a37c240..9620ea253d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -55,3 +55,4 @@ typeguard<3 # https://github.com/microsoft/nni/issues/5457 filelock!=3.12.0 # https://github.com/microsoft/nni/issues/5523 zarr lpips==0.1.4 +nvidia-ml-py diff --git a/setup.cfg b/setup.cfg index 9cd1d59f954..9d5a22963c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,7 @@ all = onnxruntime; python_version <= '3.10' zarr lpips==0.1.4 + nvidia-ml-py nibabel = nibabel ninja = @@ -153,6 +154,8 @@ zarr = zarr lpips = lpips==0.1.4 +pynvml = + nvidia-ml-py # # workaround https://github.com/Project-MONAI/MONAI/issues/5882 # MetricsReloaded = # MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support#egg=MetricsReloaded diff --git a/tests/test_version_leq.py b/tests/test_version.py similarity index 88% rename from tests/test_version_leq.py rename to tests/test_version.py index ef9e70ad862..15f8cd36c6b 100644 --- a/tests/test_version_leq.py +++ b/tests/test_version.py @@ -16,7 +16,7 @@ from parameterized import parameterized -from monai.utils import version_leq +from monai.utils import version_geq, version_leq # from pkg_resources @@ -76,10 +76,15 @@ def _pairwise(iterable): class TestVersionCompare(unittest.TestCase): @parameterized.expand(TEST_CASES) - def test_compare(self, a, b, expected=True): + def test_compare_leq(self, a, b, expected=True): """Test version_leq with `a` and `b`""" self.assertEqual(version_leq(a, b), expected) + @parameterized.expand(TEST_CASES) + def test_compare_geq(self, a, b, expected=True): + """Test version_geq with `b` and `a`""" + self.assertEqual(version_geq(b, a), expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/utils.py b/tests/utils.py index bd357aa5294..1dfd5e6e209 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,7 +46,8 @@ from monai.data.meta_tensor import MetaTensor, get_track_meta from monai.networks import convert_to_onnx, convert_to_torchscript from monai.utils import optional_import -from monai.utils.module import pytorch_after, version_leq +from monai.utils.module import pytorch_after +from monai.utils.tf32 import detect_default_tf32 from monai.utils.type_conversion import convert_data_type nib, _ = optional_import("nibabel") @@ -172,19 +173,14 @@ def test_is_quick(): def is_tf32_env(): """ - The environment variable NVIDIA_TF32_OVERRIDE=0 will override any defaults - or programmatic configuration of NVIDIA libraries, and consequently, - cuBLAS will not accelerate FP32 computations with TF32 tensor cores. + When we may be using TF32 mode, check the precision of matrix operation. + If the checking result is greater than the threshold 0.001, + set _tf32_enabled=True (and relax _rtol for tests). """ global _tf32_enabled if _tf32_enabled is None: _tf32_enabled = False - if ( - torch.cuda.is_available() - and not version_leq(f"{torch.version.cuda}", "10.100") - and os.environ.get("NVIDIA_TF32_OVERRIDE", "1") != "0" - and torch.cuda.device_count() > 0 # at least 11.0 - ): + if detect_default_tf32() or torch.backends.cuda.matmul.allow_tf32: try: # with TF32 enabled, the speed is ~8x faster, but the precision has ~2 digits less in the result g_gpu = torch.Generator(device="cuda")