From 5f23835ad92ace9e2ed50ff819aede921aaa29bd Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 16:52:49 +0800 Subject: [PATCH 01/21] rename precision doc Signed-off-by: Qingpeng Li --- docs/source/index.rst | 4 ++-- .../{precision_performance.md => precision_accelerating.md} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename docs/source/{precision_performance.md => precision_accelerating.md} (87%) diff --git a/docs/source/index.rst b/docs/source/index.rst index 54dc6e69220..050ecf4dff8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,9 +60,9 @@ Technical documentation is available at `docs.monai.io `_ .. toctree:: :maxdepth: 1 - :caption: Precision and Performance + :caption: Precision and Accelerating - precision_performance + precision_accelerating .. toctree:: :maxdepth: 1 diff --git a/docs/source/precision_performance.md b/docs/source/precision_accelerating.md similarity index 87% rename from docs/source/precision_performance.md rename to docs/source/precision_accelerating.md index 6e6c51d8c15..d6c6f4959a7 100644 --- a/docs/source/precision_performance.md +++ b/docs/source/precision_accelerating.md @@ -29,11 +29,11 @@ by TF32 mode so the impact is very wide. torch.backends.cuda.matmul.allow_tf32 = False # in PyTorch 1.12 and later. torch.backends.cudnn.allow_tf32 = True ``` -Please note that there are environment variables that can override the flags above. For example, the environment variables mentioned in [Accelerating AI Training with NVIDIA TF32 Tensor Cores](https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/) and `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE` used by PyTorch. Thus, in some cases, the flags may be accidentally changed or overridden. - -We recommend that users print out these two flags for confirmation when unsure. +Please note that there are environment variables that can override the flags above. For example, the environment variable `NVIDIA_TF32_OVERRIDE` mentioned in [Accelerating AI Training with NVIDIA TF32 Tensor Cores](https://developer.nvidia.com/blog/accelerating-ai-training-with-tf32-tensor-cores/) and `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE` used by PyTorch. Thus, in some cases, the flags may be accidentally changed or overridden. If you are using an [NGC PyTorch container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch), the container includes a layer `ENV TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1`. The default value `torch.backends.cuda.matmul.allow_tf32` will be overridden to `True`. +We recommend that users print out these two flags for confirmation when unsure. + If you can confirm through experiments that your model has no accuracy or convergence issues in TF32 mode and you have NVIDIA Ampere GPUs or above, you can set the two flags above to `True` to speed up your model. From 60142163bdd7960f70b6498be89cddb5609c91eb Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 16:54:09 +0800 Subject: [PATCH 02/21] add `version_geq` Signed-off-by: Qingpeng Li --- monai/utils/module.py | 68 ++++++++++++++----- .../{test_version_leq.py => test_version.py} | 7 +- 2 files changed, 58 insertions(+), 17 deletions(-) rename tests/{test_version_leq.py => test_version.py} (90%) diff --git a/monai/utils/module.py b/monai/utils/module.py index fcd5e04145f..f46ba7c1b31 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -25,7 +25,7 @@ from pydoc import locate from re import match from types import FunctionType, ModuleType -from typing import Any, cast +from typing import Any, Iterable, cast import torch @@ -55,6 +55,7 @@ "get_package_version", "get_torch_version_tuple", "version_leq", + "version_geq", "pytorch_after", ] @@ -518,24 +519,11 @@ def get_torch_version_tuple(): return tuple(int(x) for x in torch.__version__.split(".")[:2]) -def version_leq(lhs: str, rhs: str) -> bool: +def parse_version_strs(lhs: str, rhs: str) -> tuple[Iterable[int | str], Iterable[int | str]]: """ - Returns True if version `lhs` is earlier or equal to `rhs`. - - Args: - lhs: version name to compare with `rhs`, return True if earlier or equal to `rhs`. - rhs: version name to compare with `lhs`, return True if later or equal to `lhs`. - + Parse the version strings. """ - lhs, rhs = str(lhs), str(rhs) - pkging, has_ver = optional_import("pkg_resources", name="packaging") - if has_ver: - try: - return cast(bool, pkging.version.Version(lhs) <= pkging.version.Version(rhs)) - except pkging.version.InvalidVersion: - return True - def _try_cast(val: str) -> int | str: val = val.strip() try: @@ -554,7 +542,28 @@ def _try_cast(val: str) -> int | str: # parse the version strings in this basic way without `packaging` package lhs_ = map(_try_cast, lhs.split(".")) rhs_ = map(_try_cast, rhs.split(".")) + return lhs_, rhs_ + +def version_leq(lhs: str, rhs: str) -> bool: + """ + Returns True if version `lhs` is earlier or equal to `rhs`. + + Args: + lhs: version name to compare with `rhs`, return True if earlier or equal to `rhs`. + rhs: version name to compare with `lhs`, return True if later or equal to `lhs`. + + """ + + lhs, rhs = str(lhs), str(rhs) + pkging, has_ver = optional_import("pkg_resources", name="packaging") + if has_ver: + try: + return cast(bool, pkging.version.Version(lhs) <= pkging.version.Version(rhs)) + except pkging.version.InvalidVersion: + return True + + lhs_, rhs_ = parse_version_strs(lhs, rhs) for l, r in zip(lhs_, rhs_): if l != r: if isinstance(l, int) and isinstance(r, int): @@ -564,6 +573,33 @@ def _try_cast(val: str) -> int | str: return True +def version_geq(lhs: str, rhs: str) -> bool: + """ + Returns True if version `lhs` is later or equal to `rhs`. + + Args: + lhs: version name to compare with `rhs`, return True if later or equal to `rhs`. + rhs: version name to compare with `lhs`, return True if earlier or equal to `lhs`. + + """ + lhs, rhs = str(lhs), str(rhs) + pkging, has_ver = optional_import("pkg_resources", name="packaging") + if has_ver: + try: + return cast(bool, pkging.version.Version(lhs) >= pkging.version.Version(rhs)) + except pkging.version.InvalidVersion: + return True + + lhs_, rhs_ = parse_version_strs(lhs, rhs) + for l, r in zip(lhs_, rhs_): + if l != r: + if isinstance(l, int) and isinstance(r, int): + return l > r + return f"{l}" > f"{r}" + + return True + + @functools.lru_cache(None) def pytorch_after(major: int, minor: int, patch: int = 0, current_ver_string: str | None = None) -> bool: """ diff --git a/tests/test_version_leq.py b/tests/test_version.py similarity index 90% rename from tests/test_version_leq.py rename to tests/test_version.py index ef9e70ad862..c98edbd13e2 100644 --- a/tests/test_version_leq.py +++ b/tests/test_version.py @@ -16,7 +16,7 @@ from parameterized import parameterized -from monai.utils import version_leq +from monai.utils import version_geq, version_leq # from pkg_resources @@ -80,6 +80,11 @@ def test_compare(self, a, b, expected=True): """Test version_leq with `a` and `b`""" self.assertEqual(version_leq(a, b), expected) + @parameterized.expand(TEST_CASES) + def test_compare(self, a, b, expected=True): + """Test version_geq with `b` and `a`""" + self.assertEqual(version_geq(b, a), expected) + if __name__ == "__main__": unittest.main() From e271ae21a795cd9afcec39955ed374fe3e5c2187 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 17:45:34 +0800 Subject: [PATCH 03/21] detect default tf32 settings when a function/class is imported Signed-off-by: Qingpeng Li --- monai/losses/__init__.py | 4 +++ monai/metrics/__init__.py | 4 +++ monai/networks/__init__.py | 4 +++ monai/optimizers/__init__.py | 4 +++ monai/transforms/__init__.py | 4 +++ monai/utils/__init__.py | 2 ++ monai/utils/tf32.py | 69 ++++++++++++++++++++++++++++++++++++ 7 files changed, 91 insertions(+) create mode 100644 monai/utils/tf32.py diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py index db6b133ef03..3dc619f3651 100644 --- a/monai/losses/__init__.py +++ b/monai/losses/__init__.py @@ -11,6 +11,10 @@ from __future__ import annotations +from monai.utils import detect_default_tf32 + +detect_default_tf32() + from .cldice import SoftclDiceLoss, SoftDiceclDiceLoss from .contrastive import ContrastiveLoss from .deform import BendingEnergyLoss diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index 4af1b5760d3..d1f7f3168cc 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -11,6 +11,10 @@ from __future__ import annotations +from monai.utils import detect_default_tf32 + +detect_default_tf32() + from .active_learning_metrics import LabelQualityScore, VarianceMetric, compute_variance, label_quality_score from .confusion_matrix import ConfusionMatrixMetric, compute_confusion_matrix_metric, get_confusion_matrix from .cumulative_average import CumulativeAverage diff --git a/monai/networks/__init__.py b/monai/networks/__init__.py index 4c429ae813a..68dcfcdaddf 100644 --- a/monai/networks/__init__.py +++ b/monai/networks/__init__.py @@ -11,6 +11,10 @@ from __future__ import annotations +from monai.utils import detect_default_tf32 + +detect_default_tf32() + from .utils import ( convert_to_onnx, convert_to_torchscript, diff --git a/monai/optimizers/__init__.py b/monai/optimizers/__init__.py index f0a3858ced0..e00471c6cc4 100644 --- a/monai/optimizers/__init__.py +++ b/monai/optimizers/__init__.py @@ -11,6 +11,10 @@ from __future__ import annotations +from monai.utils import detect_default_tf32 + +detect_default_tf32() + from .lr_finder import LearningRateFinder from .lr_scheduler import ExponentialLR, LinearLR, WarmupCosineSchedule from .novograd import Novograd diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py index eb8c5af19ef..cfdba42ccb8 100644 --- a/monai/transforms/__init__.py +++ b/monai/transforms/__init__.py @@ -11,6 +11,10 @@ from __future__ import annotations +from monai.utils import detect_default_tf32 + +detect_default_tf32() + from .adaptors import FunctionSignature, adaptor, apply_alias, to_kwargs from .compose import Compose, OneOf, RandomOrder, SomeOf from .croppad.array import ( diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py index 58bde3f5e8f..c973d4bfa18 100644 --- a/monai/utils/__init__.py +++ b/monai/utils/__init__.py @@ -115,6 +115,7 @@ require_pkg, run_debug, run_eval, + version_geq, version_leq, ) from .nvtx import Range @@ -128,6 +129,7 @@ torch_profiler_time_end_to_end, ) from .state_cacher import StateCacher +from .tf32 import detect_default_tf32, has_ampere_or_later from .type_conversion import ( convert_data_type, convert_to_cupy, diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py new file mode 100644 index 00000000000..c30f1b1436a --- /dev/null +++ b/monai/utils/tf32.py @@ -0,0 +1,69 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import os +import warnings + +import torch + +from monai.utils.module import pytorch_after, version_geq + +__all__ = ["has_ampere_or_later", "detect_default_tf32"] + + +def has_ampere_or_later() -> bool: + """ + Check if there is any Ampere and later GPU. + """ + if not torch.cuda.is_available(): + return False + if not version_geq(f"{torch.version.cuda}", "11.0"): + return False + for i in range(torch.cuda.device_count()): + major, _ = torch.cuda.get_device_capability(i) + if major >= 8: # Ampere and later + return True + return False + + +def detect_default_tf32() -> bool: + """ + Dectect if there is anything that may enable TF32 mode by default. + If any, show a warning message. + """ + may_enable_tf32 = False + try: + if not has_ampere_or_later(): + return False + + if pytorch_after(1, 7, 0) and not pytorch_after(1, 12, 0): + warnings.warn( + f"torch.backends.cuda.matmul.allow_tf32 = True by default.\n" + f" This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision\n" + f" See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" + ) + may_enable_tf32 = True + + override_tf32_env_vars = {"NVIDIA_TF32_OVERRIDE": "1", "TORCH_ALLOW_TF32_CUBLAS_OVERRIDE": "1"} + for name, override_val in override_tf32_env_vars.items(): + if os.environ.get(name) == override_val: + warnings.warn( + f"Environment variable `{name} = {override_val}` is set.\n" + f" This environment variable may enable TF32 mode accidentally and affect precision.\n" + f" See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" + ) + may_enable_tf32 = True + + return may_enable_tf32 + except: + return False From 5d286aa7b2c8b1f8e895e42c43f17b8dcd8d9d53 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 17:46:05 +0800 Subject: [PATCH 04/21] refactor `is_tf32_env()` Signed-off-by: Qingpeng Li --- tests/utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index bd357aa5294..4d7e86b6eca 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,7 +46,8 @@ from monai.data.meta_tensor import MetaTensor, get_track_meta from monai.networks import convert_to_onnx, convert_to_torchscript from monai.utils import optional_import -from monai.utils.module import pytorch_after, version_leq +from monai.utils.module import pytorch_after +from monai.utils.tf32 import has_ampere_or_later from monai.utils.type_conversion import convert_data_type nib, _ = optional_import("nibabel") @@ -179,11 +180,8 @@ def is_tf32_env(): global _tf32_enabled if _tf32_enabled is None: _tf32_enabled = False - if ( - torch.cuda.is_available() - and not version_leq(f"{torch.version.cuda}", "10.100") - and os.environ.get("NVIDIA_TF32_OVERRIDE", "1") != "0" - and torch.cuda.device_count() > 0 # at least 11.0 + if has_ampere_or_later() and ( + os.environ.get("NVIDIA_TF32_OVERRIDE") == "1" or os.environ.get("TORCH_ALLOW_TF32_CUBLAS_OVERRIDE") == "1" ): try: # with TF32 enabled, the speed is ~8x faster, but the precision has ~2 digits less in the result From 3b2345ab8853ff59cafbd32ad301fda8359493f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 09:52:14 +0000 Subject: [PATCH 05/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/utils/tf32.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index c30f1b1436a..944503bd660 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -48,9 +48,9 @@ def detect_default_tf32() -> bool: if pytorch_after(1, 7, 0) and not pytorch_after(1, 12, 0): warnings.warn( - f"torch.backends.cuda.matmul.allow_tf32 = True by default.\n" - f" This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision\n" - f" See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" + "torch.backends.cuda.matmul.allow_tf32 = True by default.\n" + " This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision\n" + " See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" ) may_enable_tf32 = True From 35438b89756c75080168ab03b0e117b33fb3df17 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 18:01:37 +0800 Subject: [PATCH 06/21] fix style E402 Signed-off-by: Qingpeng Li --- monai/losses/__init__.py | 8 +++++--- monai/metrics/__init__.py | 8 +++++--- monai/networks/__init__.py | 8 +++++--- monai/optimizers/__init__.py | 8 +++++--- monai/transforms/__init__.py | 8 +++++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py index 3dc619f3651..a2fcabcb76f 100644 --- a/monai/losses/__init__.py +++ b/monai/losses/__init__.py @@ -11,9 +11,11 @@ from __future__ import annotations -from monai.utils import detect_default_tf32 - -detect_default_tf32() +try: + from ..utils import detect_default_tf32 + detect_default_tf32() +except: + pass from .cldice import SoftclDiceLoss, SoftDiceclDiceLoss from .contrastive import ContrastiveLoss diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index d1f7f3168cc..74113aaeeea 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -11,9 +11,11 @@ from __future__ import annotations -from monai.utils import detect_default_tf32 - -detect_default_tf32() +try: + from ..utils import detect_default_tf32 + detect_default_tf32() +except: + pass from .active_learning_metrics import LabelQualityScore, VarianceMetric, compute_variance, label_quality_score from .confusion_matrix import ConfusionMatrixMetric, compute_confusion_matrix_metric, get_confusion_matrix diff --git a/monai/networks/__init__.py b/monai/networks/__init__.py index 68dcfcdaddf..b0c7691b882 100644 --- a/monai/networks/__init__.py +++ b/monai/networks/__init__.py @@ -11,9 +11,11 @@ from __future__ import annotations -from monai.utils import detect_default_tf32 - -detect_default_tf32() +try: + from ..utils import detect_default_tf32 + detect_default_tf32() +except: + pass from .utils import ( convert_to_onnx, diff --git a/monai/optimizers/__init__.py b/monai/optimizers/__init__.py index e00471c6cc4..509ea3d9641 100644 --- a/monai/optimizers/__init__.py +++ b/monai/optimizers/__init__.py @@ -11,9 +11,11 @@ from __future__ import annotations -from monai.utils import detect_default_tf32 - -detect_default_tf32() +try: + from ..utils import detect_default_tf32 + detect_default_tf32() +except: + pass from .lr_finder import LearningRateFinder from .lr_scheduler import ExponentialLR, LinearLR, WarmupCosineSchedule diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py index cfdba42ccb8..5340aeb4f4d 100644 --- a/monai/transforms/__init__.py +++ b/monai/transforms/__init__.py @@ -11,9 +11,11 @@ from __future__ import annotations -from monai.utils import detect_default_tf32 - -detect_default_tf32() +try: + from ..utils import detect_default_tf32 + detect_default_tf32() +except: + pass from .adaptors import FunctionSignature, adaptor, apply_alias, to_kwargs from .compose import Compose, OneOf, RandomOrder, SomeOf From 5697a31e35b36cbd4f13d8b78bc577b7776af1f0 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Thu, 3 Aug 2023 18:06:30 +0800 Subject: [PATCH 07/21] fix style E722 Signed-off-by: Qingpeng Li --- monai/losses/__init__.py | 2 +- monai/metrics/__init__.py | 2 +- monai/networks/__init__.py | 2 +- monai/optimizers/__init__.py | 2 +- monai/transforms/__init__.py | 2 +- monai/utils/tf32.py | 2 +- tests/test_version.py | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py index a2fcabcb76f..1ecbee1bdcf 100644 --- a/monai/losses/__init__.py +++ b/monai/losses/__init__.py @@ -14,7 +14,7 @@ try: from ..utils import detect_default_tf32 detect_default_tf32() -except: +except BaseException: pass from .cldice import SoftclDiceLoss, SoftDiceclDiceLoss diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index 74113aaeeea..5d55d4ae512 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -14,7 +14,7 @@ try: from ..utils import detect_default_tf32 detect_default_tf32() -except: +except BaseException: pass from .active_learning_metrics import LabelQualityScore, VarianceMetric, compute_variance, label_quality_score diff --git a/monai/networks/__init__.py b/monai/networks/__init__.py index b0c7691b882..30da159dd62 100644 --- a/monai/networks/__init__.py +++ b/monai/networks/__init__.py @@ -14,7 +14,7 @@ try: from ..utils import detect_default_tf32 detect_default_tf32() -except: +except BaseException: pass from .utils import ( diff --git a/monai/optimizers/__init__.py b/monai/optimizers/__init__.py index 509ea3d9641..b894d1f2056 100644 --- a/monai/optimizers/__init__.py +++ b/monai/optimizers/__init__.py @@ -14,7 +14,7 @@ try: from ..utils import detect_default_tf32 detect_default_tf32() -except: +except BaseException: pass from .lr_finder import LearningRateFinder diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py index 5340aeb4f4d..2f3913c5c49 100644 --- a/monai/transforms/__init__.py +++ b/monai/transforms/__init__.py @@ -14,7 +14,7 @@ try: from ..utils import detect_default_tf32 detect_default_tf32() -except: +except BaseException: pass from .adaptors import FunctionSignature, adaptor, apply_alias, to_kwargs diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 944503bd660..73fcbcc4fa0 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -65,5 +65,5 @@ def detect_default_tf32() -> bool: may_enable_tf32 = True return may_enable_tf32 - except: + except BaseException: return False diff --git a/tests/test_version.py b/tests/test_version.py index c98edbd13e2..15f8cd36c6b 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -76,12 +76,12 @@ def _pairwise(iterable): class TestVersionCompare(unittest.TestCase): @parameterized.expand(TEST_CASES) - def test_compare(self, a, b, expected=True): + def test_compare_leq(self, a, b, expected=True): """Test version_leq with `a` and `b`""" self.assertEqual(version_leq(a, b), expected) @parameterized.expand(TEST_CASES) - def test_compare(self, a, b, expected=True): + def test_compare_geq(self, a, b, expected=True): """Test version_geq with `b` and `a`""" self.assertEqual(version_geq(b, a), expected) From 948ccc37dcbb715326304ae66a056a1f550815e5 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Thu, 3 Aug 2023 10:21:02 +0000 Subject: [PATCH 08/21] [MONAI] code formatting Signed-off-by: monai-bot --- monai/losses/__init__.py | 1 + monai/metrics/__init__.py | 1 + monai/networks/__init__.py | 1 + monai/optimizers/__init__.py | 1 + monai/transforms/__init__.py | 1 + 5 files changed, 5 insertions(+) diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py index 1ecbee1bdcf..5368656abb8 100644 --- a/monai/losses/__init__.py +++ b/monai/losses/__init__.py @@ -13,6 +13,7 @@ try: from ..utils import detect_default_tf32 + detect_default_tf32() except BaseException: pass diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index 5d55d4ae512..4be95735578 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -13,6 +13,7 @@ try: from ..utils import detect_default_tf32 + detect_default_tf32() except BaseException: pass diff --git a/monai/networks/__init__.py b/monai/networks/__init__.py index 30da159dd62..f59da9ba04a 100644 --- a/monai/networks/__init__.py +++ b/monai/networks/__init__.py @@ -13,6 +13,7 @@ try: from ..utils import detect_default_tf32 + detect_default_tf32() except BaseException: pass diff --git a/monai/optimizers/__init__.py b/monai/optimizers/__init__.py index b894d1f2056..f50fdf032ed 100644 --- a/monai/optimizers/__init__.py +++ b/monai/optimizers/__init__.py @@ -13,6 +13,7 @@ try: from ..utils import detect_default_tf32 + detect_default_tf32() except BaseException: pass diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py index 2f3913c5c49..a017f785ef1 100644 --- a/monai/transforms/__init__.py +++ b/monai/transforms/__init__.py @@ -13,6 +13,7 @@ try: from ..utils import detect_default_tf32 + detect_default_tf32() except BaseException: pass From 96ca146ab4acd195886bf2d340a3e3bc576f99a3 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Fri, 4 Aug 2023 18:04:14 +0800 Subject: [PATCH 09/21] refactor the usage of `detect_default_tf32()` Signed-off-by: Qingpeng Li --- monai/__init__.py | 10 ++++++++++ monai/losses/__init__.py | 7 ------- monai/metrics/__init__.py | 7 ------- monai/networks/__init__.py | 7 ------- monai/optimizers/__init__.py | 7 ------- monai/transforms/__init__.py | 7 ------- monai/utils/tf32.py | 11 ++++++++--- 7 files changed, 18 insertions(+), 38 deletions(-) diff --git a/monai/__init__.py b/monai/__init__.py index 9f542dd25c4..638220f6df0 100644 --- a/monai/__init__.py +++ b/monai/__init__.py @@ -78,3 +78,13 @@ "utils", "visualize", ] + +try: + from .utils.tf32 import detect_default_tf32 + + detect_default_tf32() +except BaseException: + from .utils.misc import MONAIEnvVars + + if MONAIEnvVars.debug(): + raise diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py index 5368656abb8..db6b133ef03 100644 --- a/monai/losses/__init__.py +++ b/monai/losses/__init__.py @@ -11,13 +11,6 @@ from __future__ import annotations -try: - from ..utils import detect_default_tf32 - - detect_default_tf32() -except BaseException: - pass - from .cldice import SoftclDiceLoss, SoftDiceclDiceLoss from .contrastive import ContrastiveLoss from .deform import BendingEnergyLoss diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index 4be95735578..4af1b5760d3 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -11,13 +11,6 @@ from __future__ import annotations -try: - from ..utils import detect_default_tf32 - - detect_default_tf32() -except BaseException: - pass - from .active_learning_metrics import LabelQualityScore, VarianceMetric, compute_variance, label_quality_score from .confusion_matrix import ConfusionMatrixMetric, compute_confusion_matrix_metric, get_confusion_matrix from .cumulative_average import CumulativeAverage diff --git a/monai/networks/__init__.py b/monai/networks/__init__.py index f59da9ba04a..4c429ae813a 100644 --- a/monai/networks/__init__.py +++ b/monai/networks/__init__.py @@ -11,13 +11,6 @@ from __future__ import annotations -try: - from ..utils import detect_default_tf32 - - detect_default_tf32() -except BaseException: - pass - from .utils import ( convert_to_onnx, convert_to_torchscript, diff --git a/monai/optimizers/__init__.py b/monai/optimizers/__init__.py index f50fdf032ed..f0a3858ced0 100644 --- a/monai/optimizers/__init__.py +++ b/monai/optimizers/__init__.py @@ -11,13 +11,6 @@ from __future__ import annotations -try: - from ..utils import detect_default_tf32 - - detect_default_tf32() -except BaseException: - pass - from .lr_finder import LearningRateFinder from .lr_scheduler import ExponentialLR, LinearLR, WarmupCosineSchedule from .novograd import Novograd diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py index a017f785ef1..eb8c5af19ef 100644 --- a/monai/transforms/__init__.py +++ b/monai/transforms/__init__.py @@ -11,13 +11,6 @@ from __future__ import annotations -try: - from ..utils import detect_default_tf32 - - detect_default_tf32() -except BaseException: - pass - from .adaptors import FunctionSignature, adaptor, apply_alias, to_kwargs from .compose import Compose, OneOf, RandomOrder, SomeOf from .croppad.array import ( diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 73fcbcc4fa0..19bab627321 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -10,6 +10,7 @@ # limitations under the License. from __future__ import annotations +import functools import os import warnings @@ -20,7 +21,7 @@ __all__ = ["has_ampere_or_later", "detect_default_tf32"] - +@functools.lru_cache(None) def has_ampere_or_later() -> bool: """ Check if there is any Ampere and later GPU. @@ -35,7 +36,7 @@ def has_ampere_or_later() -> bool: return True return False - +@functools.lru_cache(None) def detect_default_tf32() -> bool: """ Dectect if there is anything that may enable TF32 mode by default. @@ -49,7 +50,7 @@ def detect_default_tf32() -> bool: if pytorch_after(1, 7, 0) and not pytorch_after(1, 12, 0): warnings.warn( "torch.backends.cuda.matmul.allow_tf32 = True by default.\n" - " This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision\n" + " This value defaults to True when PyTorch version in [1.7, 1.11] and may affect precision.\n" " See https://docs.monai.io/en/latest/precision_accelerating.html#precision-and-accelerating" ) may_enable_tf32 = True @@ -66,4 +67,8 @@ def detect_default_tf32() -> bool: return may_enable_tf32 except BaseException: + from monai.utils.misc import MONAIEnvVars + + if MONAIEnvVars.debug(): + raise return False From d8a65bbf29d5521b1a5ebe76e3b646e3b1c7a5ae Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Sat, 5 Aug 2023 02:35:27 +0800 Subject: [PATCH 10/21] improve `is_tf32_env()` Signed-off-by: Qingpeng Li --- tests/utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 4d7e86b6eca..1dfd5e6e209 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -47,7 +47,7 @@ from monai.networks import convert_to_onnx, convert_to_torchscript from monai.utils import optional_import from monai.utils.module import pytorch_after -from monai.utils.tf32 import has_ampere_or_later +from monai.utils.tf32 import detect_default_tf32 from monai.utils.type_conversion import convert_data_type nib, _ = optional_import("nibabel") @@ -173,16 +173,14 @@ def test_is_quick(): def is_tf32_env(): """ - The environment variable NVIDIA_TF32_OVERRIDE=0 will override any defaults - or programmatic configuration of NVIDIA libraries, and consequently, - cuBLAS will not accelerate FP32 computations with TF32 tensor cores. + When we may be using TF32 mode, check the precision of matrix operation. + If the checking result is greater than the threshold 0.001, + set _tf32_enabled=True (and relax _rtol for tests). """ global _tf32_enabled if _tf32_enabled is None: _tf32_enabled = False - if has_ampere_or_later() and ( - os.environ.get("NVIDIA_TF32_OVERRIDE") == "1" or os.environ.get("TORCH_ALLOW_TF32_CUBLAS_OVERRIDE") == "1" - ): + if detect_default_tf32() or torch.backends.cuda.matmul.allow_tf32: try: # with TF32 enabled, the speed is ~8x faster, but the precision has ~2 digits less in the result g_gpu = torch.Generator(device="cuda") From 93ff777188fef98e4f9dc38fa7f93686a35f34fe Mon Sep 17 00:00:00 2001 From: monai-bot Date: Fri, 4 Aug 2023 22:00:36 +0000 Subject: [PATCH 11/21] [MONAI] code formatting Signed-off-by: monai-bot --- monai/utils/tf32.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 19bab627321..58efaa3dbe2 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -10,8 +10,8 @@ # limitations under the License. from __future__ import annotations -import functools +import functools import os import warnings @@ -21,6 +21,7 @@ __all__ = ["has_ampere_or_later", "detect_default_tf32"] + @functools.lru_cache(None) def has_ampere_or_later() -> bool: """ @@ -36,6 +37,7 @@ def has_ampere_or_later() -> bool: return True return False + @functools.lru_cache(None) def detect_default_tf32() -> bool: """ From 348f089bcc916bfcb2aae0113a25d664ed33e5ae Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Sun, 6 Aug 2023 00:51:39 +0800 Subject: [PATCH 12/21] resolve `torch.cuda` initialization order issue Signed-off-by: Qingpeng Li --- monai/utils/__init__.py | 2 +- monai/utils/decorators.py | 24 +++++++++++++++++++++++- monai/utils/tf32.py | 12 +++++++----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py index c973d4bfa18..fc52694fb51 100644 --- a/monai/utils/__init__.py +++ b/monai/utils/__init__.py @@ -13,7 +13,7 @@ # have to explicitly bring these in here to resolve circular import issues from .aliases import alias, resolve_name -from .decorators import MethodReplacer, RestartGenerator +from .decorators import MethodReplacer, RestartGenerator, reset_torch_cuda_after_run from .deprecate_utils import DeprecatedError, deprecated, deprecated_arg, deprecated_arg_default from .dist import RankFilter, evenly_divisible_all_gather, get_dist_device, string_list_all_gather from .enums import ( diff --git a/monai/utils/decorators.py b/monai/utils/decorators.py index 1c064468e81..e5c933dea8c 100644 --- a/monai/utils/decorators.py +++ b/monai/utils/decorators.py @@ -13,7 +13,7 @@ from functools import wraps -__all__ = ["RestartGenerator", "MethodReplacer"] +__all__ = ["RestartGenerator", "MethodReplacer", "reset_torch_cuda_after_run"] from typing import Callable, Generator @@ -80,3 +80,25 @@ def newinit(_self, *args, **kwargs): namelist.append(entry) setattr(owner, name, self.meth) + +def reset_torch_cuda_after_run(func: Callable) -> Callable: + """ + To resolve `torch.cuda` initialization order issue. + If a function calls `torch.cuda` in a `__init__.py` file, + the function should be decorated by this decorator + to maintain CUDA lazy initialization. + + See https://github.com/Project-MONAI/MONAI/issues/2161 and + https://github.com/pytorch/pytorch/issues/80876 + """ + + @wraps(func) + def wrapped_func(*args, **kwargs): + val = func(*args, **kwargs) + + import importlib + import torch + importlib.reload(torch.cuda) + + return val + return wrapped_func diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 19bab627321..6af61b9d318 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -15,19 +15,19 @@ import os import warnings -import torch - -from monai.utils.module import pytorch_after, version_geq +from monai.utils.decorators import reset_torch_cuda_after_run __all__ = ["has_ampere_or_later", "detect_default_tf32"] @functools.lru_cache(None) +@reset_torch_cuda_after_run def has_ampere_or_later() -> bool: """ Check if there is any Ampere and later GPU. """ - if not torch.cuda.is_available(): - return False + import torch + from monai.utils.module import version_geq + if not version_geq(f"{torch.version.cuda}", "11.0"): return False for i in range(torch.cuda.device_count()): @@ -47,6 +47,8 @@ def detect_default_tf32() -> bool: if not has_ampere_or_later(): return False + from monai.utils.module import pytorch_after + if pytorch_after(1, 7, 0) and not pytorch_after(1, 12, 0): warnings.warn( "torch.backends.cuda.matmul.allow_tf32 = True by default.\n" From 9a4310f1483dc8a1e0d49d1dd566655bc19d7976 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Aug 2023 16:53:04 +0000 Subject: [PATCH 13/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/utils/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/utils/decorators.py b/monai/utils/decorators.py index e5c933dea8c..fafc232d2f3 100644 --- a/monai/utils/decorators.py +++ b/monai/utils/decorators.py @@ -87,7 +87,7 @@ def reset_torch_cuda_after_run(func: Callable) -> Callable: If a function calls `torch.cuda` in a `__init__.py` file, the function should be decorated by this decorator to maintain CUDA lazy initialization. - + See https://github.com/Project-MONAI/MONAI/issues/2161 and https://github.com/pytorch/pytorch/issues/80876 """ From a783158e5fae505ceaeec6adf2af7841f32258d1 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Sat, 5 Aug 2023 16:58:05 +0000 Subject: [PATCH 14/21] [MONAI] code formatting Signed-off-by: monai-bot --- monai/utils/decorators.py | 4 ++++ monai/utils/tf32.py | 1 + 2 files changed, 5 insertions(+) diff --git a/monai/utils/decorators.py b/monai/utils/decorators.py index fafc232d2f3..f685a53d18b 100644 --- a/monai/utils/decorators.py +++ b/monai/utils/decorators.py @@ -81,6 +81,7 @@ def newinit(_self, *args, **kwargs): setattr(owner, name, self.meth) + def reset_torch_cuda_after_run(func: Callable) -> Callable: """ To resolve `torch.cuda` initialization order issue. @@ -97,8 +98,11 @@ def wrapped_func(*args, **kwargs): val = func(*args, **kwargs) import importlib + import torch + importlib.reload(torch.cuda) return val + return wrapped_func diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index cf518390cc8..64162793f22 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -27,6 +27,7 @@ def has_ampere_or_later() -> bool: Check if there is any Ampere and later GPU. """ import torch + from monai.utils.module import version_geq if not version_geq(f"{torch.version.cuda}", "11.0"): From 13cd277f718f38e109e8d27c64dde269cf8b93b0 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Mon, 7 Aug 2023 18:07:57 +0800 Subject: [PATCH 15/21] use `pynvml` to avoid `torch.cuda` call Signed-off-by: Qingpeng Li --- monai/utils/__init__.py | 2 +- monai/utils/decorators.py | 28 +--------------------------- monai/utils/tf32.py | 22 ++++++++++++++-------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py index fc52694fb51..c973d4bfa18 100644 --- a/monai/utils/__init__.py +++ b/monai/utils/__init__.py @@ -13,7 +13,7 @@ # have to explicitly bring these in here to resolve circular import issues from .aliases import alias, resolve_name -from .decorators import MethodReplacer, RestartGenerator, reset_torch_cuda_after_run +from .decorators import MethodReplacer, RestartGenerator from .deprecate_utils import DeprecatedError, deprecated, deprecated_arg, deprecated_arg_default from .dist import RankFilter, evenly_divisible_all_gather, get_dist_device, string_list_all_gather from .enums import ( diff --git a/monai/utils/decorators.py b/monai/utils/decorators.py index f685a53d18b..1c064468e81 100644 --- a/monai/utils/decorators.py +++ b/monai/utils/decorators.py @@ -13,7 +13,7 @@ from functools import wraps -__all__ = ["RestartGenerator", "MethodReplacer", "reset_torch_cuda_after_run"] +__all__ = ["RestartGenerator", "MethodReplacer"] from typing import Callable, Generator @@ -80,29 +80,3 @@ def newinit(_self, *args, **kwargs): namelist.append(entry) setattr(owner, name, self.meth) - - -def reset_torch_cuda_after_run(func: Callable) -> Callable: - """ - To resolve `torch.cuda` initialization order issue. - If a function calls `torch.cuda` in a `__init__.py` file, - the function should be decorated by this decorator - to maintain CUDA lazy initialization. - - See https://github.com/Project-MONAI/MONAI/issues/2161 and - https://github.com/pytorch/pytorch/issues/80876 - """ - - @wraps(func) - def wrapped_func(*args, **kwargs): - val = func(*args, **kwargs) - - import importlib - - import torch - - importlib.reload(torch.cuda) - - return val - - return wrapped_func diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 64162793f22..9d1a87d1944 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -15,27 +15,33 @@ import os import warnings -from monai.utils.decorators import reset_torch_cuda_after_run - __all__ = ["has_ampere_or_later", "detect_default_tf32"] @functools.lru_cache(None) -@reset_torch_cuda_after_run def has_ampere_or_later() -> bool: """ Check if there is any Ampere and later GPU. """ import torch - from monai.utils.module import version_geq if not version_geq(f"{torch.version.cuda}", "11.0"): return False - for i in range(torch.cuda.device_count()): - major, _ = torch.cuda.get_device_capability(i) - if major >= 8: # Ampere and later - return True + + from pynvml import nvmlInit, nvmlShutdown, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, nvmlDeviceGetCudaComputeCapability + try: + nvmlInit() + for i in range(nvmlDeviceGetCount()): + handle = nvmlDeviceGetHandleByIndex(i) + major, _ = nvmlDeviceGetCudaComputeCapability(handle) + if major >= 8: + return True + except BaseException: + pass + finally: + nvmlShutdown() + return False From fddc87f89735db9373c1b11d75e377fd3f219e73 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Mon, 7 Aug 2023 18:18:25 +0800 Subject: [PATCH 16/21] minor fix Signed-off-by: Qingpeng Li --- monai/utils/tf32.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index 9d1a87d1944..fd4917b2b1f 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -26,7 +26,7 @@ def has_ampere_or_later() -> bool: import torch from monai.utils.module import version_geq - if not version_geq(f"{torch.version.cuda}", "11.0"): + if not (torch.version.cuda and version_geq(f"{torch.version.cuda}", "11.0")): return False from pynvml import nvmlInit, nvmlShutdown, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, nvmlDeviceGetCudaComputeCapability diff --git a/requirements.txt b/requirements.txt index 15696467945..7f2e3ecbfdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ torch>=1.9 numpy>=1.20 +nvidia-ml-py \ No newline at end of file From 2a6ff88ee5dcc396281f171a10448b8fef82401c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 10:19:10 +0000 Subject: [PATCH 17/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7f2e3ecbfdd..1107804de11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ torch>=1.9 numpy>=1.20 -nvidia-ml-py \ No newline at end of file +nvidia-ml-py From e1b207532c4089128a65004915d2cdf3667418ec Mon Sep 17 00:00:00 2001 From: monai-bot Date: Mon, 7 Aug 2023 10:24:56 +0000 Subject: [PATCH 18/21] [MONAI] code formatting Signed-off-by: monai-bot --- monai/utils/tf32.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index fd4917b2b1f..f16b3070c7a 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -24,12 +24,20 @@ def has_ampere_or_later() -> bool: Check if there is any Ampere and later GPU. """ import torch + from monai.utils.module import version_geq if not (torch.version.cuda and version_geq(f"{torch.version.cuda}", "11.0")): return False - from pynvml import nvmlInit, nvmlShutdown, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, nvmlDeviceGetCudaComputeCapability + from pynvml import ( + nvmlDeviceGetCount, + nvmlDeviceGetCudaComputeCapability, + nvmlDeviceGetHandleByIndex, + nvmlInit, + nvmlShutdown, + ) + try: nvmlInit() for i in range(nvmlDeviceGetCount()): From 72c8a721401430e055acd5fed71b7b810ff1ade0 Mon Sep 17 00:00:00 2001 From: Qingpeng Li Date: Mon, 7 Aug 2023 22:03:08 +0800 Subject: [PATCH 19/21] fix import `pynvml` Signed-off-by: Qingpeng Li --- docs/source/installation.md | 4 ++-- monai/utils/tf32.py | 22 +++++++++------------- requirements-dev.txt | 1 + requirements.txt | 1 - setup.cfg | 3 +++ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/source/installation.md b/docs/source/installation.md index 6d63fbf08fb..6580ce67172 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -254,11 +254,11 @@ Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is - The options are ``` -[nibabel, skimage, scipy, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, clearml, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna, onnx, onnxruntime, zarr, lpips] +[nibabel, skimage, scipy, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers, mlflow, clearml, matplotlib, tensorboardX, tifffile, imagecodecs, pyyaml, fire, jsonschema, ninja, pynrrd, pydicom, h5py, nni, optuna, onnx, onnxruntime, zarr, lpips, pynvml] ``` which correspond to `nibabel`, `scikit-image`,`scipy`, `pillow`, `tensorboard`, -`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `clearml`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, `onnx`, `onnxruntime`, `zarr` and `lpips` respectively. +`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`, `transformers`, `mlflow`, `clearml`, `matplotlib`, `tensorboardX`, `tifffile`, `imagecodecs`, `pyyaml`, `fire`, `jsonschema`, `ninja`, `pynrrd`, `pydicom`, `h5py`, `nni`, `optuna`, `onnx`, `onnxruntime`, `zarr`, `lpips` and `nvidia-ml-py` respectively. - `pip install 'monai[all]'` installs all the optional dependencies. diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index f16b3070c7a..dd7b76b59f7 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -25,30 +25,26 @@ def has_ampere_or_later() -> bool: """ import torch - from monai.utils.module import version_geq + from monai.utils.module import version_geq, optional_import if not (torch.version.cuda and version_geq(f"{torch.version.cuda}", "11.0")): return False - from pynvml import ( - nvmlDeviceGetCount, - nvmlDeviceGetCudaComputeCapability, - nvmlDeviceGetHandleByIndex, - nvmlInit, - nvmlShutdown, - ) + pynvml, has_pynvml = optional_import("pynvml") + if not has_pynvml: # assuming that the user has Ampere and later GPU + return True try: - nvmlInit() - for i in range(nvmlDeviceGetCount()): - handle = nvmlDeviceGetHandleByIndex(i) - major, _ = nvmlDeviceGetCudaComputeCapability(handle) + pynvml.nvmlInit() + for i in range(pynvml.nvmlDeviceGetCount()): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + major, _ = pynvml.nvmlDeviceGetCudaComputeCapability(handle) if major >= 8: return True except BaseException: pass finally: - nvmlShutdown() + pynvml.nvmlShutdown() return False diff --git a/requirements-dev.txt b/requirements-dev.txt index f0e3a37c240..9620ea253d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -55,3 +55,4 @@ typeguard<3 # https://github.com/microsoft/nni/issues/5457 filelock!=3.12.0 # https://github.com/microsoft/nni/issues/5523 zarr lpips==0.1.4 +nvidia-ml-py diff --git a/requirements.txt b/requirements.txt index 1107804de11..15696467945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ torch>=1.9 numpy>=1.20 -nvidia-ml-py diff --git a/setup.cfg b/setup.cfg index 65d5bce2c73..f5723aca2cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,7 @@ all = onnxruntime; python_version <= '3.10' zarr lpips==0.1.4 + nvidia-ml-py nibabel = nibabel ninja = @@ -151,6 +152,8 @@ zarr = zarr lpips = lpips==0.1.4 +pynvml = + nvidia-ml-py # # workaround https://github.com/Project-MONAI/MONAI/issues/5882 # MetricsReloaded = # MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support#egg=MetricsReloaded From 055ed461d47141623c3d868efe14358f711a189c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 14:04:09 +0000 Subject: [PATCH 20/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b5877cfe29a..9d5a22963c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -154,7 +154,7 @@ zarr = zarr lpips = lpips==0.1.4 -pynvml = +pynvml = nvidia-ml-py # # workaround https://github.com/Project-MONAI/MONAI/issues/5882 # MetricsReloaded = From f4571f179e4b67ec75a9de169995bc6dcc48fa62 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Mon, 7 Aug 2023 15:28:52 +0000 Subject: [PATCH 21/21] [MONAI] code formatting Signed-off-by: monai-bot --- monai/utils/tf32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py index dd7b76b59f7..9ef425ab8bd 100644 --- a/monai/utils/tf32.py +++ b/monai/utils/tf32.py @@ -25,7 +25,7 @@ def has_ampere_or_later() -> bool: """ import torch - from monai.utils.module import version_geq, optional_import + from monai.utils.module import optional_import, version_geq if not (torch.version.cuda and version_geq(f"{torch.version.cuda}", "11.0")): return False