From 99f8e75d9a9ed2d5e8d29590dfb22753d4e1e107 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Fri, 28 Feb 2020 13:52:17 +0000 Subject: [PATCH 01/11] Adding new factory concept. --- .../engine/multi_gpu_supervised_trainer.py~ | 95 ++++++++++ monai/networks/blocks/convolutions.py | 57 ++++-- monai/networks/layers/factories.py | 165 +++++++++++++++++- monai/networks/nets/unet.py | 32 ++-- tests/test_unet.py | 61 ++++++- 5 files changed, 368 insertions(+), 42 deletions(-) create mode 100644 monai/application/engine/multi_gpu_supervised_trainer.py~ diff --git a/monai/application/engine/multi_gpu_supervised_trainer.py~ b/monai/application/engine/multi_gpu_supervised_trainer.py~ new file mode 100644 index 00000000000..912d4cc9ad2 --- /dev/null +++ b/monai/application/engine/multi_gpu_supervised_trainer.py~ @@ -0,0 +1,95 @@ +# Copyright 2020 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch + +from ignite.engine import create_supervised_trainer, create_supervised_evaluator, _prepare_batch + +import monai + + +def _default_transform(x, y, y_pred, loss): + return loss.item() + +def _default_eval_transform(x, y, y_pred): + return y_pred, y + +@monai.utils.export("monai.application.engine") +def create_multigpu_supervised_trainer(net, optimizer, loss_fn, devices=None, non_blocking=False, + prepare_batch=_prepare_batch, output_transform=_default_transform): + """ + ***Derived from `create_supervised_trainer` in Ignite. + + Factory function for creating a trainer for supervised models. + Args: + net (`torch.nn.Module`): the network to train. + optimizer (`torch.optim.Optimizer`): the optimizer to use. + loss_fn (torch.nn loss function): the loss function to use. + devices (list, optional): device(s) type specification (default: None). + Applies to both model and batches. None is all devices used, empty list is CPU only. + non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs + tuple of tensors `(batch_x, batch_y)`. + output_transform (callable, optional): function that receives 'x', 'y', 'y_pred', 'loss' and returns value + to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`. + Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is the loss + of the processed batch by default. + Returns: + Engine: a trainer engine with supervised update function. + """ + + if devices is None: + devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] + elif len(devices) == 0: + devices = [torch.device("cpu")] + + if len(devices) > 1: + net = torch.nn.parallel.DataParallel(net) + + return create_supervised_trainer(net, optimizer, loss_fn, devices[0], non_blocking, prepare_batch, output_transform) + + +@monai.utils.export("monai.application.engine") +def create_multigpu_supervised_evaluator(net, metrics=None, device=None, non_blocking=False, + prepare_batch=_prepare_batch, output_transform=_default_eval_transform): + """ + ***Derived from `create_supervised_evaluator` in Ignite. + + Factory function for creating an evaluator for supervised models. + Args: + net (`torch.nn.Module`): the model to train. + metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics. + devices (list, optional): device(s) type specification (default: None). + Applies to both model and batches. None is all devices used, empty list is CPU only. + non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs + tuple of tensors `(batch_x, batch_y)`. + output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value + to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits + output expected by metrics. If you change it you should use `output_transform` in metrics. + Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is + a tuple of `(batch_pred, batch_y)` by default. + Returns: + Engine: an evaluator engine with supervised inference function. + """ + + if devices is None: + devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] + elif len(devices) == 0: + devices = [torch.device("cpu")] + + if len(devices) > 1: + net = torch.nn.parallel.DataParallel(net) + + return create_supervised_evaluator(net, metrics, devices[0], non_blocking, prepare_batch, output_transform) diff --git a/monai/networks/blocks/convolutions.py b/monai/networks/blocks/convolutions.py index b8b2814c673..1ca2c91ec89 100644 --- a/monai/networks/blocks/convolutions.py +++ b/monai/networks/blocks/convolutions.py @@ -12,14 +12,17 @@ import numpy as np import torch.nn as nn -from monai.networks.layers.factories import get_conv_type, get_dropout_type, get_normalize_type +from monai.networks.layers.factories import Dropout, Norm, Act, Conv, split_args from monai.networks.layers.convutils import same_padding class Convolution(nn.Sequential): - - def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, instance_norm=True, dropout=0, - dilation=1, bias=True, conv_only=False, is_transposed=False): + """ + Constructs a convolution with optional dropout, normalization, and activation layers. + """ + + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, act=Act.PRELU, + norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, conv_only=False, is_transposed=False): super().__init__() self.dimensions = dimensions self.in_channels = in_channels @@ -27,9 +30,25 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size self.is_transposed = is_transposed padding = same_padding(kernel_size, dilation) - normalize_type = get_normalize_type(dimensions, instance_norm) - conv_type = get_conv_type(dimensions, is_transposed) - drop_type = get_dropout_type(dimensions) + conv_type = Conv[Conv.CONVTRANS if is_transposed else Conv.CONV, dimensions] + + # define the normalisation type and the arguments to the constructor + norm_name, norm_args = split_args(norm) + norm_type = Norm[norm_name, dimensions] + + # define the activation type and the arguments to the constructor + act_name, act_args = split_args(act) + act_type = Act[act_name] + + if dropout: + # if dropout was specified simply as a p value, use default name and make a keyword map with the value + if isinstance(dropout, (int, float)): + drop_name = Dropout.DROPOUT + drop_args = {"p": dropout} + else: + drop_name, drop_args = split_args(dropout) + + drop_type = Dropout[drop_name, dimensions] if is_transposed: conv = conv_type(in_channels, out_channels, kernel_size, strides, padding, strides - 1, 1, bias, dilation) @@ -39,17 +58,16 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size self.add_module("conv", conv) if not conv_only: - self.add_module("norm", normalize_type(out_channels)) - if dropout > 0: # omitting Dropout2d appears faster than relying on it short-circuiting when dropout==0 - self.add_module("dropout", drop_type(dropout)) + self.add_module("norm", norm_type(out_channels, **norm_args)) + if dropout: + self.add_module("dropout", drop_type(**drop_args)) - self.add_module("prelu", nn.modules.PReLU()) + self.add_module("act", act_type(**act_args)) class ResidualUnit(nn.Module): - - def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, subunits=2, instance_norm=True, - dropout=0, dilation=1, bias=True, last_conv_only=False): + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, subunits=2, + act=Act.PRELU, norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, last_conv_only=False): super().__init__() self.dimensions = dimensions self.in_channels = in_channels @@ -64,10 +82,13 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size for su in range(subunits): conv_only = last_conv_only and su == (subunits - 1) - unit = Convolution(dimensions, schannels, out_channels, sstrides, kernel_size, instance_norm, dropout, - dilation, bias, conv_only) + unit = Convolution(dimensions, schannels, out_channels, sstrides, + kernel_size, act, norm, dropout, dilation, bias, conv_only) + self.conv.add_module("unit%i" % su, unit) - schannels = out_channels # after first loop set channels and strides to what they should be for subsequent units + + # after first loop set channels and strides to what they should be for subsequent units + schannels = out_channels sstrides = 1 # apply convolution to input to change number of output channels and size to match that coming from self.conv @@ -79,7 +100,7 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size rkernel_size = 1 rpadding = 0 - conv_type = get_conv_type(dimensions, False) + conv_type = Conv[Conv.CONV, dimensions] self.residual = conv_type(in_channels, out_channels, rkernel_size, strides, rpadding, bias=bias) def forward(self, x): diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index e848c3185b7..b6486794404 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -8,8 +8,169 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +Defines factories for creating layers in generic, extensible, and dimensionally independent ways. A separate factory +object is created for each type of layer, and factory functions keyed to names are added to these objects. Whenever +a layer is requested the factory name and any necessary arguments are passed to the factory object. The return value +is typically a type but can be any callable producing a layer object. -from torch import nn as nn +The factory objects contain functions keyed to names converted to upper case, these names can be referred to as members +of the factory so that they can function as constant identifiers. eg. instance normalisation is named `Norm.INSTANCE`. + +For example, to get a transpose convolution layer the name is needed and then a dimension argument is provided which is +passed to the factory function: + + dimension = 3 + name = Conv.CONVTRANS + conv = Conv[name, dimension] + +This allows the `dimension` value to be set in constructor for example so that the dimensionality of a network is +parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. + +""" + +from typing import Callable + +import torch +import torch.nn as nn + + +class LayerFactory: + """ + Factory object for creating layers, this uses given factory functions to actually produce the types or constructing + callables. These functions are referred to by name and can be added at any time. + """ + + def __init__(self): + self.factories = {} + + @property + def names(self): + """ + Produces all factory names. + """ + + return tuple(self.factories) + + def add_factory_callable(self, name, func): + """ + Add the factory function to this object under the given name. + """ + + self.factories[name.upper()] = func + + def factory_function(self, name): + """ + Decorator for adding a factory function with the given name. + """ + + def _add(func): + self.add_factory_callable(name, func) + return func + + return _add + + def get_constructor(self, factory_name, *args): + """ + Get the constructor for the given factory name and arguments. + """ + + if not isinstance(factory_name, str): + raise ValueError("Factories must be selected by name") + + fact = self.factories[factory_name.upper()] + return fact(*args) + + def __getitem__(self, args): + """ + Get the given name or name/arguments pair. If `args` is a callable it is assumed to be the constructor + itself and is returned, otherwise it should be the factory name or a pair containing the name and arguments. + """ + + # `args` is actually a type or constructor + if callable(args): + return args + + # `args` is a factory name or a name with arguments + if isinstance(args, str): + name_obj, args = args, () + else: + name_obj, *args = args + + return self.get_constructor(name_obj, *args) + + def __getattr__(self, key): + """ + If `key` is a factory name, return it, otherwise behave as inherited. This allows referring to factory names + as if they were constants, eg. `Fact.FOO` for a factory Fact with factory function foo. + """ + + if key in self.factories: + return key + + return super().__getattr__(key) + + +def split_args(args): + """ + Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted + """ + + if isinstance(args,str): + return args,{} + else: + name_obj,args=args + + if not isinstance(name_obj,(str,Callable)) or not isinstance(args,dict): + raise ValueError("Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)") + + return name_obj, args + + +### Define factories for these layer types + +Dropout = LayerFactory() +Norm = LayerFactory() +Act = LayerFactory() +Conv = LayerFactory() + + +@Dropout.factory_function("dropout") +def get_dropout_type(dim): + types = [nn.Dropout, nn.Dropout2d, nn.Dropout3d] + return types[dim - 1] + + +@Norm.factory_function("instance") +def get_instance_type(dim): + types = [nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] + return types[dim - 1] + + +@Norm.factory_function("batch") +def get_batch_type(dim): + types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] + return types[dim - 1] + + +Act.add_factory_callable("relu", lambda: nn.modules.ReLU) +Act.add_factory_callable("leakyrelu", lambda: nn.modules.LeakyReLU) +Act.add_factory_callable("prelu", lambda: nn.modules.PReLU) + + +@Conv.factory_function("conv") +def get_conv_type(dim): + types = [nn.Conv1d, nn.Conv2d, nn.Conv3d] + return types[dim - 1] + + +@Conv.factory_function("convtrans") +def get_convtrans_type(dim): + types = [nn.ConvTranspose1d, nn.ConvTranspose2d, nn.ConvTranspose3d] + return types[dim - 1] + + +### old factory functions def get_conv_type(dim, is_transpose): @@ -33,3 +194,5 @@ def get_normalize_type(dim, is_instance): types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] return types[dim - 1] + + diff --git a/monai/networks/nets/unet.py b/monai/networks/nets/unet.py index b0d42612eb9..37c93fb9ae5 100644 --- a/monai/networks/nets/unet.py +++ b/monai/networks/nets/unet.py @@ -12,6 +12,7 @@ import torch.nn as nn from monai.networks.blocks.convolutions import Convolution, ResidualUnit +from monai.networks.layers.factories import Norm, Act from monai.networks.layers.simplelayers import SkipConnection from monai.networks.utils import predict_segmentation from monai.utils import export @@ -23,7 +24,7 @@ class UNet(nn.Module): def __init__(self, dimensions, in_channels, num_classes, channels, strides, kernel_size=3, up_kernel_size=3, - num_res_units=0, instance_norm=True, dropout=0): + num_res_units=0, act=Act.PRELU, norm=Norm.INSTANCE, dropout=0): super().__init__() assert len(channels) == (len(strides) + 1) self.dimensions = dimensions @@ -34,7 +35,8 @@ def __init__(self, dimensions, in_channels, num_classes, channels, strides, kern self.kernel_size = kernel_size self.up_kernel_size = up_kernel_size self.num_res_units = num_res_units - self.instance_norm = instance_norm + self.act = act + self.norm = norm self.dropout = dropout def _create_block(inc, outc, channels, strides, is_top): @@ -63,35 +65,21 @@ def _create_block(inc, outc, channels, strides, is_top): def _get_down_layer(self, in_channels, out_channels, strides, is_top): if self.num_res_units > 0: return ResidualUnit(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.num_res_units, - self.instance_norm, self.dropout) + self.act, self.norm, self.dropout) else: - return Convolution(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.instance_norm, + return Convolution(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.act, self.norm, self.dropout) def _get_bottom_layer(self, in_channels, out_channels): return self._get_down_layer(in_channels, out_channels, 1, False) def _get_up_layer(self, in_channels, out_channels, strides, is_top): - conv = Convolution(self.dimensions, - in_channels, - out_channels, - strides, - self.up_kernel_size, - self.instance_norm, - self.dropout, - conv_only=is_top and self.num_res_units == 0, - is_transposed=True) + conv = Convolution(self.dimensions, in_channels, out_channels, strides, self.up_kernel_size, self.act, self.norm, + self.dropout, conv_only=is_top and self.num_res_units == 0, is_transposed=True) if self.num_res_units > 0: - ru = ResidualUnit(self.dimensions, - out_channels, - out_channels, - 1, - self.kernel_size, - 1, - self.instance_norm, - self.dropout, - last_conv_only=is_top) + ru = ResidualUnit(self.dimensions, out_channels, out_channels, 1, self.kernel_size, 1, self.act, self.norm, + self.dropout, last_conv_only=is_top) return nn.Sequential(conv, ru) else: return conv diff --git a/tests/test_unet.py b/tests/test_unet.py index 98102375a61..edc9ce74673 100644 --- a/tests/test_unet.py +++ b/tests/test_unet.py @@ -14,8 +14,23 @@ import torch from parameterized import parameterized +from monai.networks.layers.factories import Norm, Act from monai.networks.nets.unet import UNet + +TEST_CASE_0 = [ # single channel 2D, batch 16, no residual + { + 'dimensions': 2, + 'in_channels': 1, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 0, + }, + torch.randn(16, 1, 32, 32), + (16, 32, 32), +] + TEST_CASE_1 = [ # single channel 2D, batch 16 { 'dimensions': 2, @@ -55,10 +70,54 @@ (16, 32, 64, 48), ] +TEST_CASE_4 = [ # 4-channel 3D, batch 16, batch normalisation + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'norm': Norm.BATCH, + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +TEST_CASE_5 = [ # 4-channel 3D, batch 16, LeakyReLU activation + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'act': (Act.LEAKYRELU, {'negative_slope': 0.2}), + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +TEST_CASE_6 = [ # 4-channel 3D, batch 16, LeakyReLU activation explicit + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'act': (torch.nn.LeakyReLU, {'negative_slope': 0.2}), + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +CASES = [TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6] + class TestUNET(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3]) + @parameterized.expand(CASES) def test_shape(self, input_param, input_data, expected_shape): net = UNet(**input_param) net.eval() From e84922bf0c72ab2197ece870c25ba0a9566e670a Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Fri, 28 Feb 2020 13:59:52 +0000 Subject: [PATCH 02/11] Tweaks --- .../engine/multi_gpu_supervised_trainer.py~ | 95 ------------------- monai/networks/blocks/convolutions.py | 6 +- monai/networks/layers/factories.py | 64 ++++++------- 3 files changed, 34 insertions(+), 131 deletions(-) delete mode 100644 monai/application/engine/multi_gpu_supervised_trainer.py~ diff --git a/monai/application/engine/multi_gpu_supervised_trainer.py~ b/monai/application/engine/multi_gpu_supervised_trainer.py~ deleted file mode 100644 index 912d4cc9ad2..00000000000 --- a/monai/application/engine/multi_gpu_supervised_trainer.py~ +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2020 MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import torch - -from ignite.engine import create_supervised_trainer, create_supervised_evaluator, _prepare_batch - -import monai - - -def _default_transform(x, y, y_pred, loss): - return loss.item() - -def _default_eval_transform(x, y, y_pred): - return y_pred, y - -@monai.utils.export("monai.application.engine") -def create_multigpu_supervised_trainer(net, optimizer, loss_fn, devices=None, non_blocking=False, - prepare_batch=_prepare_batch, output_transform=_default_transform): - """ - ***Derived from `create_supervised_trainer` in Ignite. - - Factory function for creating a trainer for supervised models. - Args: - net (`torch.nn.Module`): the network to train. - optimizer (`torch.optim.Optimizer`): the optimizer to use. - loss_fn (torch.nn loss function): the loss function to use. - devices (list, optional): device(s) type specification (default: None). - Applies to both model and batches. None is all devices used, empty list is CPU only. - non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs - tuple of tensors `(batch_x, batch_y)`. - output_transform (callable, optional): function that receives 'x', 'y', 'y_pred', 'loss' and returns value - to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`. - Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is the loss - of the processed batch by default. - Returns: - Engine: a trainer engine with supervised update function. - """ - - if devices is None: - devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] - elif len(devices) == 0: - devices = [torch.device("cpu")] - - if len(devices) > 1: - net = torch.nn.parallel.DataParallel(net) - - return create_supervised_trainer(net, optimizer, loss_fn, devices[0], non_blocking, prepare_batch, output_transform) - - -@monai.utils.export("monai.application.engine") -def create_multigpu_supervised_evaluator(net, metrics=None, device=None, non_blocking=False, - prepare_batch=_prepare_batch, output_transform=_default_eval_transform): - """ - ***Derived from `create_supervised_evaluator` in Ignite. - - Factory function for creating an evaluator for supervised models. - Args: - net (`torch.nn.Module`): the model to train. - metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics. - devices (list, optional): device(s) type specification (default: None). - Applies to both model and batches. None is all devices used, empty list is CPU only. - non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs - tuple of tensors `(batch_x, batch_y)`. - output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value - to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits - output expected by metrics. If you change it you should use `output_transform` in metrics. - Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is - a tuple of `(batch_pred, batch_y)` by default. - Returns: - Engine: an evaluator engine with supervised inference function. - """ - - if devices is None: - devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] - elif len(devices) == 0: - devices = [torch.device("cpu")] - - if len(devices) > 1: - net = torch.nn.parallel.DataParallel(net) - - return create_supervised_evaluator(net, metrics, devices[0], non_blocking, prepare_batch, output_transform) diff --git a/monai/networks/blocks/convolutions.py b/monai/networks/blocks/convolutions.py index 1ca2c91ec89..3c618636d0f 100644 --- a/monai/networks/blocks/convolutions.py +++ b/monai/networks/blocks/convolutions.py @@ -20,7 +20,7 @@ class Convolution(nn.Sequential): """ Constructs a convolution with optional dropout, normalization, and activation layers. """ - + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, act=Act.PRELU, norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, conv_only=False, is_transposed=False): super().__init__() @@ -84,9 +84,9 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size conv_only = last_conv_only and su == (subunits - 1) unit = Convolution(dimensions, schannels, out_channels, sstrides, kernel_size, act, norm, dropout, dilation, bias, conv_only) - + self.conv.add_module("unit%i" % su, unit) - + # after first loop set channels and strides to what they should be for subsequent units schannels = out_channels sstrides = 1 diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index b6486794404..644b9c509bd 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -23,15 +23,13 @@ dimension = 3 name = Conv.CONVTRANS conv = Conv[name, dimension] - + This allows the `dimension` value to be set in constructor for example so that the dimensionality of a network is parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. - """ from typing import Callable -import torch import torch.nn as nn @@ -40,7 +38,7 @@ class LayerFactory: Factory object for creating layers, this uses given factory functions to actually produce the types or constructing callables. These functions are referred to by name and can be added at any time. """ - + def __init__(self): self.factories = {} @@ -49,35 +47,35 @@ def names(self): """ Produces all factory names. """ - + return tuple(self.factories) def add_factory_callable(self, name, func): """ Add the factory function to this object under the given name. """ - + self.factories[name.upper()] = func def factory_function(self, name): """ Decorator for adding a factory function with the given name. """ - + def _add(func): self.add_factory_callable(name, func) return func return _add - + def get_constructor(self, factory_name, *args): """ Get the constructor for the given factory name and arguments. """ - + if not isinstance(factory_name, str): raise ValueError("Factories must be selected by name") - + fact = self.factories[factory_name.upper()] return fact(*args) @@ -86,11 +84,11 @@ def __getitem__(self, args): Get the given name or name/arguments pair. If `args` is a callable it is assumed to be the constructor itself and is returned, otherwise it should be the factory name or a pair containing the name and arguments. """ - + # `args` is actually a type or constructor if callable(args): return args - + # `args` is a factory name or a name with arguments if isinstance(args, str): name_obj, args = args, () @@ -104,31 +102,33 @@ def __getattr__(self, key): If `key` is a factory name, return it, otherwise behave as inherited. This allows referring to factory names as if they were constants, eg. `Fact.FOO` for a factory Fact with factory function foo. """ - + if key in self.factories: return key return super().__getattr__(key) - + def split_args(args): """ Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted """ - - if isinstance(args,str): - return args,{} + + if isinstance(args, str): + return args, {} else: - name_obj,args=args - - if not isinstance(name_obj,(str,Callable)) or not isinstance(args,dict): - raise ValueError("Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)") - + name_obj, args = args + + if not isinstance(name_obj, (str, Callable)) or not isinstance(args, dict): + raise ValueError( + "Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)" + ) + return name_obj, args - -### Define factories for these layer types - + +# Define factories for these layer types + Dropout = LayerFactory() Norm = LayerFactory() Act = LayerFactory() @@ -136,19 +136,19 @@ def split_args(args): @Dropout.factory_function("dropout") -def get_dropout_type(dim): +def dropout_factory(dim): types = [nn.Dropout, nn.Dropout2d, nn.Dropout3d] return types[dim - 1] @Norm.factory_function("instance") -def get_instance_type(dim): +def instance_factory(dim): types = [nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] return types[dim - 1] @Norm.factory_function("batch") -def get_batch_type(dim): +def batch_factory(dim): types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] return types[dim - 1] @@ -159,18 +159,18 @@ def get_batch_type(dim): @Conv.factory_function("conv") -def get_conv_type(dim): +def conv_factory(dim): types = [nn.Conv1d, nn.Conv2d, nn.Conv3d] return types[dim - 1] @Conv.factory_function("convtrans") -def get_convtrans_type(dim): +def convtrans_factory(dim): types = [nn.ConvTranspose1d, nn.ConvTranspose2d, nn.ConvTranspose3d] return types[dim - 1] -### old factory functions +# old factory functions def get_conv_type(dim, is_transpose): @@ -194,5 +194,3 @@ def get_normalize_type(dim, is_instance): types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] return types[dim - 1] - - From 6fb24ed0defb868b9d4000d6e3cd0c3f6f8616d4 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Fri, 28 Feb 2020 13:52:17 +0000 Subject: [PATCH 03/11] Adding new factory concept. --- .../engine/multi_gpu_supervised_trainer.py~ | 95 +++++++++ monai/networks/blocks/convolutions.py | 57 ++++-- monai/networks/layers/factories.py | 180 +++++++++++++++++- monai/networks/nets/unet.py | 34 ++-- tests/test_unet.py | 61 +++++- 5 files changed, 382 insertions(+), 45 deletions(-) create mode 100644 monai/application/engine/multi_gpu_supervised_trainer.py~ diff --git a/monai/application/engine/multi_gpu_supervised_trainer.py~ b/monai/application/engine/multi_gpu_supervised_trainer.py~ new file mode 100644 index 00000000000..912d4cc9ad2 --- /dev/null +++ b/monai/application/engine/multi_gpu_supervised_trainer.py~ @@ -0,0 +1,95 @@ +# Copyright 2020 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch + +from ignite.engine import create_supervised_trainer, create_supervised_evaluator, _prepare_batch + +import monai + + +def _default_transform(x, y, y_pred, loss): + return loss.item() + +def _default_eval_transform(x, y, y_pred): + return y_pred, y + +@monai.utils.export("monai.application.engine") +def create_multigpu_supervised_trainer(net, optimizer, loss_fn, devices=None, non_blocking=False, + prepare_batch=_prepare_batch, output_transform=_default_transform): + """ + ***Derived from `create_supervised_trainer` in Ignite. + + Factory function for creating a trainer for supervised models. + Args: + net (`torch.nn.Module`): the network to train. + optimizer (`torch.optim.Optimizer`): the optimizer to use. + loss_fn (torch.nn loss function): the loss function to use. + devices (list, optional): device(s) type specification (default: None). + Applies to both model and batches. None is all devices used, empty list is CPU only. + non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs + tuple of tensors `(batch_x, batch_y)`. + output_transform (callable, optional): function that receives 'x', 'y', 'y_pred', 'loss' and returns value + to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`. + Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is the loss + of the processed batch by default. + Returns: + Engine: a trainer engine with supervised update function. + """ + + if devices is None: + devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] + elif len(devices) == 0: + devices = [torch.device("cpu")] + + if len(devices) > 1: + net = torch.nn.parallel.DataParallel(net) + + return create_supervised_trainer(net, optimizer, loss_fn, devices[0], non_blocking, prepare_batch, output_transform) + + +@monai.utils.export("monai.application.engine") +def create_multigpu_supervised_evaluator(net, metrics=None, device=None, non_blocking=False, + prepare_batch=_prepare_batch, output_transform=_default_eval_transform): + """ + ***Derived from `create_supervised_evaluator` in Ignite. + + Factory function for creating an evaluator for supervised models. + Args: + net (`torch.nn.Module`): the model to train. + metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics. + devices (list, optional): device(s) type specification (default: None). + Applies to both model and batches. None is all devices used, empty list is CPU only. + non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs + tuple of tensors `(batch_x, batch_y)`. + output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value + to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits + output expected by metrics. If you change it you should use `output_transform` in metrics. + Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is + a tuple of `(batch_pred, batch_y)` by default. + Returns: + Engine: an evaluator engine with supervised inference function. + """ + + if devices is None: + devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] + elif len(devices) == 0: + devices = [torch.device("cpu")] + + if len(devices) > 1: + net = torch.nn.parallel.DataParallel(net) + + return create_supervised_evaluator(net, metrics, devices[0], non_blocking, prepare_batch, output_transform) diff --git a/monai/networks/blocks/convolutions.py b/monai/networks/blocks/convolutions.py index b8b2814c673..1ca2c91ec89 100644 --- a/monai/networks/blocks/convolutions.py +++ b/monai/networks/blocks/convolutions.py @@ -12,14 +12,17 @@ import numpy as np import torch.nn as nn -from monai.networks.layers.factories import get_conv_type, get_dropout_type, get_normalize_type +from monai.networks.layers.factories import Dropout, Norm, Act, Conv, split_args from monai.networks.layers.convutils import same_padding class Convolution(nn.Sequential): - - def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, instance_norm=True, dropout=0, - dilation=1, bias=True, conv_only=False, is_transposed=False): + """ + Constructs a convolution with optional dropout, normalization, and activation layers. + """ + + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, act=Act.PRELU, + norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, conv_only=False, is_transposed=False): super().__init__() self.dimensions = dimensions self.in_channels = in_channels @@ -27,9 +30,25 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size self.is_transposed = is_transposed padding = same_padding(kernel_size, dilation) - normalize_type = get_normalize_type(dimensions, instance_norm) - conv_type = get_conv_type(dimensions, is_transposed) - drop_type = get_dropout_type(dimensions) + conv_type = Conv[Conv.CONVTRANS if is_transposed else Conv.CONV, dimensions] + + # define the normalisation type and the arguments to the constructor + norm_name, norm_args = split_args(norm) + norm_type = Norm[norm_name, dimensions] + + # define the activation type and the arguments to the constructor + act_name, act_args = split_args(act) + act_type = Act[act_name] + + if dropout: + # if dropout was specified simply as a p value, use default name and make a keyword map with the value + if isinstance(dropout, (int, float)): + drop_name = Dropout.DROPOUT + drop_args = {"p": dropout} + else: + drop_name, drop_args = split_args(dropout) + + drop_type = Dropout[drop_name, dimensions] if is_transposed: conv = conv_type(in_channels, out_channels, kernel_size, strides, padding, strides - 1, 1, bias, dilation) @@ -39,17 +58,16 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size self.add_module("conv", conv) if not conv_only: - self.add_module("norm", normalize_type(out_channels)) - if dropout > 0: # omitting Dropout2d appears faster than relying on it short-circuiting when dropout==0 - self.add_module("dropout", drop_type(dropout)) + self.add_module("norm", norm_type(out_channels, **norm_args)) + if dropout: + self.add_module("dropout", drop_type(**drop_args)) - self.add_module("prelu", nn.modules.PReLU()) + self.add_module("act", act_type(**act_args)) class ResidualUnit(nn.Module): - - def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, subunits=2, instance_norm=True, - dropout=0, dilation=1, bias=True, last_conv_only=False): + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, subunits=2, + act=Act.PRELU, norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, last_conv_only=False): super().__init__() self.dimensions = dimensions self.in_channels = in_channels @@ -64,10 +82,13 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size for su in range(subunits): conv_only = last_conv_only and su == (subunits - 1) - unit = Convolution(dimensions, schannels, out_channels, sstrides, kernel_size, instance_norm, dropout, - dilation, bias, conv_only) + unit = Convolution(dimensions, schannels, out_channels, sstrides, + kernel_size, act, norm, dropout, dilation, bias, conv_only) + self.conv.add_module("unit%i" % su, unit) - schannels = out_channels # after first loop set channels and strides to what they should be for subsequent units + + # after first loop set channels and strides to what they should be for subsequent units + schannels = out_channels sstrides = 1 # apply convolution to input to change number of output channels and size to match that coming from self.conv @@ -79,7 +100,7 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size rkernel_size = 1 rpadding = 0 - conv_type = get_conv_type(dimensions, False) + conv_type = Conv[Conv.CONV, dimensions] self.residual = conv_type(in_channels, out_channels, rkernel_size, strides, rpadding, bias=bias) def forward(self, x): diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 139de926553..e6c3e3ed152 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -8,12 +8,186 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ -handles spatial 1D, 2D, 3D network components with a factory pattern. +Defines factories for creating layers in generic, extensible, and dimensionally independent ways. A separate factory +object is created for each type of layer, and factory functions keyed to names are added to these objects. Whenever +a layer is requested the factory name and any necessary arguments are passed to the factory object. The return value +is typically a type but can be any callable producing a layer object. + +The factory objects contain functions keyed to names converted to upper case, these names can be referred to as members +of the factory so that they can function as constant identifiers. eg. instance normalisation is named `Norm.INSTANCE`. + +For example, to get a transpose convolution layer the name is needed and then a dimension argument is provided which is +passed to the factory function: + + dimension = 3 + name = Conv.CONVTRANS + conv = Conv[name, dimension] + +This allows the `dimension` value to be set in constructor for example so that the dimensionality of a network is +parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. + """ -from torch import nn as nn +from typing import Callable + +import torch +import torch.nn as nn + + +class LayerFactory: + """ + Factory object for creating layers, this uses given factory functions to actually produce the types or constructing + callables. These functions are referred to by name and can be added at any time. + """ + + def __init__(self): + self.factories = {} + + @property + def names(self): + """ + Produces all factory names. + """ + + return tuple(self.factories) + + def add_factory_callable(self, name, func): + """ + Add the factory function to this object under the given name. + """ + + self.factories[name.upper()] = func + + def factory_function(self, name): + """ + Decorator for adding a factory function with the given name. + """ + + def _add(func): + self.add_factory_callable(name, func) + return func + + return _add + + def get_constructor(self, factory_name, *args): + """ + Get the constructor for the given factory name and arguments. + """ + + if not isinstance(factory_name, str): + raise ValueError("Factories must be selected by name") + + fact = self.factories[factory_name.upper()] + return fact(*args) + + def __getitem__(self, args): + """ + Get the given name or name/arguments pair. If `args` is a callable it is assumed to be the constructor + itself and is returned, otherwise it should be the factory name or a pair containing the name and arguments. + """ + + # `args` is actually a type or constructor + if callable(args): + return args + + # `args` is a factory name or a name with arguments + if isinstance(args, str): + name_obj, args = args, () + else: + name_obj, *args = args + + return self.get_constructor(name_obj, *args) + + def __getattr__(self, key): + """ + If `key` is a factory name, return it, otherwise behave as inherited. This allows referring to factory names + as if they were constants, eg. `Fact.FOO` for a factory Fact with factory function foo. + """ + + if key in self.factories: + return key + + return super().__getattr__(key) + + +def split_args(args): + """ + Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted + """ + + if isinstance(args,str): + return args,{} + else: + name_obj,args=args + + if not isinstance(name_obj,(str,Callable)) or not isinstance(args,dict): + raise ValueError("Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)") + + return name_obj, args + + +### Define factories for these layer types + +Dropout = LayerFactory() +Norm = LayerFactory() +Act = LayerFactory() +Conv = LayerFactory() + + +@Dropout.factory_function("dropout") +def get_dropout_type(dim): + types = [nn.Dropout, nn.Dropout2d, nn.Dropout3d] + return types[dim - 1] + + +@Norm.factory_function("instance") +def get_instance_type(dim): + types = [nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] + return types[dim - 1] + + +@Norm.factory_function("batch") +def get_batch_type(dim): + types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] + return types[dim - 1] + + +Act.add_factory_callable("relu", lambda: nn.modules.ReLU) +Act.add_factory_callable("leakyrelu", lambda: nn.modules.LeakyReLU) +Act.add_factory_callable("prelu", lambda: nn.modules.PReLU) + + +@Conv.factory_function("conv") +def get_conv_type(dim): + types = [nn.Conv1d, nn.Conv2d, nn.Conv3d] + return types[dim - 1] + + +@Conv.factory_function("convtrans") +def get_convtrans_type(dim): + types = [nn.ConvTranspose1d, nn.ConvTranspose2d, nn.ConvTranspose3d] + return types[dim - 1] + + +@Conv.factory_function("maxpool") +def get_maxpooling_type(dim, is_adaptive): + if is_adaptive: + types = [nn.AdaptiveMaxPool1d, nn.AdaptiveMaxPool2d, nn.AdaptiveMaxPool3d] + else: + types = [nn.MaxPool1d, nn.MaxPool2d, nn.MaxPool3d] + return types[dim - 1] + + +@Conv.factory_function("avgpool") +def get_avgpooling_type(dim, is_adaptive): + if is_adaptive: + types = [nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d] + else: + types = [nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d] + return types[dim - 1] + +### old factory functions def get_conv_type(dim, is_transpose): diff --git a/monai/networks/nets/unet.py b/monai/networks/nets/unet.py index ad9b3ddbf4e..8aecc9b205f 100644 --- a/monai/networks/nets/unet.py +++ b/monai/networks/nets/unet.py @@ -12,6 +12,7 @@ import torch.nn as nn from monai.networks.blocks.convolutions import Convolution, ResidualUnit +from monai.networks.layers.factories import Norm, Act from monai.networks.layers.simplelayers import SkipConnection from monai.utils import export from monai.utils.aliases import alias @@ -21,8 +22,8 @@ @alias("Unet", "unet") class UNet(nn.Module): - def __init__(self, dimensions, in_channels, out_channels, channels, strides, kernel_size=3, up_kernel_size=3, - num_res_units=0, instance_norm=True, dropout=0): + def __init__(self, dimensions, in_channels, num_classes, channels, strides, kernel_size=3, up_kernel_size=3, + num_res_units=0, act=Act.PRELU, norm=Norm.INSTANCE, dropout=0): super().__init__() assert len(channels) == (len(strides) + 1) self.dimensions = dimensions @@ -33,7 +34,8 @@ def __init__(self, dimensions, in_channels, out_channels, channels, strides, ker self.kernel_size = kernel_size self.up_kernel_size = up_kernel_size self.num_res_units = num_res_units - self.instance_norm = instance_norm + self.act = act + self.norm = norm self.dropout = dropout def _create_block(inc, outc, channels, strides, is_top): @@ -62,35 +64,21 @@ def _create_block(inc, outc, channels, strides, is_top): def _get_down_layer(self, in_channels, out_channels, strides, is_top): if self.num_res_units > 0: return ResidualUnit(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.num_res_units, - self.instance_norm, self.dropout) + self.act, self.norm, self.dropout) else: - return Convolution(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.instance_norm, + return Convolution(self.dimensions, in_channels, out_channels, strides, self.kernel_size, self.act, self.norm, self.dropout) def _get_bottom_layer(self, in_channels, out_channels): return self._get_down_layer(in_channels, out_channels, 1, False) def _get_up_layer(self, in_channels, out_channels, strides, is_top): - conv = Convolution(self.dimensions, - in_channels, - out_channels, - strides, - self.up_kernel_size, - self.instance_norm, - self.dropout, - conv_only=is_top and self.num_res_units == 0, - is_transposed=True) + conv = Convolution(self.dimensions, in_channels, out_channels, strides, self.up_kernel_size, self.act, self.norm, + self.dropout, conv_only=is_top and self.num_res_units == 0, is_transposed=True) if self.num_res_units > 0: - ru = ResidualUnit(self.dimensions, - out_channels, - out_channels, - 1, - self.kernel_size, - 1, - self.instance_norm, - self.dropout, - last_conv_only=is_top) + ru = ResidualUnit(self.dimensions, out_channels, out_channels, 1, self.kernel_size, 1, self.act, self.norm, + self.dropout, last_conv_only=is_top) return nn.Sequential(conv, ru) else: return conv diff --git a/tests/test_unet.py b/tests/test_unet.py index 5b8e85f915d..b8f68fc299c 100644 --- a/tests/test_unet.py +++ b/tests/test_unet.py @@ -14,8 +14,23 @@ import torch from parameterized import parameterized +from monai.networks.layers.factories import Norm, Act from monai.networks.nets.unet import UNet + +TEST_CASE_0 = [ # single channel 2D, batch 16, no residual + { + 'dimensions': 2, + 'in_channels': 1, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 0, + }, + torch.randn(16, 1, 32, 32), + (16, 32, 32), +] + TEST_CASE_1 = [ # single channel 2D, batch 16 { 'dimensions': 2, @@ -55,10 +70,54 @@ (16, 3, 32, 64, 48), ] +TEST_CASE_4 = [ # 4-channel 3D, batch 16, batch normalisation + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'norm': Norm.BATCH, + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +TEST_CASE_5 = [ # 4-channel 3D, batch 16, LeakyReLU activation + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'act': (Act.LEAKYRELU, {'negative_slope': 0.2}), + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +TEST_CASE_6 = [ # 4-channel 3D, batch 16, LeakyReLU activation explicit + { + 'dimensions': 3, + 'in_channels': 4, + 'num_classes': 3, + 'channels': (16, 32, 64), + 'strides': (2, 2), + 'num_res_units': 1, + 'act': (torch.nn.LeakyReLU, {'negative_slope': 0.2}), + }, + torch.randn(16, 4, 32, 64, 48), + (16, 32, 64, 48), +] + +CASES = [TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6] + class TestUNET(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3]) + @parameterized.expand(CASES) def test_shape(self, input_param, input_data, expected_shape): net = UNet(**input_param) net.eval() From 2892d49207a81f8f2938984dc02f646297236b17 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Fri, 28 Feb 2020 13:59:52 +0000 Subject: [PATCH 04/11] Tweaks --- .../engine/multi_gpu_supervised_trainer.py~ | 95 ------------------- monai/networks/blocks/convolutions.py | 6 +- monai/networks/layers/factories.py | 67 +++++++------ 3 files changed, 40 insertions(+), 128 deletions(-) delete mode 100644 monai/application/engine/multi_gpu_supervised_trainer.py~ diff --git a/monai/application/engine/multi_gpu_supervised_trainer.py~ b/monai/application/engine/multi_gpu_supervised_trainer.py~ deleted file mode 100644 index 912d4cc9ad2..00000000000 --- a/monai/application/engine/multi_gpu_supervised_trainer.py~ +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2020 MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import torch - -from ignite.engine import create_supervised_trainer, create_supervised_evaluator, _prepare_batch - -import monai - - -def _default_transform(x, y, y_pred, loss): - return loss.item() - -def _default_eval_transform(x, y, y_pred): - return y_pred, y - -@monai.utils.export("monai.application.engine") -def create_multigpu_supervised_trainer(net, optimizer, loss_fn, devices=None, non_blocking=False, - prepare_batch=_prepare_batch, output_transform=_default_transform): - """ - ***Derived from `create_supervised_trainer` in Ignite. - - Factory function for creating a trainer for supervised models. - Args: - net (`torch.nn.Module`): the network to train. - optimizer (`torch.optim.Optimizer`): the optimizer to use. - loss_fn (torch.nn loss function): the loss function to use. - devices (list, optional): device(s) type specification (default: None). - Applies to both model and batches. None is all devices used, empty list is CPU only. - non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs - tuple of tensors `(batch_x, batch_y)`. - output_transform (callable, optional): function that receives 'x', 'y', 'y_pred', 'loss' and returns value - to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`. - Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is the loss - of the processed batch by default. - Returns: - Engine: a trainer engine with supervised update function. - """ - - if devices is None: - devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] - elif len(devices) == 0: - devices = [torch.device("cpu")] - - if len(devices) > 1: - net = torch.nn.parallel.DataParallel(net) - - return create_supervised_trainer(net, optimizer, loss_fn, devices[0], non_blocking, prepare_batch, output_transform) - - -@monai.utils.export("monai.application.engine") -def create_multigpu_supervised_evaluator(net, metrics=None, device=None, non_blocking=False, - prepare_batch=_prepare_batch, output_transform=_default_eval_transform): - """ - ***Derived from `create_supervised_evaluator` in Ignite. - - Factory function for creating an evaluator for supervised models. - Args: - net (`torch.nn.Module`): the model to train. - metrics (dict of str - :class:`~ignite.metrics.Metric`): a map of metric names to Metrics. - devices (list, optional): device(s) type specification (default: None). - Applies to both model and batches. None is all devices used, empty list is CPU only. - non_blocking (bool, optional): if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - prepare_batch (callable, optional): function that receives `batch`, `device`, `non_blocking` and outputs - tuple of tensors `(batch_x, batch_y)`. - output_transform (callable, optional): function that receives 'x', 'y', 'y_pred' and returns value - to be assigned to engine's state.output after each iteration. Default is returning `(y_pred, y,)` which fits - output expected by metrics. If you change it you should use `output_transform` in metrics. - Note: `engine.state.output` for this engine is defind by `output_transform` parameter and is - a tuple of `(batch_pred, batch_y)` by default. - Returns: - Engine: an evaluator engine with supervised inference function. - """ - - if devices is None: - devices = [torch.device('cuda:%i' % d) for d in range(torch.cuda.device_count())] - elif len(devices) == 0: - devices = [torch.device("cpu")] - - if len(devices) > 1: - net = torch.nn.parallel.DataParallel(net) - - return create_supervised_evaluator(net, metrics, devices[0], non_blocking, prepare_batch, output_transform) diff --git a/monai/networks/blocks/convolutions.py b/monai/networks/blocks/convolutions.py index 1ca2c91ec89..3c618636d0f 100644 --- a/monai/networks/blocks/convolutions.py +++ b/monai/networks/blocks/convolutions.py @@ -20,7 +20,7 @@ class Convolution(nn.Sequential): """ Constructs a convolution with optional dropout, normalization, and activation layers. """ - + def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size=3, act=Act.PRELU, norm=Norm.INSTANCE, dropout=None, dilation=1, bias=True, conv_only=False, is_transposed=False): super().__init__() @@ -84,9 +84,9 @@ def __init__(self, dimensions, in_channels, out_channels, strides=1, kernel_size conv_only = last_conv_only and su == (subunits - 1) unit = Convolution(dimensions, schannels, out_channels, sstrides, kernel_size, act, norm, dropout, dilation, bias, conv_only) - + self.conv.add_module("unit%i" % su, unit) - + # after first loop set channels and strides to what they should be for subsequent units schannels = out_channels sstrides = 1 diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index e6c3e3ed152..69280ae7689 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -23,15 +23,13 @@ dimension = 3 name = Conv.CONVTRANS conv = Conv[name, dimension] - + This allows the `dimension` value to be set in constructor for example so that the dimensionality of a network is parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. - """ from typing import Callable -import torch import torch.nn as nn @@ -40,7 +38,7 @@ class LayerFactory: Factory object for creating layers, this uses given factory functions to actually produce the types or constructing callables. These functions are referred to by name and can be added at any time. """ - + def __init__(self): self.factories = {} @@ -49,35 +47,35 @@ def names(self): """ Produces all factory names. """ - + return tuple(self.factories) def add_factory_callable(self, name, func): """ Add the factory function to this object under the given name. """ - + self.factories[name.upper()] = func def factory_function(self, name): """ Decorator for adding a factory function with the given name. """ - + def _add(func): self.add_factory_callable(name, func) return func return _add - + def get_constructor(self, factory_name, *args): """ Get the constructor for the given factory name and arguments. """ - + if not isinstance(factory_name, str): raise ValueError("Factories must be selected by name") - + fact = self.factories[factory_name.upper()] return fact(*args) @@ -86,11 +84,11 @@ def __getitem__(self, args): Get the given name or name/arguments pair. If `args` is a callable it is assumed to be the constructor itself and is returned, otherwise it should be the factory name or a pair containing the name and arguments. """ - + # `args` is actually a type or constructor if callable(args): return args - + # `args` is a factory name or a name with arguments if isinstance(args, str): name_obj, args = args, () @@ -104,31 +102,33 @@ def __getattr__(self, key): If `key` is a factory name, return it, otherwise behave as inherited. This allows referring to factory names as if they were constants, eg. `Fact.FOO` for a factory Fact with factory function foo. """ - + if key in self.factories: return key return super().__getattr__(key) - + def split_args(args): """ Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted """ - - if isinstance(args,str): - return args,{} + + if isinstance(args, str): + return args, {} else: - name_obj,args=args - - if not isinstance(name_obj,(str,Callable)) or not isinstance(args,dict): - raise ValueError("Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)") - + name_obj, args = args + + if not isinstance(name_obj, (str, Callable)) or not isinstance(args, dict): + raise ValueError( + "Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)" + ) + return name_obj, args - -### Define factories for these layer types - + +# Define factories for these layer types + Dropout = LayerFactory() Norm = LayerFactory() Act = LayerFactory() @@ -136,19 +136,19 @@ def split_args(args): @Dropout.factory_function("dropout") -def get_dropout_type(dim): +def dropout_factory(dim): types = [nn.Dropout, nn.Dropout2d, nn.Dropout3d] return types[dim - 1] @Norm.factory_function("instance") -def get_instance_type(dim): +def instance_factory(dim): types = [nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] return types[dim - 1] @Norm.factory_function("batch") -def get_batch_type(dim): +def batch_factory(dim): types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] return types[dim - 1] @@ -159,17 +159,18 @@ def get_batch_type(dim): @Conv.factory_function("conv") -def get_conv_type(dim): +def conv_factory(dim): types = [nn.Conv1d, nn.Conv2d, nn.Conv3d] return types[dim - 1] @Conv.factory_function("convtrans") -def get_convtrans_type(dim): +def convtrans_factory(dim): types = [nn.ConvTranspose1d, nn.ConvTranspose2d, nn.ConvTranspose3d] return types[dim - 1] +<<<<<<< 6fb24ed0defb868b9d4000d6e3cd0c3f6f8616d4 @Conv.factory_function("maxpool") def get_maxpooling_type(dim, is_adaptive): if is_adaptive: @@ -188,6 +189,9 @@ def get_avgpooling_type(dim, is_adaptive): return types[dim - 1] ### old factory functions +======= +# old factory functions +>>>>>>> Tweaks def get_conv_type(dim, is_transpose): @@ -211,6 +215,7 @@ def get_normalize_type(dim, is_instance): types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] return types[dim - 1] +<<<<<<< 6fb24ed0defb868b9d4000d6e3cd0c3f6f8616d4 def get_maxpooling_type(dim, is_adaptive): @@ -227,3 +232,5 @@ def get_avgpooling_type(dim, is_adaptive): else: types = [nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d] return types[dim - 1] +======= +>>>>>>> Tweaks From 4480e36e9935db866e636ffd0a05171c32f8afe2 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Thu, 26 Mar 2020 13:29:27 +0000 Subject: [PATCH 05/11] Factory refinements and fixes --- monai/networks/layers/factories.py | 70 +++++++++++++++++++++--------- monai/networks/nets/densenet3d.py | 38 ++++++++++------ monai/networks/nets/unet.py | 2 +- tests/test_unet.py | 16 +++---- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 9e6f24ffec5..d551a886518 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -23,9 +23,33 @@ dimension = 3 name = Conv.CONVTRANS conv = Conv[name, dimension] - -This allows the `dimension` value to be set in constructor for example so that the dimensionality of a network is + +This allows the `dimension` value to be set in the constructor, for example so that the dimensionality of a network is parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. + +Defining new factories involves creating the object then associating it with factory functions: + + fact = LayerFactory() + + @fact.factory_function('test') + def make_something(x, y): + # do something with x and y to choose which layer type to return + return SomeLayerType + ... + + layer = fact[fact.TEST, 1, 2] # request object from factory TEST with 1 and 2 as values for x and y + +Typically the caller of a factory would know what arguments to pass (ie. the dimensionality of the requested type) but +can be parameterized with the factory name and the arguments to pass to the created type at instantiation time: + + def use_factory(fact_args): + fact_name, type_args = split_args + layer_type = fact[fact_name, 1, 2] + return layer_type(**type_args) + ... + + kw_args = {'arg0':0, 'arg1':True} + layer = use_factory( (fact.TEST, kwargs) ) """ from typing import Callable @@ -85,9 +109,9 @@ def __getitem__(self, args): itself and is returned, otherwise it should be the factory name or a pair containing the name and arguments. """ - # `args` is actually a type or constructor - if callable(args): - return args + # `args[0]` is actually a type or constructor + if callable(args[0]): + return args[0] # `args` is a factory name or a name with arguments if isinstance(args, str): @@ -120,9 +144,8 @@ def split_args(args): name_obj, args = args if not isinstance(name_obj, (str, Callable)) or not isinstance(args, dict): - raise ValueError( - "Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)" - ) + msg = "Layer specifiers must be single strings or pairs of the form (name/object-types, argument dict)" + raise ValueError(msg) return name_obj, args @@ -133,6 +156,7 @@ def split_args(args): Norm = LayerFactory() Act = LayerFactory() Conv = LayerFactory() +Pool = LayerFactory() @Dropout.factory_function("dropout") @@ -170,21 +194,27 @@ def convtrans_factory(dim): return types[dim - 1] -@Conv.factory_function("maxpool") -def get_maxpooling_type(dim, is_adaptive): - if is_adaptive: - types = [nn.AdaptiveMaxPool1d, nn.AdaptiveMaxPool2d, nn.AdaptiveMaxPool3d] - else: - types = [nn.MaxPool1d, nn.MaxPool2d, nn.MaxPool3d] +@Pool.factory_function("max") +def maxpooling_factory(dim): + types = [nn.MaxPool1d, nn.MaxPool2d, nn.MaxPool3d] return types[dim - 1] -@Conv.factory_function("avgpool") -def get_avgpooling_type(dim, is_adaptive): - if is_adaptive: - types = [nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d] - else: - types = [nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d] +@Pool.factory_function("adaptivemax") +def adaptive_maxpooling_factory(dim): + types = [nn.AdaptiveMaxPool1d, nn.AdaptiveMaxPool2d, nn.AdaptiveMaxPool3d] + return types[dim - 1] + + +@Pool.factory_function("avg") +def avgpooling_factory(dim): + types = [nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d] + return types[dim - 1] + + +@Pool.factory_function("adaptiveavg") +def adaptive_avgpooling_factory(dim): + types = [nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d] return types[dim - 1] diff --git a/monai/networks/nets/densenet3d.py b/monai/networks/nets/densenet3d.py index 78fab167c48..1ba929f5d46 100644 --- a/monai/networks/nets/densenet3d.py +++ b/monai/networks/nets/densenet3d.py @@ -14,8 +14,11 @@ import torch import torch.nn as nn -from monai.networks.layers.factories import (get_avgpooling_type, get_conv_type, get_dropout_type, get_maxpooling_type, - get_normalize_type) +# from monai.networks.layers.factories import (get_avgpooling_type, get_conv_type, get_dropout_type, get_maxpooling_type, +# get_normalize_type) + + +from monai.networks.layers.factories import Conv, Dropout, Pool, Norm def densenet121(**kwargs): @@ -44,17 +47,20 @@ def __init__(self, spatial_dims, in_channels, growth_rate, bn_size, dropout_prob super(_DenseLayer, self).__init__() out_channels = bn_size * growth_rate - conv_type = get_conv_type(spatial_dims, is_transpose=False) - self.add_module('norm1', get_normalize_type(spatial_dims, is_instance=False)(in_channels)) + conv_type = Conv[Conv.CONV, spatial_dims] + norm_type = Norm[Norm.BATCH, spatial_dims] + dropout_type = Dropout[Dropout.DROPOUT, spatial_dims] + + self.add_module('norm1', norm_type(in_channels)) self.add_module('relu1', nn.ReLU(inplace=True)) self.add_module('conv1', conv_type(in_channels, out_channels, kernel_size=1, bias=False)) - self.add_module('norm2', get_normalize_type(spatial_dims, is_instance=False)(out_channels)) + self.add_module('norm2', norm_type(out_channels)) self.add_module('relu2', nn.ReLU(inplace=True)) self.add_module('conv2', conv_type(out_channels, growth_rate, kernel_size=3, padding=1, bias=False)) if dropout_prob > 0: - self.add_module('dropout', get_dropout_type(spatial_dims)(dropout_prob)) + self.add_module('dropout', dropout_type(dropout_prob)) def forward(self, x): new_features = super(_DenseLayer, self).forward(x) @@ -75,12 +81,15 @@ class _Transition(nn.Sequential): def __init__(self, spatial_dims, in_channels, out_channels): super(_Transition, self).__init__() - conv_type = get_conv_type(spatial_dims, is_transpose=False) + + conv_type = Conv[Conv.CONV, spatial_dims] + norm_type = Norm[Norm.BATCH, spatial_dims] + pool_type = Pool[Pool.AVG, spatial_dims] - self.add_module('norm', get_normalize_type(spatial_dims, is_instance=False)(in_channels)) + self.add_module('norm', norm_type(in_channels)) self.add_module('relu', nn.ReLU(inplace=True)) self.add_module('conv', conv_type(in_channels, out_channels, kernel_size=1, bias=False)) - self.add_module('pool', get_avgpooling_type(spatial_dims, is_adaptive=False)(kernel_size=2, stride=2)) + self.add_module('pool', pool_type(kernel_size=2, stride=2)) class DenseNet(nn.Module): @@ -112,15 +121,18 @@ def __init__(self, dropout_prob=0): super(DenseNet, self).__init__() - conv_type = get_conv_type(spatial_dims, is_transpose=False) - norm_type = get_normalize_type(spatial_dims, is_instance=False) + + conv_type = Conv[Conv.CONV, spatial_dims] + norm_type = Norm[Norm.BATCH, spatial_dims] + pool_type = Pool[Pool.MAX, spatial_dims] + avg_pool_type = Pool[Pool.ADAPTIVEAVG, spatial_dims] self.features = nn.Sequential( OrderedDict([ ('conv0', conv_type(in_channels, init_features, kernel_size=7, stride=2, padding=3, bias=False)), ('norm0', norm_type(init_features)), ('relu0', nn.ReLU(inplace=True)), - ('pool0', get_maxpooling_type(spatial_dims, is_adaptive=False)(kernel_size=3, stride=2, padding=1)), + ('pool0', pool_type(kernel_size=3, stride=2, padding=1)), ])) in_channels = init_features @@ -145,7 +157,7 @@ def __init__(self, self.class_layers = nn.Sequential( OrderedDict([ ('relu', nn.ReLU(inplace=True)), - ('norm', get_avgpooling_type(spatial_dims, is_adaptive=True)(1)), + ('norm', avg_pool_type(1)), ('flatten', nn.Flatten(1)), ('class', nn.Linear(in_channels, out_channels)), ])) diff --git a/monai/networks/nets/unet.py b/monai/networks/nets/unet.py index 8aecc9b205f..6789f958f51 100644 --- a/monai/networks/nets/unet.py +++ b/monai/networks/nets/unet.py @@ -22,7 +22,7 @@ @alias("Unet", "unet") class UNet(nn.Module): - def __init__(self, dimensions, in_channels, num_classes, channels, strides, kernel_size=3, up_kernel_size=3, + def __init__(self, dimensions, in_channels, out_channels, channels, strides, kernel_size=3, up_kernel_size=3, num_res_units=0, act=Act.PRELU, norm=Norm.INSTANCE, dropout=0): super().__init__() assert len(channels) == (len(strides) + 1) diff --git a/tests/test_unet.py b/tests/test_unet.py index b8f68fc299c..be661a4dbb8 100644 --- a/tests/test_unet.py +++ b/tests/test_unet.py @@ -22,13 +22,13 @@ { 'dimensions': 2, 'in_channels': 1, - 'num_classes': 3, + 'out_channels': 3, 'channels': (16, 32, 64), 'strides': (2, 2), 'num_res_units': 0, }, torch.randn(16, 1, 32, 32), - (16, 32, 32), + (16, 3, 32, 32), ] TEST_CASE_1 = [ # single channel 2D, batch 16 @@ -74,42 +74,42 @@ { 'dimensions': 3, 'in_channels': 4, - 'num_classes': 3, + 'out_channels': 3, 'channels': (16, 32, 64), 'strides': (2, 2), 'num_res_units': 1, 'norm': Norm.BATCH, }, torch.randn(16, 4, 32, 64, 48), - (16, 32, 64, 48), + (16, 3, 32, 64, 48), ] TEST_CASE_5 = [ # 4-channel 3D, batch 16, LeakyReLU activation { 'dimensions': 3, 'in_channels': 4, - 'num_classes': 3, + 'out_channels': 3, 'channels': (16, 32, 64), 'strides': (2, 2), 'num_res_units': 1, 'act': (Act.LEAKYRELU, {'negative_slope': 0.2}), }, torch.randn(16, 4, 32, 64, 48), - (16, 32, 64, 48), + (16, 3, 32, 64, 48), ] TEST_CASE_6 = [ # 4-channel 3D, batch 16, LeakyReLU activation explicit { 'dimensions': 3, 'in_channels': 4, - 'num_classes': 3, + 'out_channels': 3, 'channels': (16, 32, 64), 'strides': (2, 2), 'num_res_units': 1, 'act': (torch.nn.LeakyReLU, {'negative_slope': 0.2}), }, torch.randn(16, 4, 32, 64, 48), - (16, 32, 64, 48), + (16, 3, 32, 64, 48), ] CASES = [TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6] From 508744f67aa79aa7955c87f81978953e3ffc8898 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Thu, 26 Mar 2020 13:34:59 +0000 Subject: [PATCH 06/11] Updates --- monai/networks/nets/densenet3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/networks/nets/densenet3d.py b/monai/networks/nets/densenet3d.py index 1ba929f5d46..7d687e04f9a 100644 --- a/monai/networks/nets/densenet3d.py +++ b/monai/networks/nets/densenet3d.py @@ -121,7 +121,7 @@ def __init__(self, dropout_prob=0): super(DenseNet, self).__init__() - + conv_type = Conv[Conv.CONV, spatial_dims] norm_type = Norm[Norm.BATCH, spatial_dims] pool_type = Pool[Pool.MAX, spatial_dims] From d7a8737ba56feaf8a367b00453028794c77fe11a Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Thu, 26 Mar 2020 13:36:22 +0000 Subject: [PATCH 07/11] Updates --- monai/networks/layers/factories.py | 8 ++++---- monai/networks/nets/densenet3d.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index d551a886518..8ee7a1b92d0 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -23,22 +23,22 @@ dimension = 3 name = Conv.CONVTRANS conv = Conv[name, dimension] - + This allows the `dimension` value to be set in the constructor, for example so that the dimensionality of a network is parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. Defining new factories involves creating the object then associating it with factory functions: fact = LayerFactory() - + @fact.factory_function('test') def make_something(x, y): # do something with x and y to choose which layer type to return return SomeLayerType ... - + layer = fact[fact.TEST, 1, 2] # request object from factory TEST with 1 and 2 as values for x and y - + Typically the caller of a factory would know what arguments to pass (ie. the dimensionality of the requested type) but can be parameterized with the factory name and the arguments to pass to the created type at instantiation time: diff --git a/monai/networks/nets/densenet3d.py b/monai/networks/nets/densenet3d.py index 7d687e04f9a..687fa2f95a9 100644 --- a/monai/networks/nets/densenet3d.py +++ b/monai/networks/nets/densenet3d.py @@ -50,7 +50,7 @@ def __init__(self, spatial_dims, in_channels, growth_rate, bn_size, dropout_prob conv_type = Conv[Conv.CONV, spatial_dims] norm_type = Norm[Norm.BATCH, spatial_dims] dropout_type = Dropout[Dropout.DROPOUT, spatial_dims] - + self.add_module('norm1', norm_type(in_channels)) self.add_module('relu1', nn.ReLU(inplace=True)) self.add_module('conv1', conv_type(in_channels, out_channels, kernel_size=1, bias=False)) @@ -81,7 +81,7 @@ class _Transition(nn.Sequential): def __init__(self, spatial_dims, in_channels, out_channels): super(_Transition, self).__init__() - + conv_type = Conv[Conv.CONV, spatial_dims] norm_type = Norm[Norm.BATCH, spatial_dims] pool_type = Pool[Pool.AVG, spatial_dims] From a4d15a33d083e25bf1f81d28237191d5a93b7fa3 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Thu, 26 Mar 2020 13:44:35 +0000 Subject: [PATCH 08/11] Change to factory --- monai/networks/layers/factories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 8ee7a1b92d0..5d607b99db9 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -110,8 +110,8 @@ def __getitem__(self, args): """ # `args[0]` is actually a type or constructor - if callable(args[0]): - return args[0] + if callable(args): + return args # `args` is a factory name or a name with arguments if isinstance(args, str): From cb4429835f0172ef99d5b16b544974a0265a3e24 Mon Sep 17 00:00:00 2001 From: Eric Kerfoot Date: Thu, 26 Mar 2020 13:56:05 +0000 Subject: [PATCH 09/11] Update to HighResNet --- monai/networks/layers/factories.py | 42 ------------------------------ monai/networks/nets/densenet3d.py | 4 --- monai/networks/nets/highresnet.py | 12 ++++----- 3 files changed, 6 insertions(+), 52 deletions(-) diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 5d607b99db9..0f49abc8d63 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -216,45 +216,3 @@ def avgpooling_factory(dim): def adaptive_avgpooling_factory(dim): types = [nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d] return types[dim - 1] - - -# old factory functions - - -def get_conv_type(dim, is_transpose): - if is_transpose: - types = [nn.ConvTranspose1d, nn.ConvTranspose2d, nn.ConvTranspose3d] - else: - types = [nn.Conv1d, nn.Conv2d, nn.Conv3d] - - return types[dim - 1] - - -def get_dropout_type(dim): - types = [nn.Dropout, nn.Dropout2d, nn.Dropout3d] - return types[dim - 1] - - -def get_normalize_type(dim, is_instance): - if is_instance: - types = [nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] - else: - types = [nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d] - - return types[dim - 1] - - -def get_maxpooling_type(dim, is_adaptive): - if is_adaptive: - types = [nn.AdaptiveMaxPool1d, nn.AdaptiveMaxPool2d, nn.AdaptiveMaxPool3d] - else: - types = [nn.MaxPool1d, nn.MaxPool2d, nn.MaxPool3d] - return types[dim - 1] - - -def get_avgpooling_type(dim, is_adaptive): - if is_adaptive: - types = [nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d] - else: - types = [nn.AvgPool1d, nn.AvgPool2d, nn.AvgPool3d] - return types[dim - 1] diff --git a/monai/networks/nets/densenet3d.py b/monai/networks/nets/densenet3d.py index 687fa2f95a9..cbc90d209bc 100644 --- a/monai/networks/nets/densenet3d.py +++ b/monai/networks/nets/densenet3d.py @@ -14,10 +14,6 @@ import torch import torch.nn as nn -# from monai.networks.layers.factories import (get_avgpooling_type, get_conv_type, get_dropout_type, get_maxpooling_type, -# get_normalize_type) - - from monai.networks.layers.factories import Conv, Dropout, Pool, Norm diff --git a/monai/networks/nets/highresnet.py b/monai/networks/nets/highresnet.py index 4486f5b5851..711ab815af1 100644 --- a/monai/networks/nets/highresnet.py +++ b/monai/networks/nets/highresnet.py @@ -13,11 +13,11 @@ import torch.nn.functional as F from monai.networks.layers.convutils import same_padding -from monai.networks.layers.factories import (get_conv_type, get_dropout_type, get_normalize_type) +from monai.networks.layers.factories import Conv, Dropout, Norm SUPPORTED_NORM = { - 'batch': lambda spatial_dims: get_normalize_type(spatial_dims, is_instance=False), - 'instance': lambda spatial_dims: get_normalize_type(spatial_dims, is_instance=True), + 'batch': lambda spatial_dims: Norm[Norm.BATCH, spatial_dims], + 'instance': lambda spatial_dims: Norm[Norm.INSTANCE, spatial_dims], } SUPPORTED_ACTI = {'relu': nn.ReLU, 'prelu': nn.PReLU, 'relu6': nn.ReLU6} DEFAULT_LAYER_PARAMS_3D = ( @@ -48,7 +48,7 @@ def __init__(self, layers = nn.ModuleList() - conv_type = get_conv_type(spatial_dims, is_transpose=False) + conv_type = Conv[Conv.CONV, spatial_dims] padding_size = same_padding(kernel_size) conv = conv_type(in_channels, out_channels, kernel_size, padding=padding_size) layers.append(conv) @@ -58,7 +58,7 @@ def __init__(self, if acti_type is not None: layers.append(SUPPORTED_ACTI[acti_type](inplace=True)) if dropout_prob is not None: - dropout_type = get_dropout_type(spatial_dims) + dropout_type = Dropout[Dropout.DROPOUT, spatial_dims] layers.append(dropout_type(p=dropout_prob)) self.layers = nn.Sequential(*layers) @@ -84,7 +84,7 @@ def __init__(self, with either zero padding ('pad') or a trainable conv with kernel size 1 ('project'). """ super(HighResBlock, self).__init__() - conv_type = get_conv_type(spatial_dims, is_transpose=False) + conv_type = Conv[Conv.CONV, spatial_dims] self.project, self.pad = None, None if in_channels != out_channels: From 894beacf262c50a07bca9c86d1aea4f63acd4f5c Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Fri, 27 Mar 2020 21:20:10 +0800 Subject: [PATCH 10/11] [DLMED] add introduction of layer factory to highlights (#217) --- docs/source/highlights.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/highlights.md b/docs/source/highlights.md index f7895bf65d5..113fc1a1c5c 100644 --- a/docs/source/highlights.md +++ b/docs/source/highlights.md @@ -72,7 +72,16 @@ torch.backends.cudnn.benchmark = False There are domain-specific loss functions in the medical research area which are different from the generic computer vision ones. As an important module of MONAI, these loss functions are implemented in PyTorch, such as Dice loss and generalized Dice loss. ## Network architectures -Some deep neural network architectures have shown to be particularly effective for medical imaging analysis tasks. MONAI implements reference networks with the aims of both flexibility and code readability. +Some deep neural network architectures have shown to be particularly effective for medical imaging analysis tasks. MONAI implements reference networks with the aims of both flexibility and code readability. +In order to leverage the common network layers and blocks, MONAI provides several predefined layers and blocks which are compatible with 1D, 2D and 3D networks. Users can easily integrate the layer factories in their own networks. +For example: +```py +# add Convolution layer to the network which is compatible with different spatial dimensions. +dimension = 3 +name = Conv.CONVTRANS +conv_type = Conv[name, dimension] +add_module('conv1', conv_type(in_channels, out_channels, kernel_size=1, bias=False)) +``` ## Evaluation To run model inferences and evaluate the model quality, MONAI provides reference implementations for the relevant widely-used approaches. Currently, several popular evaluation metrics and inference patterns are included: From ab9d86c3cff7ce715325c4ee69c01b924a05b4e6 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 27 Mar 2020 14:58:49 +0000 Subject: [PATCH 11/11] adds factories doc to docs/ (#218) --- docs/source/networks.rst | 28 ++++++++-------------------- monai/networks/layers/factories.py | 22 +++++++++++++++------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/docs/source/networks.rst b/docs/source/networks.rst index 5d456c1528d..8b05382ba92 100644 --- a/docs/source/networks.rst +++ b/docs/source/networks.rst @@ -26,26 +26,14 @@ Blocks Layers ------ -`get_conv_type` -~~~~~~~~~~~~~~~ -.. automethod:: monai.networks.layers.factories.get_conv_type - -`get_dropout_type` -~~~~~~~~~~~~~~~~~~ -.. automethod:: monai.networks.layers.factories.get_dropout_type - -`get_normalize_type` -~~~~~~~~~~~~~~~~~~~~ -.. automethod:: monai.networks.layers.factories.get_normalize_type - -`get_maxpooling_type` -~~~~~~~~~~~~~~~~~~~~~ -.. automethod:: monai.networks.layers.factories.get_maxpooling_type - -`get_avgpooling_type` -~~~~~~~~~~~~~~~~~~~~~ -.. automethod:: monai.networks.layers.factories.get_avgpooling_type - +`Factories` +~~~~~~~~~~~ +.. automodule:: monai.networks.layers.factories +.. currentmodule:: monai.networks.layers.factories + +`LayerFactory` +############## +.. autoclass:: LayerFactory .. automodule:: monai.networks.layers.simplelayers .. currentmodule:: monai.networks.layers.simplelayers diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 0f49abc8d63..05213ab68bc 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -8,45 +8,53 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """ Defines factories for creating layers in generic, extensible, and dimensionally independent ways. A separate factory object is created for each type of layer, and factory functions keyed to names are added to these objects. Whenever a layer is requested the factory name and any necessary arguments are passed to the factory object. The return value -is typically a type but can be any callable producing a layer object. +is typically a type but can be any callable producing a layer object. The factory objects contain functions keyed to names converted to upper case, these names can be referred to as members of the factory so that they can function as constant identifiers. eg. instance normalisation is named `Norm.INSTANCE`. -For example, to get a transpose convolution layer the name is needed and then a dimension argument is provided which is +For example, to get a transpose convolution layer the name is needed and then a dimension argument is provided which is passed to the factory function: +.. code-block:: python + dimension = 3 name = Conv.CONVTRANS conv = Conv[name, dimension] -This allows the `dimension` value to be set in the constructor, for example so that the dimensionality of a network is +This allows the `dimension` value to be set in the constructor, for example so that the dimensionality of a network is parameterizable. Not all factories require arguments after the name, the caller must be aware which are required. Defining new factories involves creating the object then associating it with factory functions: +.. code-block:: python + fact = LayerFactory() @fact.factory_function('test') def make_something(x, y): # do something with x and y to choose which layer type to return return SomeLayerType - ... + ... - layer = fact[fact.TEST, 1, 2] # request object from factory TEST with 1 and 2 as values for x and y + # request object from factory TEST with 1 and 2 as values for x and y + layer = fact[fact.TEST, 1, 2] Typically the caller of a factory would know what arguments to pass (ie. the dimensionality of the requested type) but can be parameterized with the factory name and the arguments to pass to the created type at instantiation time: +.. code-block:: python + def use_factory(fact_args): fact_name, type_args = split_args layer_type = fact[fact_name, 1, 2] return layer_type(**type_args) - ... + ... kw_args = {'arg0':0, 'arg1':True} layer = use_factory( (fact.TEST, kwargs) ) @@ -135,7 +143,7 @@ def __getattr__(self, key): def split_args(args): """ - Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted + Split arguments in a way to be suitable for using with the factory types. If `args` is a name it's interpreted """ if isinstance(args, str):