PyTorch Lightning 1.1: research : CIFAR100 (ResNet with Stochastic Depth)
作成 : (株)クラスキャット セールスインフォメーション
作成日時 : 02/25/2021 (1.1.x)

* 本ページは以下の CIFAR10 用リソースを参考に CIFAR100 で遂行した実験結果のレポートです：

• notebooks : PyTorch Lightning CIFAR10 ~94% Baseline Tutorial
• Train CIFAR10 with PyTorch
• Pytorch-cifar100

* ご自由にリンクを張って頂いてかまいませんが、sales-info@classcat.com までご一報いただけると嬉しいです。

research: CIFAR100 (ResNet with Stochastic Depth)

仕様

• Total params: (11.2M)
• Trainable params:
• Non-trainable params: 0

 
結果

• stochastic_depth_resnet18
• {‘test_acc’: 0.7181000113487244, ‘test_loss’: 1.0404026508331299}
• 100 エポック ; Wall time: 1h 30min 58s
• Tesla T4
• ReduceLROnPlateau

CIFAR 100 DM

from typing import Any, Callable, Optional, Sequence, Union
 
from pl_bolts.datamodules.vision_datamodule import VisionDataModule
#from pl_bolts.datasets import TrialCIFAR10
#from pl_bolts.transforms.dataset_normalizations import cifar10_normalization
from pl_bolts.utils import _TORCHVISION_AVAILABLE
from pl_bolts.utils.warnings import warn_missing_pkg
 
if _TORCHVISION_AVAILABLE:
    from torchvision import transforms
    #from torchvision import transforms as transform_lib
    from torchvision.datasets import CIFAR100
else:  # pragma: no cover
    warn_missing_pkg('torchvision')
    CIFAR100 = None

def cifar100_normalization():
    if not _TORCHVISION_AVAILABLE:  # pragma: no cover
        raise ModuleNotFoundError(
            'You want to use `torchvision` which is not installed yet, install it with `pip install torchvision`.'
        )

    normalize = transforms.Normalize(
        mean=[x / 255.0 for x in [129.3, 124.1, 112.4]],
        std=[x / 255.0 for x in [68.2, 65.4, 70.4]],
        # cifar10
        #mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
        #std=[x / 255.0 for x in [63.0, 62.1, 66.7]],
    )
    return normalize

class CIFAR100DataModule(VisionDataModule):
    """
    .. figure:: https://3qeqpr26caki16dnhd19sv6by6v-wpengine.netdna-ssl.com/wp-content/uploads/2019/01/
        Plot-of-a-Subset-of-Images-from-the-CIFAR-10-Dataset.png
        :width: 400
        :alt: CIFAR-10
    Specs:
        - 10 classes (1 per class)
        - Each image is (3 x 32 x 32)
    Standard CIFAR10, train, val, test splits and transforms
    Transforms::
        mnist_transforms = transform_lib.Compose([
            transform_lib.ToTensor(),
            transforms.Normalize(
                mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
                std=[x / 255.0 for x in [63.0, 62.1, 66.7]]
            )
        ])
    Example::
        from pl_bolts.datamodules import CIFAR10DataModule
        dm = CIFAR10DataModule(PATH)
        model = LitModel()
        Trainer().fit(model, datamodule=dm)
    Or you can set your own transforms
    Example::
        dm.train_transforms = ...
        dm.test_transforms = ...
        dm.val_transforms  = ...
    """
    name = "cifar100"
    dataset_cls = CIFAR100
    dims = (3, 32, 32)

    def __init__(
        self,
        data_dir: Optional[str] = None,
        val_split: Union[int, float] = 0.2,
        num_workers: int = 16,
        normalize: bool = False,
        batch_size: int = 32,
        seed: int = 42,
        shuffle: bool = False,
        pin_memory: bool = False,
        drop_last: bool = False,
        *args: Any,
        **kwargs: Any,
    ) -> None:
        """
        Args:
            data_dir: Where to save/load the data
            val_split: Percent (float) or number (int) of samples to use for the validation split
            num_workers: How many workers to use for loading data
            normalize: If true applies image normalize
            batch_size: How many samples per batch to load
            seed: Random seed to be used for train/val/test splits
            shuffle: If true shuffles the train data every epoch
            pin_memory: If true, the data loader will copy Tensors into CUDA pinned memory before
                        returning them
            drop_last: If true drops the last incomplete batch
        """
        super().__init__(  # type: ignore[misc]
            data_dir=data_dir,
            val_split=val_split,
            num_workers=num_workers,
            normalize=normalize,
            batch_size=batch_size,
            seed=seed,
            shuffle=shuffle,
            pin_memory=pin_memory,
            drop_last=drop_last,
            *args,
            **kwargs,
        )

    @property
    def num_samples(self) -> int:
        train_len, _ = self._get_splits(len_dataset=50_000)
        return train_len

    @property
    def num_classes(self) -> int:
        """
        Return:
            10
        """
        return 100

    def default_transforms(self) -> Callable:
        if self.normalize:
            cf100_transforms = transforms.Compose([transform_lib.ToTensor(), cifar100_normalization()])
        else:
            cf100_transforms = transforms.Compose([transform_lib.ToTensor()])

        return cf100_transforms

モデル

import torch
import torch.nn as nn
from torch.distributions.bernoulli import Bernoulli
import random

class StochasticDepthBasicBlock(torch.jit.ScriptModule):

    expansion=1

    def __init__(self, p, in_channels, out_channels, stride=1):
        super().__init__()

        #self.p = torch.tensor(p).float()
        self.p = p
        self.residual = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels * StochasticDepthBasicBlock.expansion, kernel_size=3, padding=1),
            nn.BatchNorm2d(out_channels)
        )

        self.shortcut = nn.Sequential()

        if stride != 1 or in_channels != out_channels * StochasticDepthBasicBlock.expansion:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels * StochasticDepthBasicBlock.expansion, kernel_size=1, stride=stride),
                nn.BatchNorm2d(out_channels)
            )
    def survival(self):
        var = torch.bernoulli(torch.tensor(self.p).float())
        return torch.equal(var, torch.tensor(1).float().to(var.device))

    @torch.jit.script_method
    def forward(self, x):

        if self.training:
            if self.survival():
                # official torch implementation
                # function ResidualDrop:updateOutput(input)
                #    local skip_forward = self.skip:forward(input)
                #    self.output:resizeAs(skip_forward):copy(skip_forward)
                #    if self.train then
                #        if self.gate then -- only compute convolutional output when gate is open
                #            self.output:add(self.net:forward(input))
                #        end
                #    else
                #            self.output:add(self.net:forward(input):mul(1-self.deathRate))
                #        end
                #    return self.output
                # end

                # paper:
                # Hl = ReLU(bl*fl(Hl−1) + id(Hl−1)).

                # paper and their official implementation are different
                # paper use relu after output
                # official implementation dosen't
                #
                # other implementions which use relu:
                # https://github.com/jiweeo/pytorch-stochastic-depth/blob/a6f95aaffee82d273c1cd73d9ed6ef0718c6683d/models/resnet.py
                # https://github.com/dblN/stochastic_depth_keras/blob/master/train.py

                # implementations which doesn't use relu:
                # https://github.com/transcranial/stochastic-depth/blob/master/stochastic-depth.ipynb
                # https://github.com/shamangary/Pytorch-Stochastic-Depth-Resnet/blob/master/TYY_stodepth_lineardecay.py

                # I will just stick with the official implementation, I think
                # whether add relu after residual won't effect the network
                # performance too much
                x = self.residual(x) + self.shortcut(x)
            else:
                # If bl = 0, the ResBlock reduces to the identity function
                x = self.shortcut(x)

        else:
            x = self.residual(x) * self.p + self.shortcut(x)

        return x


class StochasticDepthBottleNeck(torch.jit.ScriptModule):
    """Residual block for resnet over 50 layers
    """
    expansion = 4
    def __init__(self, p, in_channels, out_channels, stride=1):
        super().__init__()

        self.p = p
        self.residual = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels * StochasticDepthBottleNeck.expansion, kernel_size=1, bias=False),
            nn.BatchNorm2d(out_channels * StochasticDepthBottleNeck.expansion),
        )

        self.shortcut = nn.Sequential()

        if stride != 1 or in_channels != out_channels * StochasticDepthBottleNeck.expansion:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels * StochasticDepthBottleNeck.expansion, stride=stride, kernel_size=1, bias=False),
                nn.BatchNorm2d(out_channels * StochasticDepthBottleNeck.expansion)
            )

    def survival(self):
        var = torch.bernoulli(torch.tensor(self.p).float())
        return torch.equal(var, torch.tensor(1).float().to(var.device))

    @torch.jit.script_method
    def forward(self, x):

        if self.training:
            if self.survival():
                x = self.residual(x) + self.shortcut(x)
            else:
                x = self.shortcut(x)
        else:
            x = self.residual(x) * self.p + self.shortcut(x)

        return x

class StochasticDepthResNet(nn.Module):

    def __init__(self, block, num_block, num_classes=100):
        super().__init__()

        self.in_channels = 64
        self.conv1 = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True)
        )

        self.step = (1 - 0.5) / (sum(num_block) - 1)
        self.pl = 1
        self.conv2_x = self._make_layer(block, 64, num_block[0], 1)
        self.conv3_x = self._make_layer(block, 128, num_block[1], 2)
        self.conv4_x = self._make_layer(block, 256, num_block[2], 2)
        self.conv5_x = self._make_layer(block, 512, num_block[3], 2)
        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * block.expansion, num_classes)

    def _make_layer(self, block, out_channels, num_blocks, stride):

        strides = [stride] + [1] * (num_blocks - 1)
        layers = []
        for stride in strides:
            layers.append(block(self.pl, self.in_channels, out_channels, stride))
            self.in_channels = out_channels * block.expansion
            self.pl -= self.step

        return nn.Sequential(*layers)

    def forward(self, x):
        output = self.conv1(x)
        output = self.conv2_x(output)
        output = self.conv3_x(output)
        output = self.conv4_x(output)
        output = self.conv5_x(output)
        output = self.avg_pool(output)
        output = output.view(output.size(0), -1)
        output = self.fc(output)

        return output


def stochastic_depth_resnet18():
    """ return a ResNet 18 object
    """
    return StochasticDepthResNet(StochasticDepthBasicBlock, [2, 2, 2, 2])

def stochastic_depth_resnet34():
    """ return a ResNet 34 object
    """
    return StochasticDepthResNet(StochasticDepthBasicBlock, [3, 4, 6, 3])

def stochastic_depth_resnet50():

    """ return a ResNet 50 object
    """
    return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 4, 6, 3])

def stochastic_depth_resnet101():
    """ return a ResNet 101 object
    """
    return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 4, 23, 3])

def stochastic_depth_resnet152():
    """ return a ResNet 152 object
    """
    return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 8, 36, 3])

net = stochastic_depth_resnet18()
print(net)
y = net(torch.randn(1, 3, 32, 32))
print(y.size())

StochasticDepthResNet(
  (conv1): Sequential(
    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
  )
  (conv2_x): Sequential(
    (0): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(original_name=Sequential)
    )
    (1): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(original_name=Sequential)
    )
  )
  (conv3_x): Sequential(
    (0): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
      )
    )
    (1): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(original_name=Sequential)
    )
  )
  (conv4_x): Sequential(
    (0): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
      )
    )
    (1): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(original_name=Sequential)
    )
  )
  (conv5_x): Sequential(
    (0): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
      )
    )
    (1): StochasticDepthBasicBlock(
      (residual): RecursiveScriptModule(
        original_name=Sequential
        (0): RecursiveScriptModule(original_name=Conv2d)
        (1): RecursiveScriptModule(original_name=BatchNorm2d)
        (2): RecursiveScriptModule(original_name=ReLU)
        (3): RecursiveScriptModule(original_name=Conv2d)
        (4): RecursiveScriptModule(original_name=BatchNorm2d)
      )
      (shortcut): RecursiveScriptModule(original_name=Sequential)
    )
  )
  (avg_pool): AdaptiveAvgPool2d(output_size=(1, 1))
  (fc): Linear(in_features=512, out_features=100, bias=True)
)
torch.Size([1, 100])

from torchsummary import summary
 
summary(stochastic_depth_resnet18().to('cuda'), (3, 32, 32))

RuntimeError: register_forward_hook is not supported on ScriptModules

Lightning モジュール

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim.lr_scheduler import OneCycleLR, CyclicLR, ExponentialLR, CosineAnnealingLR, ReduceLROnPlateau
from torch.optim.swa_utils import AveragedModel, update_bn
import torchvision
 
import pytorch_lightning as pl
from pytorch_lightning.callbacks import LearningRateMonitor, GPUStatsMonitor, EarlyStopping
from pytorch_lightning.metrics.functional import accuracy
#from pl_bolts.datamodules import CIFAR10DataModule
#from pl_bolts.transforms.dataset_normalizations import cifar10_normalization

pl.seed_everything(7);

batch_size = 50
 
train_transforms = torchvision.transforms.Compose([
    torchvision.transforms.RandomCrop(32, padding=4),
    torchvision.transforms.RandomHorizontalFlip(),
    torchvision.transforms.ToTensor(),
    cifar100_normalization(),
])
 
test_transforms = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor(),
    cifar100_normalization(),
])
 
cifar100_dm = CIFAR100DataModule(
    batch_size=batch_size,
    num_workers=8,
    train_transforms=train_transforms,
    test_transforms=test_transforms,
    val_transforms=test_transforms,
)

class LitCifar100(pl.LightningModule):
    def __init__(self, lr=0.05, factor=0.8):
        super().__init__()
  
        self.save_hyperparameters()
        self.model = stochastic_depth_resnet18()
 
    def forward(self, x):
        out = self.model(x)
        return F.log_softmax(out, dim=1)
  
    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = F.log_softmax(self.model(x), dim=1)
        loss = F.nll_loss(logits, y)
        self.log('train_loss', loss)
        return loss
  
    def evaluate(self, batch, stage=None):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)
  
        if stage:
            self.log(f'{stage}_loss', loss, prog_bar=True)
            self.log(f'{stage}_acc', acc, prog_bar=True)
  
    def validation_step(self, batch, batch_idx):
        self.evaluate(batch, 'val')
  
    def test_step(self, batch, batch_idx):
        self.evaluate(batch, 'test')
  
    def configure_optimizers(self):
        optimizer = torch.optim.SGD(self.parameters(), lr=self.hparams.lr, momentum=0.9, weight_decay=5e-4)
 
        return {
          'optimizer': optimizer,
          'lr_scheduler': ReduceLROnPlateau(optimizer, 'max', patience=5, factor=self.hparams.factor, verbose=True, threshold=0.0001, threshold_mode='abs', cooldown=1, min_lr=1e-5),
          'monitor': 'val_acc'
        }

訓練 / 評価

%%time
 
model = LitCifar100(lr=0.05, factor=0.5)
model.datamodule = cifar100_dm
  
trainer = pl.Trainer(
    gpus=1,
    max_epochs=100,
    progress_bar_refresh_rate=100,
    logger=pl.loggers.TensorBoardLogger('tblogs/', name='stochastic_depth_resnet18'),
    callbacks=[LearningRateMonitor(logging_interval='step')],
)
  
trainer.fit(model, cifar100_dm)
trainer.test(model, datamodule=cifar100_dm);

  | Name  | Type                  | Params
------------------------------------------------
0 | model | StochasticDepthResNet | 11.2 M
------------------------------------------------
11.2 M    Trainable params
0         Non-trainable params
11.2 M    Total params
(...)
Epoch    34: reducing learning rate of group 0 to 2.5000e-02.
Epoch    43: reducing learning rate of group 0 to 1.2500e-02.
Epoch    51: reducing learning rate of group 0 to 6.2500e-03.
Epoch    61: reducing learning rate of group 0 to 3.1250e-03.
Epoch    77: reducing learning rate of group 0 to 1.5625e-03.
Epoch    85: reducing learning rate of group 0 to 7.8125e-04.
Epoch    97: reducing learning rate of group 0 to 3.9063e-04.
(...)
--------------------------------------------------------------------------------
DATALOADER:0 TEST RESULTS
{'test_acc': 0.7181000113487244, 'test_loss': 1.0404026508331299}
--------------------------------------------------------------------------------
CPU times: user 1h 2min 17s, sys: 25min 52s, total: 1h 28min 9s
Wall time: 1h 30min 58s

以上