4.6 Dropout 的代码实现

定义dropout_layer函数

1
import torch
2
from torch import nn
3
from d2l import torch as d2l
4

5

6
def dropout_layer(X, dropout):
7
    assert 0 <= dropout <= 1
8
    #丢弃所有元素
9
    if dropout == 1:
10
        return torch.zeros_like(X)
11
    #保留所有元素
12
    if dropout == 0:
13
        return X
14
    mask = (torch.rand(X.shape) > dropout).float()    # 注意是.rand()不是.randn()
15
    return mask * X / (1.0 - dropout)

定义模型
我们使用Fashion-MNIST数据集，定义具有两个隐藏层的多层感知机，每个隐藏层包含256个单元。

1
num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256
2

3
dropout1, dropout2 = 0.2, 0.5
4

5
class Net(nn.Module):
6
    def __init__(self, num_inputs, num_outputs, num_hiddens1, num_hiddens2,
7
                 is_training = True):
8
        super().__init__()
9
        self.num_inputs = num_inputs
10
        self.training = is_training
11
        self.lin1 = nn.Linear(num_inputs, num_hiddens1)
12
        self.lin2 = nn.Linear(num_hiddens1, num_hiddens2)
13
        self.lin3 = nn.Linear(num_hiddens2, num_outputs)
14
        self.relu = nn.ReLU()
15

16
    def forward(self, X):
17
        H1 = self.relu(self.lin1(X.reshape(-1, self.num_inputs)))
18
        # 只有在训练模型时才使用dropout
19
        if self.training == True:
20
           # 在第一个全连接层之后添加一个dropout层
21
            H1 = dropout_layer(H1, dropout1)
22
        H2 = self.relu(self.lin2(H1))
23
        if self.training == True:
24
            # 在第二个全连接层之后添加一个dropout层
25
            H2 = dropout_layer(H2, dropout2)
26
        out = self.lin3(H2)
27
        return out
28

29
net = Net(num_inputs, num_outputs, num_hiddens1, num_hiddens2)

在靠近输入层的地方设置较低的暂退概率是常见的技巧。

为了对照，我们再定义一个Net2类，继承自Net，去掉Dropout操作，其他不变。

1
class Net2(Net):
2
    def __init__(self, num_inputs, num_outputs, num_hiddens1, num_hiddens2,
3
                is_training = True):
4
        super().__init__(num_inputs, num_outputs, num_hiddens1, num_hiddens2,
5
                is_training = True)
6

7
    def forward(self, X):
8
        H1 = self.relu(self.lin1(X.reshape(-1, num_inputs)))
9
        H2 = self.relu(self.lin2(H1))
10
        out = self.lin3(H2)
11
        return out
12

13
net2 = Net2(num_inputs, num_outputs, num_hiddens1, num_hiddens2)

训练和测试

1
num_epochs, lr, batch_size = 10, 0.5, 256
2
loss = nn.CrossEntropyLoss(reduction='none')
3
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)

先来试试有Dropout，再试试没有Dropout。

1
trainer = torch.optim.SGD(net.parameters(), lr=lr)
2
d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)

svg

1
trainer2 = torch.optim.SGD(net2.parameters(), lr=lr)
2
d2l.train_ch3(net2, train_iter, test_iter, loss, num_epochs, trainer2)

svg

简洁实现

1
net = nn.Sequential(nn.Flatten(),
2
                   nn.Linear(784, 256), nn.ReLU(),
3
                   nn.Dropout(dropout1),
4
                   nn.Linear(256, 256), nn.ReLU(),
5
                   nn.Dropout(dropout2),
6
                   nn.Linear(256, 10))
7

8
def init_weights(m):
9
    if type(m) == nn.Linear:
10
        nn.init.normal_(m.weight, std=0.01)
11

12
net.apply(init_weights)
13

14
trainer = torch.optim.SGD(net.parameters(), lr=lr)
15
d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)

svg