多层感知机实现

激活函数

非线性

ReLU函数

修正线性单元 rectified linear unit
relu(x)=max(0,x)

relu的导数：

sigmoid函数

$sigmoid(x)=\frac{1}{1+e^{-x}}$
是一个早期的激活函数
缺点是：

1. 幂运算相对耗时，因此函数计算量较大
2. 在反向传播时容易出现梯度消失的情况
3. 收敛缓慢

导数是sigmoid(x)(1-sigmoid(x))：

Tanh 函数

双曲正切函数
$tanh(x)=\frac{1-e^{-2x}}{1+e^{-2x}}$

导数：$1-tan{h}^2(x)$

幂运算，同样存在计算量大的问题

阶跃函数

Leaky ReLU 函数 (LReLU)

如果在学习过程中，a 并不设定为一个常量，而是一个可通过反向传播算法学习的变量，则此时带泄露线性整流又被称为参数线性整流 (Parametric ReLU, PReLU)。

SoftPlus 函数

可以看成是 ReLU 函数的平滑版

多层感知机

multilayer perceptron

手写实现

import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as plt#加载fashion_mnist数据集
def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以#批大小
batch_size = 256
#训练和测试的迭代器
train_iter, test_iter = load_data_fashion_mnist(batch_size)num_inputs,num_outputs,num_hiddens=784,10,256W1=nn.Parameter(torch.randn(num_inputs,num_hiddens,requires_grad=True)*0.01)
b1=nn.Parameter(torch.zeros(num_hiddens),requires_grad=True)
W2=nn.Parameter(torch.randn(num_hiddens,num_outputs,requires_grad=True)*0.01)
b2=nn.Parameter(torch.zeros(num_outputs),requires_grad=True)
params=[W1,b1,W2,b2]def relu(x):a=torch.zeros_like(x)return torch.max(x,a)
def net(x):x=x.reshape(-1,num_inputs)h=relu(x@W1+b1)return (h@W2+b2)
loss=nn.CrossEntropyLoss(reduction='none')
num_epochs,lr=10,0.01
updater=torch.optim.SGD(params,lr=lr)
def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())
class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metrics# assert train_loss < 0.5, train_loss# assert train_acc <= 1 and train_acc > 0.7, train_acc# assert test_acc <= 1 and test_acc > 0.7, test_acctrain_ch3(net,train_iter,test_iter,loss,num_epochs,updater)
plt.show()def get_fashion_mnist_labels(labels):text_labels=['t-shirt', 'trouser', 'pullover', 'dress', 'coat','sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']return [text_labels[int(i)] for i in labels]
#展示数据集图片的函数
def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5): """绘制图像列表"""figsize = (num_cols * scale, num_rows * scale)_, axes = plt.subplots(num_rows, num_cols, figsize=figsize)axes = axes.flatten()for i, (ax, img) in enumerate(zip(axes, imgs)):if torch.is_tensor(img):# 图⽚张量ax.imshow(img.numpy())else:# PIL图⽚ax.imshow(img)ax.axes.get_xaxis().set_visible(False)ax.axes.get_yaxis().set_visible(False)if titles:ax.set_title(titles[i])plt.show()return axes
def predict_ch3(net, test_iter, n=6):"""预测标签（定义⻅第3章）"""for X, y in test_iter:breaktrues = get_fashion_mnist_labels(y)preds = get_fashion_mnist_labels(net(X).argmax(axis=1))titles = [true +'\n' + pred for true, pred in zip(trues, preds)]show_images(X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])predict_ch3(net, test_iter)

结果：

调库实现

import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as pltnet=nn.Sequential(nn.Flatten(),nn.Linear(784,256),nn.ReLU(),nn.Linear(256,10)                 )
def init_weights(m):if type(m)==nn.Linear:nn.init.normal_(m.weight,std=0.01)
net.apply(init_weights)
batch_size,lr,num_epochs=256,0.1,10
loss=nn.CrossEntropyLoss(reduction='none')
trainer=torch.optim.SGD(net.parameters(),lr=lr)
def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以
train_iter, test_iter = load_data_fashion_mnist(batch_size)
def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())
class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metrics# assert train_loss < 0.5, train_loss# assert train_acc <= 1 and train_acc > 0.7, train_acc# assert test_acc <= 1 and test_acc > 0.7, test_acctrain_ch3(net,train_iter,test_iter,loss,num_epochs,trainer)
plt.show()

欠拟合和过拟合

影响模型泛化的因素：

1. 参数的数量，越多越容易过拟合
2. 参数的取值，取值范围越大，越容易过拟合
3. 训练样本的数量，越少越容易过拟合

欠拟合：泛化能力差，训练样本集准确率低，测试样本集准确率低。
过拟合：泛化能力差，训练样本集准确率高，测试样本集准确率低。
合适的拟合程度：泛化能力强，训练样本集准确率高，测试样本集准确率高

欠拟合原因：

• 训练样本数量少
• 模型复杂度过低
• 参数还未收敛就停止循环

欠拟合的解决办法：

• 增加样本数量
• 增加模型参数，提高模型复杂度
• 增加循环次数
• 查看是否是学习率过高导致模型无法收敛

防止过拟合的方法：

• 增加训练集的样本数
• 交叉验证
• 数据增强
• 早停法
• 降低模型复杂度
• Dropout（随机丢弃）
• 正则化regularization（在loss里加入惩罚项）

权重衰减 weight_decay是正则化技术之一，即L2正则化

L2范数惩罚项指的是模型权重参数每个元素的平方和与一个正的常数的乘积。
$L(w,b)+\frac{\lambda }{2}||w|{|}^2$

L2正则化线性模型构成岭回归算法，L1构成套索回归算法。

def l2_penalty(w):return (w**2).sum() / 2

l = loss(net(X, w, b), y) + lambd * l2_penalty(w)

暂退法dropout

手动实现

import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as pltnum_inputs,num_outputs,num_hiddens1,num_hiddens2=784,10,256,256
dropout1,dropout2=0.2,0.5
def dropout_layer(x,dropout):assert 0<=dropout<=1if dropout==1:return torch.zeros_like(x)if dropout==0:return xmask=(torch.rand(x.shape)>dropout).float()return mask*x/(1.0-dropout)class Net(nn.Module):def __init__(self,num_inputs,num_outputs,num_hiddens1,num_hiddens2,is_training=True):super(Net,self).__init__()self.num_inputs=num_inputsself.training=is_trainingself.lin1=nn.Linear(num_inputs,num_hiddens1)self.lin2=nn.Linear(num_hiddens1,num_hiddens2)self.lin3=nn.Linear(num_hiddens2,num_outputs)self.relu=nn.ReLU()def forward(self,x):h1=self.relu(self.lin1(x.reshape((-1,self.num_inputs))))if self.training==True:h1=dropout_layer(h1,dropout1)h2=self.relu(self.lin2(h1))if self.training==True:h2=dropout_layer(h2,dropout2)out=self.lin3(h2)return outnet=Net(num_inputs,num_outputs,num_hiddens1,num_hiddens2)num_epochs,lr,batch_size=10,0.5,256
loss=nn.CrossEntropyLoss(reduction='none')def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以
train_iter, test_iter = load_data_fashion_mnist(batch_size)trainer=torch.optim.SGD(net.parameters(),lr=lr)def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())
class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metrics# assert train_loss < 0.5, train_loss# assert train_acc <= 1 and train_acc > 0.7, train_acc# assert test_acc <= 1 and test_acc > 0.7, test_acctrain_ch3(net,train_iter,test_iter,loss,num_epochs,trainer)
plt.show()

调库实现

import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as plt
dropout1,dropout2=0.2,0.5
net=nn.Sequential(nn.Flatten(),nn.Linear(784,256),nn.ReLU(),nn.Dropout(dropout1),nn.Linear(256,256),nn.ReLU(),nn.Dropout(dropout2),nn.Linear(256,10)
)
def init_weights(m):if type(m)==nn.Linear:nn.init.normal_(m.weight,std=0.01)
net.apply(init_weights)num_epochs,lr,batch_size=10,0.5,256
loss=nn.CrossEntropyLoss(reduction='none')def load_data_fashion_mnist(batch_size, resize=None):"""下载Fashion-MNIST数据集，然后将其加载到内存中"""trans = [transforms.ToTensor()]if resize:trans.insert(0, transforms.Resize(resize))trans = transforms.Compose(trans)mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)#print(len(mnist_train),len(mnist_test))return (data.DataLoader(mnist_train, batch_size, shuffle=True),data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以
train_iter, test_iter = load_data_fashion_mnist(batch_size)trainer=torch.optim.SGD(net.parameters(),lr=lr)def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):"""设置matplotlib的轴"""axes.set_xlabel(xlabel)axes.set_ylabel(ylabel)axes.set_xscale(xscale)axes.set_yscale(yscale)axes.set_xlim(xlim)axes.set_ylim(ylim)if legend:axes.legend(legend)axes.grid()
class Animator: """在动画中绘制数据"""def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,ylim=None, xscale='linear', yscale='linear',fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,figsize=(3.5, 2.5)):# 增量地绘制多条线if legend is None:legend = []self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)if nrows * ncols == 1:self.axes = [self.axes, ]# 使⽤lambda函数捕获参数self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)self.X, self.Y, self.fmts = None, None, fmtsdef add(self, x, y):# 向图表中添加多个数据点if not hasattr(y, "__len__"):y = [y]n = len(y)if not hasattr(x, "__len__"):x = [x] * nif not self.X:self.X = [[] for _ in range(n)]if not self.Y:self.Y = [[] for _ in range(n)]for i, (a, b) in enumerate(zip(x, y)):if a is not None and b is not None:self.X[i].append(a)self.Y[i].append(b)self.axes[0].cla()for x, y, fmt in zip(self.X, self.Y, self.fmts):self.axes[0].plot(x, y, fmt)self.config_axes()#display.display(self.fig)# 通过以下两行代码实现了在PyCharm中显示动图plt.draw()#plt.pause(interval=0.001)#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y): """计算预测正确的数量"""if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:y_hat = y_hat.argmax(axis=1)cmp = y_hat.type(y.dtype) == yreturn float(cmp.type(y.dtype).sum())
class Accumulator: """在n个变量上累加"""def __init__(self, n):self.data = [0.0] * ndef add(self, *args):self.data = [a + float(b) for a, b in zip(self.data, args)]def reset(self):self.data = [0.0] * len(self.data)def __getitem__(self, idx):return self.data[idx]
def evaluate_accuracy(net, data_iter):"""计算在指定数据集上模型的精度"""if isinstance(net, torch.nn.Module):net.eval() # 将模型设置为评估模式metric = Accumulator(2) # 正确预测数、预测总数with torch.no_grad():for X, y in data_iter:metric.add(accuracy(net(X), y), y.numel())return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater): """训练模型⼀个迭代周期（定义⻅第3章）"""# 将模型设置为训练模式if isinstance(net, torch.nn.Module):net.train()# 训练损失总和、训练准确度总和、样本数metric = Accumulator(3)for X, y in train_iter:# 计算梯度并更新参数y_hat = net(X)l = loss(y_hat, y)if isinstance(updater, torch.optim.Optimizer):# 使⽤PyTorch内置的优化器和损失函数updater.zero_grad() #清除梯度l.mean().backward() #反向传播updater.step()else:# 使⽤定制的优化器和损失函数l.sum().backward()updater(X.shape[0])metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())# 返回训练损失和训练精度return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save"""训练模型（定义⻅第3章）"""animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],legend=['train loss', 'train acc', 'test acc'])for epoch in range(num_epochs):train_metrics = train_epoch_ch3(net, train_iter, loss, updater)test_acc = evaluate_accuracy(net, test_iter)animator.add(epoch + 1, train_metrics + (test_acc,))train_loss, train_acc = train_metrics# assert train_loss < 0.5, train_loss# assert train_acc <= 1 and train_acc > 0.7, train_acc# assert test_acc <= 1 and test_acc > 0.7, test_acctrain_ch3(net,train_iter,test_iter,loss,num_epochs,trainer)
plt.show()

梯度消失

gradient vanishing =梯度弥散=参数更新过小，每次基本不会移动
sigmoid会导致这个问题

梯度爆炸

初始化设置不合理，没有机会让梯度下降优化器收敛，解决方法：参数初始化
初始化可以是正态分布，或者框架的默认初始化方法。
或Xavier初始化方法：从均值为0，方差为$\frac{2}{n_{in}+n_{out}}$的高斯分布里抽样权重，或从同方差的均匀分布里抽样（均匀分布U(-a,a)的方差是a^2/3），这个初始化方法很常用。