guog算法笔记

V1

2023/04/06阅读：61主题：默认主题

PyTorch中的9种常见梯度下降算法与案例

@

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数loss_fn = torch.nn.MSELoss()# 定义优化器和超参数optimizer = torch.optim.SGD([w, b], lr=0.01)epochs = 100# 批量梯度下降for epoch in range(epochs):    # 前向传播    Y_pred = w * X + b    # 计算损失    loss = loss_fn(Y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 1.9999034404754639, b = 0.00041007260753999686``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.SGD([w, b], lr=0.01)# 定义超参数batch_size = 1epochs = 100# 随机梯度下降for epoch in range(epochs):    # 创建DataLoader    loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(X, Y), batch_size=batch_size, shuffle=True)    for x_batch, y_batch in loader:        # 前向传播        y_pred = w * x_batch + b        # 计算损失        loss = loss_fn(y_pred, y_batch)        # 反向传播        loss.backward()        # 更新参数        optimizer.step()        # 清空梯度        optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 2.0002050399780273, b = -0.0005163848866038325``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.SGD([w, b], lr=0.01)# 定义超参数batch_size = 2epochs = 100# 小批量梯度下降for epoch in range(epochs):    # 创建DataLoader    loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(X, Y), batch_size=batch_size, shuffle=True)    for x_batch, y_batch in loader:        # 前向传播        y_pred = w * x_batch + b        # 计算损失        loss = loss_fn(y_pred, y_batch)        # 反向传播        loss.backward()        # 更新参数        optimizer.step()        # 清空梯度        optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 1.9998304843902588, b = -0.00010240276428798664``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数和动量w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)momentum = 0.9# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.SGD([w, b], lr=0.01, momentum=momentum)# 定义超参数epochs = 100# 动量梯度下降for epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 1.9999991655349731, b = -3.718109681471333e-05``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.Adagrad([w, b], lr=0.1)# 定义超参数epochs = 100# AdaGradfor epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 1.999985694885254, b = -0.0010528544095081096``

6. RMSprop

RMSprop是一种自适应学习率的优化算法，在更新参数时根据历史梯度平方的加权平均来动态调整每个参数的学习率。在PyTorch中，可以通过设置optim.RMSprop()来使用该优化器。

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.RMSprop([w, b], lr=0.01)# 定义超参数epochs = 100# RMSpropfor epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 2.000011920928955, b = -0.00020079404229614145``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.Adam([w, b], lr=0.01)# 定义超参数epochs = 100# Adamfor epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 2.0000016689300537, b = -1.788223307551633e-05``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.AdamW([w, b], lr=0.01, weight_decay=0.1)# 定义超参数epochs = 100# AdamWfor epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 1.9564942121505737, b = 0.063056387424469``

``import torchimport matplotlib.pyplot as plt# 定义训练数据X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)# 初始化模型参数w = torch.tensor(0.0, requires_grad=True)b = torch.tensor(0.0, requires_grad=True)# 定义损失函数和优化器loss_fn = torch.nn.MSELoss()optimizer = torch.optim.Adadelta([w, b], lr=0.1)# 定义超参数epochs = 100# Adadeltafor epoch in range(epochs):    # 前向传播    y_pred = w * X + b    # 计算损失    loss = loss_fn(y_pred, Y)    # 反向传播    loss.backward()    # 更新参数    optimizer.step()    # 清空梯度    optimizer.zero_grad()# 输出结果print(f"w = {w.item()}, b = {b.item()}")# 绘制拟合直线plt.scatter(X.numpy(), Y.numpy())plt.plot(X.numpy(), (w * X + b).detach().numpy(), 'r')plt.show()``

``w = 2.0000007152557373, b = 4.5047908031606675e-08``

V1