強化學習中的演算法選擇問題

import numpy as np

# 创建迷宫
maze = np.array([
    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
    [1, 0, 0, 1, 0, 0, 0, 1, 0, 1],
    [1, 0, 0, 1, 0, 0, 0, 1, 0, 1],
    [1, 0, 0, 0, 0, 1, 1, 0, 0, 1],
    [1, 0, 1, 1, 1, 0, 0, 0, 0, 1],
    [1, 0, 0, 0, 1, 0, 0, 0, 1, 1],
    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
])

# 定义Q表格
Q = np.zeros((maze.shape[0], maze.shape[1], 4))

# 设置超参数
epochs = 5000
epsilon = 0.9
alpha = 0.1
gamma = 0.6

# Q-Learning算法
for episode in range(epochs):
    state = (1, 1)  # 设置起点
    while state != (6, 8):  # 终点
        x, y = state
        possible_actions = np.where(maze[x, y] == 0)[0]  # 可能的动作
        action = np.random.choice(possible_actions)  # 选择动作

        next_state = None
        if action == 0:
            next_state = (x - 1, y)
        elif action == 1:
            next_state = (x + 1, y)
        elif action == 2:
            next_state = (x, y - 1)
        elif action == 3:
            next_state = (x, y + 1)

        reward = -1 if next_state == (6, 8) else 0  # 终点奖励为0，其他状态奖励为-1

        Q[x, y, action] = (1 - alpha) * Q[x, y, action] + alpha * (reward + gamma * np.max(Q[next_state]))

        state = next_state

print(Q)

import torch
import torch.nn as nn
import torch.optim as optim
import random

# 创建神经网络
class DQN(nn.Module):
    def __init__(self, input_size, output_size):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(input_size, 16)
        self.fc2 = nn.Linear(16, output_size)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

# 定义超参数
input_size = 4
output_size = 2
epochs = 1000
batch_size = 128
gamma = 0.99
epsilon = 0.2

# 创建经验回放内存
memory = []
capacity = 10000

# 创建神经网络和优化器
model = DQN(input_size, output_size)
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 定义经验回放函数
def append_memory(state, action, next_state, reward):
    memory.append((state, action, next_state, reward))
    if len(memory) > capacity:
        del memory[0]

# 定义训练函数
def train():
    if len(memory) < batch_size:
        return

    batch = random.sample(memory, batch_size)
    state_batch, action_batch, next_state_batch, reward_batch = zip(*batch)

    state_batch = torch.tensor(state_batch, dtype=torch.float)
    action_batch = torch.tensor(action_batch, dtype=torch.long)
    next_state_batch = torch.tensor(next_state_batch, dtype=torch.float)
    reward_batch = torch.tensor(reward_batch, dtype=torch.float)

    current_q = model(state_batch).gather(1, action_batch.unsqueeze(1))
    next_q = model(next_state_batch).max(1)[0].detach()
    target_q = reward_batch + gamma * next_q

    loss = nn.MSELoss()(current_q, target_q.unsqueeze(1))

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

# DQN算法
for episode in range(epochs):
    state = env.reset()
    total_reward = 0

    while True:
        if random.random() < epsilon:
            action = env.action_space.sample()
        else:
            action = model(torch.tensor(state, dtype=torch.float)).argmax().item()

        next_state, reward, done, _ = env.step(action)

        append_memory(state, action, next_state, reward)
        train()

        state = next_state
        total_reward += reward

        if done:
            break

    if episode % 100 == 0:
        print("Episode: ", episode, " Total Reward: ", total_reward)

print("Training finished.")