強化学習におけるアルゴリズム選択の問題

import numpy as np # 创建迷宫 maze = np.array([ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 0, 0, 1, 0, 0, 0, 1, 0, 1], [1, 0, 0, 1, 0, 0, 0, 1, 0, 1], [1, 0, 0, 0, 0, 1, 1, 0, 0, 1], [1, 0, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 0, 1, 0, 0, 0, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] ]) # 定义Q表格 Q = np.zeros((maze.shape[0], maze.shape[1], 4)) # 设置超参数 epochs = 5000 epsilon = 0.9 alpha = 0.1 gamma = 0.6 # Q-Learning算法 for episode in range(epochs): state = (1, 1) # 设置起点 while state != (6, 8): # 终点 x, y = state possible_actions = np.where(maze[x, y] == 0)[0] # 可能的动作 action = np.random.choice(possible_actions) # 选择动作 next_state = None if action == 0: next_state = (x - 1, y) elif action == 1: next_state = (x + 1, y) elif action == 2: next_state = (x, y - 1) elif action == 3: next_state = (x, y + 1) reward = -1 if next_state == (6, 8) else 0 # 终点奖励为0，其他状态奖励为-1 Q[x, y, action] = (1 - alpha) * Q[x, y, action] + alpha * (reward + gamma * np.max(Q[next_state])) state = next_state print(Q)

import torch import torch.nn as nn import torch.optim as optim import random # 创建神经网络 class DQN(nn.Module): def __init__(self, input_size, output_size): super(DQN, self).__init__() self.fc1 = nn.Linear(input_size, 16) self.fc2 = nn.Linear(16, output_size) def forward(self, x): x = torch.relu(self.fc1(x)) x = self.fc2(x) return x # 定义超参数 input_size = 4 output_size = 2 epochs = 1000 batch_size = 128 gamma = 0.99 epsilon = 0.2 # 创建经验回放内存 memory = [] capacity = 10000 # 创建神经网络和优化器 model = DQN(input_size, output_size) optimizer = optim.Adam(model.parameters(), lr=0.001) # 定义经验回放函数 def append_memory(state, action, next_state, reward): memory.append((state, action, next_state, reward)) if len(memory) > capacity: del memory[0] # 定义训练函数 def train(): if len(memory) < batch_size: return batch = random.sample(memory, batch_size) state_batch, action_batch, next_state_batch, reward_batch = zip(*batch) state_batch = torch.tensor(state_batch, dtype=torch.float) action_batch = torch.tensor(action_batch, dtype=torch.long) next_state_batch = torch.tensor(next_state_batch, dtype=torch.float) reward_batch = torch.tensor(reward_batch, dtype=torch.float) current_q = model(state_batch).gather(1, action_batch.unsqueeze(1)) next_q = model(next_state_batch).max(1)[0].detach() target_q = reward_batch + gamma * next_q loss = nn.MSELoss()(current_q, target_q.unsqueeze(1)) optimizer.zero_grad() loss.backward() optimizer.step() # DQN算法 for episode in range(epochs): state = env.reset() total_reward = 0 while True: if random.random() < epsilon: action = env.action_space.sample() else: action = model(torch.tensor(state, dtype=torch.float)).argmax().item() next_state, reward, done, _ = env.step(action) append_memory(state, action, next_state, reward) train() state = next_state total_reward += reward if done: break if episode % 100 == 0: print("Episode: ", episode, " Total Reward: ", total_reward) print("Training finished.")