자율주행 시스템 구현을 위해 Python을 배워보세요

pip install gym

pip install --user git+https://github.com/eleurent/highway-env

import gym import highway_env %matplotlib inline env = gym.make('highway-v0') env.reset() for _ in range(3): action = env.action_type.actions_indexes["IDLE"] obs, reward, done, info = env.step(action) env.render()

config = { "observation": { "type": "Kinematics", #选取5辆车进行观察（包括ego vehicle） "vehicles_count": 5, #共7个特征 "features": ["presence", "x", "y", "vx", "vy", "cos_h", "sin_h"], "features_range": { "x": [-100, 100], "y": [-100, 100], "vx": [-20, 20], "vy": [-20, 20] }, "absolute": False, "order": "sorted" }, "simulation_frequency": 8,# [Hz] "policy_frequency": 2,# [Hz] }

ACTIONS_ALL = { 0: 'LANE_LEFT', 1: 'IDLE', 2: 'LANE_RIGHT', 3: 'FASTER', 4: 'SLOWER' }

import torch import torch.nn as nn from torch.autograd import Variable import torch.nn.functional as F import torch.optim as optim import torchvision.transforms as T from torch import FloatTensor, LongTensor, ByteTensor from collections import namedtuple import random Tensor = FloatTensor EPSILON = 0# epsilon used for epsilon greedy approach GAMMA = 0.9 TARGET_NETWORK_REPLACE_FREQ = 40 # How frequently target netowrk updates MEMORY_CAPACITY = 100 BATCH_SIZE = 80 LR = 0.01 # learning rate class DQNNet(nn.Module): def __init__(self): super(DQNNet,self).__init__() self.linear1 = nn.Linear(35,35) self.linear2 = nn.Linear(35,5) def forward(self,s): s=torch.FloatTensor(s) s = s.view(s.size(0),1,35) s = self.linear1(s) s = self.linear2(s) return s class DQN(object): def __init__(self): self.net,self.target_net = DQNNet(),DQNNet() self.learn_step_counter = 0 self.memory = [] self.position = 0 self.capacity = MEMORY_CAPACITY self.optimizer = torch.optim.Adam(self.net.parameters(), lr=LR) self.loss_func = nn.MSELoss() def choose_action(self,s,e): x=np.expand_dims(s, axis=0) if np.random.uniform() < 1-e: actions_value = self.net.forward(x) action = torch.max(actions_value,-1)[1].data.numpy() action = action.max() else: action = np.random.randint(0, 5) return action def push_memory(self, s, a, r, s_): if len(self.memory) < self.capacity: self.memory.append(None) self.memory[self.position] = Transition(torch.unsqueeze(torch.FloatTensor(s), 0),torch.unsqueeze(torch.FloatTensor(s_), 0), torch.from_numpy(np.array([a])),torch.from_numpy(np.array([r],dtype='float32')))# self.position = (self.position + 1) % self.capacity def get_sample(self,batch_size): sample = random.sample(self.memory,batch_size) return sample def learn(self): if self.learn_step_counter % TARGET_NETWORK_REPLACE_FREQ == 0: self.target_net.load_state_dict(self.net.state_dict()) self.learn_step_counter += 1 transitions = self.get_sample(BATCH_SIZE) batch = Transition(*zip(*transitions)) b_s = Variable(torch.cat(batch.state)) b_s_ = Variable(torch.cat(batch.next_state)) b_a = Variable(torch.cat(batch.action)) b_r = Variable(torch.cat(batch.reward)) q_eval = self.net.forward(b_s).squeeze(1).gather(1,b_a.unsqueeze(1).to(torch.int64)) q_next = self.target_net.forward(b_s_).detach() # q_target = b_r + GAMMA * q_next.squeeze(1).max(1)[0].view(BATCH_SIZE, 1).t() loss = self.loss_func(q_eval, q_target.t()) self.optimizer.zero_grad() # reset the gradient to zero loss.backward() self.optimizer.step() # execute back propagation for one step return loss Transition = namedtuple('Transition',('state', 'next_state','action', 'reward'))

import gym import highway_env from matplotlib import pyplot as plt import numpy as np import time config = { "observation": { "type": "Kinematics", "vehicles_count": 5, "features": ["presence", "x", "y", "vx", "vy", "cos_h", "sin_h"], "features_range": { "x": [-100, 100], "y": [-100, 100], "vx": [-20, 20], "vy": [-20, 20] }, "absolute": False, "order": "sorted" }, "simulation_frequency": 8,# [Hz] "policy_frequency": 2,# [Hz] } env = gym.make("highway-v0") env.configure(config)

dqn=DQN() count=0 reward=[] avg_reward=0 all_reward=[] time_=[] all_time=[] collision_his=[] all_collision=[] while True: done = False start_time=time.time() s = env.reset() while not done: e = np.exp(-count/300)#随机选择action的概率，随着训练次数增多逐渐降低 a = dqn.choose_action(s,e) s_, r, done, info = env.step(a) env.render() dqn.push_memory(s, a, r, s_) if ((dqn.position !=0)&(dqn.position % 99==0)): loss_=dqn.learn() count+=1 print('trained times:',count) if (count%40==0): avg_reward=np.mean(reward) avg_time=np.mean(time_) collision_rate=np.mean(collision_his) all_reward.append(avg_reward) all_time.append(avg_time) all_collision.append(collision_rate) plt.plot(all_reward) plt.show() plt.plot(all_time) plt.show() plt.plot(all_collision) plt.show() reward=[] time_=[] collision_his=[] s = s_ reward.append(r) end_time=time.time() episode_time=end_time-start_time time_.append(episode_time) is_collision=1 if info['crashed']==True else 0 collision_his.append(is_collision)