From 5f8ba1601ff9407077c7dd13f5d5095eb34a83b7 Mon Sep 17 00:00:00 2001 From: zhm-real Date: Sun, 21 Jun 2020 21:57:29 -0700 Subject: [PATCH] regulate format --- .../.idea/Model-free Control.iml | 5 +- Model-free Control/.idea/misc.xml | 2 +- Model-free Control/Q-learning.py | 52 +++++++------- Model-free Control/Sarsa.py | 47 +++++++------ Model-free Control/env.py | 8 +-- Model-free Control/motion_model.py | 9 +-- Model-free Control/plotting.py | 17 ++--- .../.idea/Search-based Planning.iml | 2 +- Search-based Planning/.idea/misc.xml | 2 +- Search-based Planning/.idea/workspace.xml | 33 ++++----- .../__pycache__/env.cpython-37.pyc | Bin 1141 -> 1141 bytes .../__pycache__/plotting.cpython-35.pyc | Bin 2941 -> 2899 bytes .../__pycache__/plotting.cpython-37.pyc | Bin 2691 -> 2651 bytes .../__pycache__/queue.cpython-35.pyc | Bin 2705 -> 2667 bytes .../__pycache__/queue.cpython-37.pyc | Bin 2565 -> 2529 bytes .../.idea/Stochastic Shortest Path.iml | 3 +- Stochastic Shortest Path/.idea/misc.xml | 2 +- .../Q-policy_iteration.py | 45 +++++------- Stochastic Shortest Path/Q-value_iteration.py | 35 ++++------ .../__pycache__/env.cpython-37.pyc | Bin 2208 -> 2157 bytes .../__pycache__/motion_model.cpython-37.pyc | Bin 1274 -> 1223 bytes .../__pycache__/plotting.cpython-37.pyc | Bin 4377 -> 4326 bytes Stochastic Shortest Path/env.py | 20 ++---- Stochastic Shortest Path/motion_model.py | 10 +-- Stochastic Shortest Path/plotting.py | 15 +--- Stochastic Shortest Path/policy_iteration.py | 41 ++++------- Stochastic Shortest Path/value_iteration.py | 65 ++++++++---------- 27 files changed, 156 insertions(+), 257 deletions(-) diff --git a/Model-free Control/.idea/Model-free Control.iml b/Model-free Control/.idea/Model-free Control.iml index f3d7bc9..5965bde 100644 --- a/Model-free Control/.idea/Model-free Control.iml +++ b/Model-free Control/.idea/Model-free Control.iml @@ -2,10 +2,7 @@ - + - - \ No newline at end of file diff --git a/Model-free Control/.idea/misc.xml b/Model-free Control/.idea/misc.xml index a2e120d..0e7ac62 100644 --- a/Model-free Control/.idea/misc.xml +++ b/Model-free Control/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Model-free Control/Q-learning.py b/Model-free Control/Q-learning.py index 88a5622..bc428bf 100644 --- a/Model-free Control/Q-learning.py +++ b/Model-free Control/Q-learning.py @@ -1,20 +1,15 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model import numpy as np + class QLEARNING: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.M = 500 # iteration numbers - self.gamma = 0.9 # discount factor + self.M = 500 # iteration numbers + self.gamma = 0.9 # discount factor self.alpha = 0.5 self.epsilon = 0.1 @@ -22,10 +17,10 @@ class QLEARNING: self.motion = motion_model.Motion_model(self.xI, self.xG) self.plotting = plotting.Plotting(self.xI, self.xG) - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "SARSA, M=" + str(self.M) @@ -33,7 +28,6 @@ class QLEARNING: self.path = self.extract_path(self.xI, self.xG, self.policy) self.plotting.animation(self.path, self.name1) - def Monte_Carlo(self, xI, xG): """ Monte_Carlo experiments @@ -41,21 +35,21 @@ class QLEARNING: :return: Q_table, policy """ - Q_table = self.table_init() # Q_table initialization - policy = {} # policy table + Q_table = self.table_init() # Q_table initialization + policy = {} # policy table - for k in range(self.M): # iterations - x = self.state_init() # initial state - while x != xG: # stop condition - u = self.epsilon_greedy(int(np.argmax(Q_table[x])), self.epsilon) # epsilon_greedy policy - x_next = self.move_next(x, self.u_set[u]) # next state - reward = self.env.get_reward(x_next) # reward observed + for k in range(self.M): # iterations + x = self.state_init() # initial state + while x != xG: # stop condition + u = self.epsilon_greedy(int(np.argmax(Q_table[x])), self.epsilon) # epsilon_greedy policy + x_next = self.move_next(x, self.u_set[u]) # next state + reward = self.env.get_reward(x_next) # reward observed Q_table[x][u] = (1 - self.alpha) * Q_table[x][u] + \ self.alpha * (reward + self.gamma * max(Q_table[x_next])) x = x_next for x in Q_table: - policy[x] = int(np.argmax(Q_table[x])) # extract policy + policy[x] = int(np.argmax(Q_table[x])) # extract policy return Q_table, policy @@ -89,7 +83,6 @@ class QLEARNING: if (i, j) not in self.obs: return (i, j) - def epsilon_greedy(self, u, error): """ generate a policy using epsilon_greedy algorithm @@ -103,14 +96,17 @@ class QLEARNING: u_e = u while u_e == u: p = np.random.random_sample() - if p < 0.25: u_e = 0 - elif p < 0.5: u_e = 1 - elif p < 0.75: u_e = 2 - else: u_e = 3 + if p < 0.25: + u_e = 0 + elif p < 0.5: + u_e = 1 + elif p < 0.75: + u_e = 2 + else: + u_e = 3 return u_e return u - def move_next(self, x, u): """ get next state. diff --git a/Model-free Control/Sarsa.py b/Model-free Control/Sarsa.py index c637734..2278a74 100644 --- a/Model-free Control/Sarsa.py +++ b/Model-free Control/Sarsa.py @@ -1,20 +1,15 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model import numpy as np + class SARSA: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.M = 500 # iteration numbers - self.gamma = 0.9 # discount factor + self.M = 500 # iteration numbers + self.gamma = 0.9 # discount factor self.alpha = 0.5 self.epsilon = 0.1 @@ -22,10 +17,10 @@ class SARSA: self.motion = motion_model.Motion_model(self.xI, self.xG) self.plotting = plotting.Plotting(self.xI, self.xG) - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "Q-learning, M=" + str(self.M) @@ -40,22 +35,22 @@ class SARSA: :return: Q_table, policy """ - Q_table = self.table_init() # Q_table initialization - policy = {} # policy table + Q_table = self.table_init() # Q_table initialization + policy = {} # policy table - for k in range(self.M): # iterations - x = self.state_init() # initial state + for k in range(self.M): # iterations + x = self.state_init() # initial state u = self.epsilon_greedy(int(np.argmax(Q_table[x])), self.epsilon) - while x != xG: # stop condition - x_next = self.move_next(x, self.u_set[u]) # next state - reward = self.env.get_reward(x_next) # reward observed + while x != xG: # stop condition + x_next = self.move_next(x, self.u_set[u]) # next state + reward = self.env.get_reward(x_next) # reward observed u_next = self.epsilon_greedy(int(np.argmax(Q_table[x_next])), self.epsilon) Q_table[x][u] = (1 - self.alpha) * Q_table[x][u] + \ self.alpha * (reward + self.gamma * Q_table[x_next][u_next]) x, u = x_next, u_next for x in Q_table: - policy[x] = int(np.argmax(Q_table[x])) # extract policy + policy[x] = int(np.argmax(Q_table[x])) # extract policy return Q_table, policy @@ -102,10 +97,14 @@ class SARSA: u_e = u while u_e == u: p = np.random.random_sample() - if p < 0.25: u_e = 0 - elif p < 0.5: u_e = 1 - elif p < 0.75: u_e = 2 - else: u_e = 3 + if p < 0.25: + u_e = 0 + elif p < 0.5: + u_e = 1 + elif p < 0.75: + u_e = 2 + else: + u_e = 3 return u_e return u diff --git a/Model-free Control/env.py b/Model-free Control/env.py index 6b1dcc1..8deed5d 100644 --- a/Model-free Control/env.py +++ b/Model-free Control/env.py @@ -1,10 +1,4 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - -class Env(): +class Env: def __init__(self, xI, xG): self.x_range = 14 # size of background self.y_range = 6 diff --git a/Model-free Control/motion_model.py b/Model-free Control/motion_model.py index 266a7d8..0285570 100644 --- a/Model-free Control/motion_model.py +++ b/Model-free Control/motion_model.py @@ -1,11 +1,6 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env + class Motion_model(): def __init__(self, xI, xG): self.env = env.Env(xI, xG) @@ -40,4 +35,4 @@ class Motion_model(): else: x_next.append(x_check) - return x_next, p_next \ No newline at end of file + return x_next, p_next diff --git a/Model-free Control/plotting.py b/Model-free Control/plotting.py index 5fbe957..98f8d16 100644 --- a/Model-free Control/plotting.py +++ b/Model-free Control/plotting.py @@ -1,12 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import matplotlib.pyplot as plt import env + class Plotting(): def __init__(self, xI, xG): self.xI, self.xG = xI, xG @@ -39,10 +34,10 @@ class Plotting(): obs_x = [self.obs[i][0] for i in range(len(self.obs))] obs_y = [self.obs[i][1] for i in range(len(self.obs))] - plt.plot(self.xI[0], self.xI[1], "bs", ms = 24) - plt.plot(self.xG[0], self.xG[1], "gs", ms = 24) + plt.plot(self.xI[0], self.xI[1], "bs", ms=24) + plt.plot(self.xG[0], self.xG[1], "gs", ms=24) - plt.plot(obs_x, obs_y, "sk", ms = 24) + plt.plot(obs_x, obs_y, "sk", ms=24) plt.title(name) plt.axis("equal") @@ -55,7 +50,7 @@ class Plotting(): lose_x = [self.lose[i][0] for i in range(len(self.lose))] lose_y = [self.lose[i][1] for i in range(len(self.lose))] - plt.plot(lose_x, lose_y, color = '#A52A2A', marker = 's', ms = 24) + plt.plot(lose_x, lose_y, color='#A52A2A', marker='s', ms=24) def plot_visited(self, visited): """ @@ -88,7 +83,7 @@ class Plotting(): path.remove(self.xG) for x in path: - plt.plot(x[0], x[1], color='#808080', marker='o', ms = 23) + plt.plot(x[0], x[1], color='#808080', marker='o', ms=23) plt.gcf().canvas.mpl_connect('key_release_event', lambda event: [exit(0) if event.key == 'escape' else None]) plt.pause(0.001) diff --git a/Search-based Planning/.idea/Search-based Planning.iml b/Search-based Planning/.idea/Search-based Planning.iml index fee2751..5965bde 100644 --- a/Search-based Planning/.idea/Search-based Planning.iml +++ b/Search-based Planning/.idea/Search-based Planning.iml @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/Search-based Planning/.idea/misc.xml b/Search-based Planning/.idea/misc.xml index 6c993b7..0e7ac62 100644 --- a/Search-based Planning/.idea/misc.xml +++ b/Search-based Planning/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Search-based Planning/.idea/workspace.xml b/Search-based Planning/.idea/workspace.xml index efd2fab..fe08e6f 100644 --- a/Search-based Planning/.idea/workspace.xml +++ b/Search-based Planning/.idea/workspace.xml @@ -2,16 +2,7 @@ - - - - - - - - - - + - + @@ -186,25 +177,25 @@ - + - - + + - - + + - - + + - - + + - + \ No newline at end of file diff --git a/Search-based Planning/__pycache__/env.cpython-37.pyc b/Search-based Planning/__pycache__/env.cpython-37.pyc index d07c6920fad0e30fa2b4e5b22b47e5dd1e047eaa..c88f4cebe3319fd2c40951dfc519d1160d316753 100644 GIT binary patch delta 34 ocmey$@s)$uiI^JGT0$&M^00HqNLH~;_u delta 34 pcmey$@s)$uiIc3CV+jF*?o%KSr|F&hKJV+JI^3}ib1aj_AQNC6^7h8zZlEJlVXCK#8QA&Qy7 zJq5^TW@rJbU}k7$Vu)e^DGkxSFL{K@Q*o5diQ_avZf;^KWMq_>{DpZgqvYgCEIo|ko8?(wGcsyU=4L;}sI>VK zdom-VG@G2r)vU|~awUl1VdP;0LT(0#waw|oTZ0RVDk}{WsHo9le5{5 zF-mPVV%K40Gyuv=O|ImqU^JZkiDNRO?&LPkW+_&noy8z$a4>SP0wEWNa1j%btI0Z9 Rhs%SP5hTq5BzPD(m;v-oBpCnz delta 218 zcmcaD(k#mB#LLUY00c`EKg6|6D2y6y_9` z7KRj-W~M0C6sBMXP1Y)LE{DX@l8pQ!D}{{G%-qbpbcL#n{8FxsL6(e++>?74_b>`> z_GAKbBqldAPh*svEXC5pD86|)%Q8kr&B=aj#~771i?Zu5GFnb{;+Vu}HTf#XWJa^e znVikaAg2|B2o6RLHX!8Y5Gi5;ay8j*vB$@!|i0F3Py%tO^%lg8|O7MGICE= zVya=}n>-Qm?`&SMmr+|69Z$hw({#h#H-Zn6_=6{F(hy{r|CDw7woiB0xl3t?0T zvSJw}HveZcW@I#+Y|K&3Xfk;{kc^tF%_+-hKG~DAfzfL6Db6@4ZICy(fCLW<8xs#R c2o{L}xtgq#-MP#K89~w)z0FKi$jsO4v delta 314 zcmaDYGEuZ%jF*>diQh($ diff --git a/Search-based Planning/__pycache__/queue.cpython-37.pyc b/Search-based Planning/__pycache__/queue.cpython-37.pyc index 6c1e8ed0fd660d51da282ebbea92cf331da6f4ec..e1eea171de274cc2c058317faa45ecf23bf03d09 100644 GIT binary patch delta 238 zcmZn_c__^1#LLUY00dR$AL4$nOyrYru>7@{CFV-%x1Lkd$0a|=TXb2C#E z6OhM}!V2WE0(s0p9$N}KkjD<>v7|5tGiY*bY|mn3SE0$4W^BguQMn=QQ z+u6$)O(yGclyF6H6clCV7iE@IPCm#g!e~DEAV&?O6_6C49Lnj=s6BZurvxMGpB3KpuMv2av}BtB$k$B&)E8FhfHSVq~+^=wj%jHZ)kv6nGgOcvxQk-x=JP?VWplvz?)#13>m zgx~ - + - \ No newline at end of file diff --git a/Stochastic Shortest Path/.idea/misc.xml b/Stochastic Shortest Path/.idea/misc.xml index a2e120d..0e7ac62 100644 --- a/Stochastic Shortest Path/.idea/misc.xml +++ b/Stochastic Shortest Path/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Stochastic Shortest Path/Q-policy_iteration.py b/Stochastic Shortest Path/Q-policy_iteration.py index 825420d..f29cee0 100644 --- a/Stochastic Shortest Path/Q-policy_iteration.py +++ b/Stochastic Shortest Path/Q-policy_iteration.py @@ -1,9 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model @@ -12,20 +6,21 @@ import numpy as np import copy import sys + class Q_policy_iteration: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.e = 0.001 # threshold for convergence - self.gamma = 0.9 # discount factor + self.e = 0.001 # threshold for convergence + self.gamma = 0.9 # discount factor - self.env = env.Env(self.xI, self.xG) # class Env - self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model - self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting + self.env = env.Env(self.xI, self.xG) # class Env + self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model + self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "Q-policy_iteration, gamma=" + str(self.gamma) @@ -33,7 +28,6 @@ class Q_policy_iteration: self.path = self.extract_path(self.xI, self.xG, self.policy) self.plotting.animation(self.path, self.name1) - def policy_evaluation(self, policy, value): """ evaluation process using current policy. @@ -45,7 +39,7 @@ class Q_policy_iteration: delta = sys.maxsize - while delta > self.e: # convergence condition + while delta > self.e: # convergence condition x_value = 0 for x in value: if x not in self.xG: @@ -60,7 +54,6 @@ class Q_policy_iteration: return value - def policy_improvement(self, policy, value): """ policy improvement process. @@ -76,7 +69,6 @@ class Q_policy_iteration: return policy - def iteration(self): """ Q-policy iteration @@ -88,21 +80,20 @@ class Q_policy_iteration: count = 0 for x in self.stateSpace: - Q_table[x] = [0, 0, 0, 0] # initialize Q_value table - policy[x] = 0 # initialize policy table + Q_table[x] = [0, 0, 0, 0] # initialize Q_value table + policy[x] = 0 # initialize policy table while True: count += 1 policy_back = copy.deepcopy(policy) - Q_table = self.policy_evaluation(policy, Q_table) # evaluation process - policy = self.policy_improvement(policy, Q_table) # improvement process - if policy_back == policy: break # convergence condition + Q_table = self.policy_evaluation(policy, Q_table) # evaluation process + policy = self.policy_improvement(policy, Q_table) # improvement process + if policy_back == policy: break # convergence condition self.message(count) return Q_table, policy - def cal_Q_value(self, x, p, policy, table): """ cal Q_value. @@ -114,13 +105,12 @@ class Q_policy_iteration: """ value = 0 - reward = self.env.get_reward(x) # get reward of next state + reward = self.env.get_reward(x) # get reward of next state for i in range(len(x)): value += p[i] * (reward[i] + self.gamma * table[x[i]][policy[x[i]]]) return value - def extract_path(self, xI, xG, policy): """ extract path from converged policy. @@ -143,7 +133,6 @@ class Q_policy_iteration: x = x_next return path - def message(self, count): """ print important message. diff --git a/Stochastic Shortest Path/Q-value_iteration.py b/Stochastic Shortest Path/Q-value_iteration.py index e324395..a8bf45a 100644 --- a/Stochastic Shortest Path/Q-value_iteration.py +++ b/Stochastic Shortest Path/Q-value_iteration.py @@ -1,9 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model @@ -11,20 +5,21 @@ import motion_model import numpy as np import sys + class Q_value_iteration: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.e = 0.001 # threshold for convergence - self.gamma = 0.9 # discount factor + self.e = 0.001 # threshold for convergence + self.gamma = 0.9 # discount factor - self.env = env.Env(self.xI, self.xG) # class Env - self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model - self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting + self.env = env.Env(self.xI, self.xG) # class Env + self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model + self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "Q-value_iteration, gamma=" + str(self.gamma) self.name2 = "converge process, e=" + str(self.e) @@ -34,7 +29,6 @@ class Q_value_iteration: self.plotting.animation(self.path, self.name1) self.plotting.plot_diff(self.diff, self.name2) - def iteration(self, xI, xG): """ Q_value_iteration @@ -48,9 +42,9 @@ class Q_value_iteration: count = 0 for x in self.stateSpace: - Q_table[x] = [0, 0, 0, 0] # initialize Q_table + Q_table[x] = [0, 0, 0, 0] # initialize Q_table - while delta > self.e: # convergence condition + while delta > self.e: # convergence condition count += 1 x_value = 0 for x in self.stateSpace: @@ -73,7 +67,6 @@ class Q_value_iteration: return Q_table, policy, diff - def cal_Q_value(self, x, p, table): """ cal Q_value. @@ -85,13 +78,12 @@ class Q_value_iteration: """ value = 0 - reward = self.env.get_reward(x) # get reward of next state + reward = self.env.get_reward(x) # get reward of next state for i in range(len(x)): value += p[i] * (reward[i] + self.gamma * max(table[x[i]])) return value - def extract_path(self, xI, xG, policy): """ extract path from converged policy. @@ -114,7 +106,6 @@ class Q_value_iteration: x = x_next return path - def message(self, count): """ print important message. diff --git a/Stochastic Shortest Path/__pycache__/env.cpython-37.pyc b/Stochastic Shortest Path/__pycache__/env.cpython-37.pyc index e73551b3c940813e8004c8bf7a9a74410143e43f..b12fe1668901320ad2840eb32c6b50f6f461b741 100644 GIT binary patch delta 193 zcmZ1=_*Q_|iI`l)$k zdIgoYIO5|o^D;}~uXkyp}G49H1gh+=SONMTH2YGFuWYG#UJOkoaY&}69+ z=W<9aEy>6)vQo$>&CJcrOIN7M$S>vEXqUy96y|Id6Iz^FR2<_0m5Tz(#kiyvXP4v` z#1tf!WGEEmB s9}Y$iZVp8bUJm{uMj+Qulj#CazViFL|P%=)|P?7V*wjF`>n& zMa3~5P(@KdMKLa^#n~nK1u+GQB^e3@If;3BApMCs>G?&OB^kNJF_ZNeC8T3=^Gh=G z^Wt;!Q&MyE3My}L#K&jmWtPOpb8Sv!>}O)+pZt-zUzQVSVKEDk;9%rnEn)&PG?{O) Y$H%ASC&$N6p2DKR%L-G$!o2e%u&ql3@J=0%qlgWSB3mA1b=W+-!F}h7|;JU_Wu{oT(mXR@P z@>QNWj6s`=cn>o&W=%HZpT&4<@jh-wSb@GP<^vKOj2vtnd>q0Y e+#FIId__z^Ax+lFhXg!$89_=|fCLXC2QvU)}H33;(CL#61}F zK`TI$erV^VBAeWm|AP{RJcH=Jfyw{@N+1nQAsNjfZOrB3!-QpFuVz_vFj3aTNE>G$ I@G>C&2cutXO#lD@ diff --git a/Stochastic Shortest Path/env.py b/Stochastic Shortest Path/env.py index 7529f24..fb4fa89 100644 --- a/Stochastic Shortest Path/env.py +++ b/Stochastic Shortest Path/env.py @@ -1,12 +1,6 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - -class Env(): +class Env: def __init__(self, xI, xG): - self.x_range = 51 # size of background + self.x_range = 51 # size of background self.y_range = 31 self.motions = [(1, 0), (-1, 0), (0, 1), (0, -1)] self.xI = xI @@ -15,7 +9,6 @@ class Env(): self.lose = self.lose_map() self.stateSpace = self.state_space() - def obs_map(self): """ Initialize obstacles' positions @@ -48,7 +41,6 @@ class Env(): return obs - def lose_map(self): """ Initialize losing states' positions @@ -61,7 +53,6 @@ class Env(): return lose - def state_space(self): """ generate state space @@ -76,7 +67,6 @@ class Env(): return state_space - def get_reward(self, x_next): """ calculate reward of next state @@ -88,10 +78,10 @@ class Env(): reward = [] for x in x_next: if x in self.xG: - reward.append(10) # reward : 10, for goal states + reward.append(10) # reward : 10, for goal states elif x in self.lose: - reward.append(-10) # reward : -10, for lose states + reward.append(-10) # reward : -10, for lose states else: - reward.append(0) # reward : 0, for other states + reward.append(0) # reward : 0, for other states return reward diff --git a/Stochastic Shortest Path/motion_model.py b/Stochastic Shortest Path/motion_model.py index a1d6372..0285570 100644 --- a/Stochastic Shortest Path/motion_model.py +++ b/Stochastic Shortest Path/motion_model.py @@ -1,17 +1,11 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env + class Motion_model(): def __init__(self, xI, xG): self.env = env.Env(xI, xG) self.obs = self.env.obs_map() - def move_next(self, x, u, eta=0.2): """ Motion model of robots, @@ -41,4 +35,4 @@ class Motion_model(): else: x_next.append(x_check) - return x_next, p_next \ No newline at end of file + return x_next, p_next diff --git a/Stochastic Shortest Path/plotting.py b/Stochastic Shortest Path/plotting.py index 558b7d8..05fdc34 100644 --- a/Stochastic Shortest Path/plotting.py +++ b/Stochastic Shortest Path/plotting.py @@ -1,12 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import matplotlib.pyplot as plt import env + class Plotting(): def __init__(self, xI, xG): self.xI, self.xG = xI, xG @@ -14,7 +9,6 @@ class Plotting(): self.obs = self.env.obs_map() self.lose = self.env.lose_map() - def animation(self, path, name): """ animation. @@ -29,7 +23,6 @@ class Plotting(): self.plot_lose() self.plot_path(path) - def plot_grid(self, name): """ plot the obstacles in environment. @@ -49,7 +42,6 @@ class Plotting(): plt.title(name) plt.axis("equal") - def plot_lose(self): """ plot losing states in environment. @@ -59,8 +51,7 @@ class Plotting(): lose_x = [self.lose[i][0] for i in range(len(self.lose))] lose_y = [self.lose[i][1] for i in range(len(self.lose))] - plt.plot(lose_x, lose_y, color = '#A52A2A', marker = 's') - + plt.plot(lose_x, lose_y, color='#A52A2A', marker='s') def plot_visited(self, visited): """ @@ -88,7 +79,6 @@ class Plotting(): if count % length == 0: plt.pause(0.001) - def plot_path(self, path): path.remove(self.xI) for x in self.xG: @@ -103,7 +93,6 @@ class Plotting(): plt.show() plt.pause(0.5) - def plot_diff(self, diff, name): plt.figure(2) plt.title(name, fontdict=None) diff --git a/Stochastic Shortest Path/policy_iteration.py b/Stochastic Shortest Path/policy_iteration.py index bf1f478..3c482c3 100644 --- a/Stochastic Shortest Path/policy_iteration.py +++ b/Stochastic Shortest Path/policy_iteration.py @@ -1,9 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model @@ -12,20 +6,21 @@ import numpy as np import sys import copy + class Policy_iteration: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.e = 0.001 # threshold for convergence - self.gamma = 0.9 # discount factor + self.e = 0.001 # threshold for convergence + self.gamma = 0.9 # discount factor self.env = env.Env(self.xI, self.xG) self.motion = motion_model.Motion_model(self.xI, self.xG) self.plotting = plotting.Plotting(self.xI, self.xG) - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "policy_iteration, gamma=" + str(self.gamma) @@ -33,7 +28,6 @@ class Policy_iteration: self.path = self.extract_path(self.xI, self.xG, self.policy) self.plotting.animation(self.path, self.name1) - def policy_evaluation(self, policy, value): """ Evaluate current policy. @@ -45,7 +39,7 @@ class Policy_iteration: delta = sys.maxsize - while delta > self.e: # convergence condition + while delta > self.e: # convergence condition x_value = 0 for x in self.stateSpace: if x not in self.xG: @@ -59,7 +53,6 @@ class Policy_iteration: return value - def policy_improvement(self, policy, value): """ Improve policy using current value table. @@ -79,7 +72,6 @@ class Policy_iteration: return policy - def iteration(self): """ polity iteration: using evaluate and improvement process until convergence. @@ -91,21 +83,20 @@ class Policy_iteration: count = 0 for x in self.stateSpace: - value_table[x] = 0 # initialize value table - policy[x] = self.u_set[0] # initialize policy table + value_table[x] = 0 # initialize value table + policy[x] = self.u_set[0] # initialize policy table while True: count += 1 policy_back = copy.deepcopy(policy) - value_table = self.policy_evaluation(policy, value_table) # evaluation process - policy = self.policy_improvement(policy, value_table) # policy improvement process - if policy_back == policy: break # convergence condition + value_table = self.policy_evaluation(policy, value_table) # evaluation process + policy = self.policy_improvement(policy, value_table) # policy improvement process + if policy_back == policy: break # convergence condition self.message(count) return value_table, policy - def cal_Q_value(self, x, p, table): """ cal Q_value. @@ -117,13 +108,12 @@ class Policy_iteration: """ value = 0 - reward = self.env.get_reward(x) # get reward of next state + reward = self.env.get_reward(x) # get reward of next state for i in range(len(x)): - value += p[i] * (reward[i] + self.gamma * table[x[i]]) # cal Q-value + value += p[i] * (reward[i] + self.gamma * table[x[i]]) # cal Q-value return value - def extract_path(self, xI, xG, policy): """ extract path from converged policy. @@ -146,7 +136,6 @@ class Policy_iteration: x = x_next return path - def message(self, count): """ print important message. diff --git a/Stochastic Shortest Path/value_iteration.py b/Stochastic Shortest Path/value_iteration.py index 82dffc8..932e4a2 100644 --- a/Stochastic Shortest Path/value_iteration.py +++ b/Stochastic Shortest Path/value_iteration.py @@ -1,9 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -@author: huiming zhou -""" - import env import plotting import motion_model @@ -11,20 +5,21 @@ import motion_model import numpy as np import sys + class Value_iteration: def __init__(self, x_start, x_goal): self.xI, self.xG = x_start, x_goal - self.e = 0.001 # threshold for convergence - self.gamma = 0.9 # discount factor + self.e = 0.001 # threshold for convergence + self.gamma = 0.9 # discount factor - self.env = env.Env(self.xI, self.xG) # class Env - self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model - self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting + self.env = env.Env(self.xI, self.xG) # class Env + self.motion = motion_model.Motion_model(self.xI, self.xG) # class Motion_model + self.plotting = plotting.Plotting(self.xI, self.xG) # class Plotting - self.u_set = self.env.motions # feasible input set - self.stateSpace = self.env.stateSpace # state space - self.obs = self.env.obs_map() # position of obstacles - self.lose = self.env.lose_map() # position of lose states + self.u_set = self.env.motions # feasible input set + self.stateSpace = self.env.stateSpace # state space + self.obs = self.env.obs_map() # position of obstacles + self.lose = self.env.lose_map() # position of lose states self.name1 = "value_iteration, gamma=" + str(self.gamma) self.name2 = "converge process, e=" + str(self.e) @@ -34,7 +29,6 @@ class Value_iteration: self.plotting.animation(self.path, self.name1) self.plotting.plot_diff(self.diff, self.name2) - def iteration(self, xI, xG): """ value_iteration. @@ -42,36 +36,35 @@ class Value_iteration: :return: converged value table, optimal policy and variation of difference, """ - value_table = {} # value table - policy = {} # policy - diff = [] # maximum difference between two successive iteration - delta = sys.maxsize # initialize maximum difference - count = 0 # iteration times + value_table = {} # value table + policy = {} # policy + diff = [] # maximum difference between two successive iteration + delta = sys.maxsize # initialize maximum difference + count = 0 # iteration times - for x in self.stateSpace: # initialize value table for feasible states + for x in self.stateSpace: # initialize value table for feasible states value_table[x] = 0 - while delta > self.e: # converged condition + while delta > self.e: # converged condition count += 1 x_value = 0 for x in self.stateSpace: if x not in xG: value_list = [] for u in self.u_set: - [x_next, p_next] = self.motion.move_next(x, u) # recall motion model - value_list.append(self.cal_Q_value(x_next, p_next, value_table)) # cal Q value - policy[x] = self.u_set[int(np.argmax(value_list))] # update policy - v_diff = abs(value_table[x] - max(value_list)) # maximum difference - value_table[x] = max(value_list) # update value table + [x_next, p_next] = self.motion.move_next(x, u) # recall motion model + value_list.append(self.cal_Q_value(x_next, p_next, value_table)) # cal Q value + policy[x] = self.u_set[int(np.argmax(value_list))] # update policy + v_diff = abs(value_table[x] - max(value_list)) # maximum difference + value_table[x] = max(value_list) # update value table x_value = max(x_value, v_diff) - delta = x_value # update delta + delta = x_value # update delta diff.append(delta) - self.message(count) # print messages + self.message(count) # print messages return value_table, policy, diff - def cal_Q_value(self, x, p, table): """ cal Q_value. @@ -83,13 +76,12 @@ class Value_iteration: """ value = 0 - reward = self.env.get_reward(x) # get reward of next state + reward = self.env.get_reward(x) # get reward of next state for i in range(len(x)): - value += p[i] * (reward[i] + self.gamma * table[x[i]]) # cal Q-value + value += p[i] * (reward[i] + self.gamma * table[x[i]]) # cal Q-value return value - def extract_path(self, xI, xG, policy): """ extract path from converged policy. @@ -112,7 +104,6 @@ class Value_iteration: x = x_next return path - def message(self, count): """ print important message. @@ -129,7 +120,7 @@ class Value_iteration: if __name__ == '__main__': - x_Start = (5, 5) # starting state - x_Goal = [(49, 5), (49, 25)] # goal states + x_Start = (5, 5) # starting state + x_Goal = [(49, 5), (49, 25)] # goal states VI = Value_iteration(x_Start, x_Goal)