-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtrain_teacher.py
123 lines (94 loc) · 4.42 KB
/
train_teacher.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# python3 train_teacher.py ; python3 train_student.py
"""
Last good train run : train it with high kp (60) first. Then wih low kp (45). Then student (with low kp)
Mabe unnecessary.
"""
from environments.dog_env import DogEnv
from config import Config
import time
import numpy as np
import gym
from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv
from stable_baselines3.common.monitor import Monitor
from stable_baselines3.common.env_checker import check_env
from stable_baselines3.common.callbacks import BaseCallback
from stable_baselines3.common.callbacks import CallbackList
from stable_baselines3.common.policies import ActorCriticPolicy
from stable_baselines3.common import utils
from stable_baselines3.common.utils import get_schedule_fn
import torch as th
class CustomCallback(BaseCallback):
def __init__(self, config, init_log_std=-1, verbose=0):
super(CustomCallback, self).__init__(verbose)
# Those variables will be accessible in the callback
# (they are defined in the base class)
# The RL model
# self.model = None # type: BaseAlgorithm
# An alias for self.model.get_env(), the environment used for training
# self.training_env = None # type: Union[gym.Env, VecEnv, None]
# Number of time the callback was called
# self.n_calls = 0 # type: int
# self.num_timesteps = 0 # type: int
# local and global variables
# self.locals = None # type: Dict[str, Any]
# self.globals = None # type: Dict[str, Any]
# The logger object, used to report things in the terminal
# self.logger = None # stable_baselines3.common.logger
# # Sometimes, for event callback, it is useful
# # to have access to the parent object
# self.parent = None # type: Optional[BaseCallback]
self.config = config
self.init_log_std = init_log_std
def _on_training_start(self) -> None:
th.nn.Parameter(self.model.policy.log_std*0 + self.init_log_std)
def _on_rollout_start(self) -> None:
pass
def _on_step(self) -> bool:
return True
def _on_training_end(self) -> None:
pass
def _on_rollout_end(self) -> None:
self.model.save(config.models_save_path["teacher/PPO"].format(epoch=self.num_timesteps), exclude=["default_env"])
# self.model.save(config.models_save_path["teacher/actor"].format(epoch=self.num_timesteps))
# self.model.batch_size = 128 # self.num_timesteps
decay = 700000
alpha = 1 if self.num_timesteps > decay else self.num_timesteps/decay
des_log_std = (self.init_log_std) * (1-alpha) + (-3) * alpha
self.model.policy.log_std = th.nn.Parameter(self.model.policy.log_std*0 + des_log_std)
from my_ppo import MyPPO, TeacherActorCriticPolicy
from environments.dog_env import DogEnv_follow
from my_ppo import MyPPO, MotorActorCriticPolicy
from config import Config
def create_dog_env ():
config = Config("follow_0", models_names=["follower/PPO"])
env = DogEnv_follow(debug=False)
motor_model = MyPPO.load(config.models_best_path["follower/PPO"], env=env, policy=MotorActorCriticPolicy)
return Monitor(DogEnv(motor_model=motor_model))
continual_learning = False
if __name__ == "__main__":
if False:
check_env(DogEnv(debug=False))
print("working !!")
exit()
# config = Config("follow_0", models_names=["follower/PPO"])
# env = DogEnv_follow(debug=False)
# motor_model = MyPPO.load(config.models_best_path["follower/PPO"], env=env, policy=MotorActorCriticPolicy)
src_config = Config("exp_0_highkp", models_names=["teacher/PPO", "teacher/tensorboard"])
config = Config("exp_1", models_names=["teacher/PPO", "teacher/tensorboard"])
env = SubprocVecEnv([lambda : Monitor(DogEnv()) for i in range(2)])
# env = SubprocVecEnv([create_dog_env for i in range(2)])
default_env = DogEnv(debug=False)
init_log_std = -3 if continual_learning else -1
my_callback = CustomCallback(config, init_log_std=init_log_std)
callback = CallbackList([my_callback])
policy_kwargs = dict(
log_std_init=-1.,
)
if continual_learning:
model = MyPPO.load(src_config.models_best_path["teacher/PPO"], env=env, default_env=default_env, policy_kwargs=policy_kwargs, verbose=1, batch_size=1024, tensorboard_log=config.models_path["teacher/tensorboard"])
else:
model = MyPPO(TeacherActorCriticPolicy, env, default_env, policy_kwargs=policy_kwargs, verbose=1, batch_size=1024*2, tensorboard_log=config.models_path["teacher/tensorboard"])
model.learn(total_timesteps=13000000, callback=callback)
# model.learn(total_timesteps=1000, callback=callback)
print("working !!")
# python3 train_teacher.py ; python3 train_student.py