Update dependencies (#209)

* Updating minor packages

* Updating pytorch

* Updating numpy

Author: Federico-PizarroBejarano

.pre-commit-config.yaml

@@ -9,7 +9,7 @@
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     -   id: check-ast
     -   id: check-yaml
@@ -26,7 +26,7 @@ repos:
     -   id: double-quote-string-fixer
 
 -   repo: https://github.com/pycqa/isort
-    rev: 6.0.1
+    rev: 6.1.0
     hooks:
       - id: isort
         name: isort

examples/rl/train_rl_model.sh

@@ -4,8 +4,8 @@ SYS='cartpole'
 # SYS='quadrotor_2D'
 # SYS='quadrotor_3D'
 
-TASK='stab'
-# TASK='track'
+# TASK='stab'
+TASK='track'
 
 ALGO='ppo'
 # ALGO='sac'

pyproject.toml

@@ -7,27 +7,27 @@ license = "MIT"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-matplotlib = "^3.5.1"
-munch = "^2.5.0"
+matplotlib = "^3.10"
+munch = "^4.0"
 PyYAML = "^6.0"
-imageio = "^2.14.1"
-dict-deep = "^4.1.2"
-scikit-optimize = "^0.9.0"
-scikit-learn = "^1.3.0"
+imageio = "^2.37"
+dict-deep = "^4.1"
+scikit-optimize = "^0.10"
+scikit-learn = "^1.7"
 gymnasium = "^0.28"
-torch = "^1.10.2"
-gpytorch = "^1.6.0"
-tensorboard = "^2.12.0"
-casadi = "^3.6.0"
-pybullet = "^3.2.0"
-numpy = "^1.22.1"
-cvxpy = "^1.1.18"
-pycddlib = "^2.1.7"
+torch = "^2.8"
+gpytorch = "^1.14"
+tensorboard = "^2.20"
+casadi = "^3.7"
+pybullet = "^3.2"
+numpy = "^2.2"
+cvxpy = "^1.7"
+pycddlib = "^2.1"
 pytope = "^0.0.4"
-Mosek = "^10.0.18"
-termcolor = "^1.1.0"
-pytest = "^7.2.2"
-pre-commit = "^3.3.2"
+Mosek = "^11.0"
+termcolor = "^3.1"
+pytest = "^8.4"
+pre-commit = "^4.3"
 optuna = "^3.0"
 optuna-dashboard = "^0.9"
 mysql-connector-python = "8.0.33"
@@ -36,5 +36,5 @@ pymysql = "1.1.1"
 [tool.poetry.dev-dependencies]
 
 [build-system]
-requires = ["poetry-core @ git+https://github.com/python-poetry/poetry-core.git@main"]
+requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

safe_control_gym/controllers/base_controller.py

@@ -42,7 +42,7 @@ def __init__(self,
             self.__dict__[key] = value
 
         self.use_gpu = self.use_gpu and torch.cuda.is_available()
-        self.device = 'cpu' if self.use_gpu is False else 'cuda'
+        self.device = torch.device('cuda' if self.use_gpu and torch.cuda.is_available() else 'cpu')
 
         self.setup_results_dict()
 

safe_control_gym/controllers/ddpg/ddpg.py

@@ -142,7 +142,7 @@ def save(self, path, save_buffer=True):
 
     def load(self, path):
         '''Restores model and experiment given checkpoint path.'''
-        state = torch.load(path)
+        state = torch.load(path, weights_only=False)  # Safe since we're loading our own models
 
         # restore params
         self.agent.load_state_dict(state['agent'])
@@ -194,7 +194,7 @@ def learn(self, env=None, **kwargs):
                                                                                                             eval_results['ep_returns'].std()))
                 # save best model
                 eval_score = eval_results['ep_returns'].mean()
-                eval_best_score = getattr(self, 'eval_best_score', -np.infty)
+                eval_best_score = getattr(self, 'eval_best_score', -np.inf)
                 if self.eval_save_best and eval_best_score < eval_score:
                     self.eval_best_score = eval_score
                     self.save(os.path.join(self.output_dir, 'model_best.pt'))
@@ -214,7 +214,7 @@ def select_action(self, obs, info=None):
             action (ndarray): The action chosen by the controller.
         '''
 
-        with torch.no_grad():
+        with torch.inference_mode():
             obs = torch.FloatTensor(obs).to(self.device)
             action = self.agent.ac.act(obs)
 
@@ -278,7 +278,7 @@ def train_step(self, **kwargs):
         if self.total_steps < self.warm_up_steps:
             act = np.stack([self.env.action_space.sample() for _ in range(self.rollout_batch_size)])
         else:
-            with torch.no_grad():
+            with torch.inference_mode():
                 act = self.agent.ac.act(torch.FloatTensor(obs).to(self.device))
                 # apply action noise if specified in training config
                 if self.noise_process:

safe_control_gym/controllers/ddpg/ddpg_utils.py

@@ -172,7 +172,7 @@ def unscale_fn(x):  # Rescale action from [-1, 1] to [low, high]
 
     def act(self, obs, **kwargs):
         a = self.actor(obs)
-        return a.cpu().numpy()
+        return a.cpu().numpy().astype(np.float32)
 
 
 # -----------------------------------------------------------------------------------

safe_control_gym/controllers/mpc/gp_utils.py

@@ -670,11 +670,11 @@ def init_with_hyperparam(self,
         if self.target_mask is not None:
             train_targets = train_targets[:, self.target_mask]
         device = torch.device('cpu')
-        state_dict = torch.load(path_to_statedict, map_location=device)
+        state_dict = torch.load(path_to_statedict, map_location=device, _use_new_zipfile_serialization=True)
         self._init_model(train_inputs, train_targets)
 
         self.model.load_state_dict(state_dict)
-        self.model.double()  # needed otherwise loads state_dict as float32
+        self.model = self.model.to(dtype=torch.float64)  # needed otherwise loads state_dict as float32
         self._compute_GP_covariances(train_inputs)
         self.casadi_predict = self.make_casadi_prediction_func(train_inputs, train_targets)
 
@@ -724,8 +724,8 @@ def train(self,
             test_y = test_y.cuda()
             self.model = self.model.cuda()
             self.likelihood = self.likelihood.cuda()
-        self.model.double()
-        self.likelihood.double()
+        self.model = self.model.to(dtype=torch.float64)
+        self.likelihood = self.likelihood.to(dtype=torch.float64)
         self.model.train()
         self.likelihood.train()
         self.optimizer = torch.optim.Adam(self.model.parameters(), lr=learning_rate)
@@ -735,7 +735,7 @@ def train(self,
         loss = torch.tensor(0)
         i = 0
         while i < n_train and torch.abs(loss - last_loss) > 1e-2:
-            with torch.no_grad():
+            with torch.inference_mode():
                 self.model.eval()
                 self.likelihood.eval()
                 test_output = self.model(test_x.unsqueeze(0).repeat(self.output_dimension, 1, 1))
@@ -754,7 +754,7 @@ def train(self,
             if test_loss < best_loss:
                 best_loss = test_loss
                 state_dict = self.model.state_dict()
-                torch.save(state_dict, fname)
+                torch.save(state_dict, fname, _use_new_zipfile_serialization=True)
                 best_epoch = i
 
             i += 1
@@ -765,7 +765,7 @@ def train(self,
         self.likelihood = self.likelihood.cpu()
         train_x = train_x.cpu()
         train_y = train_y.cpu()
-        self.model.load_state_dict(torch.load(fname))
+        self.model.load_state_dict(torch.load(fname, weights_only=False))
         self._compute_GP_covariances(train_x)
         self.casadi_predict = self.make_casadi_prediction_func(train_x, train_y)
 
@@ -791,7 +791,7 @@ def predict(self,
         self.model.eval()
         self.likelihood.eval()
         if isinstance(x, np.ndarray):
-            x = torch.from_numpy(x).double()
+            x = torch.tensor(x, dtype=torch.float64)
         if self.input_mask is not None:
             x = x[:, self.input_mask]
         if requires_grad:
@@ -986,12 +986,12 @@ def init_with_hyperparam(self,
         if self.target_mask is not None:
             train_targets = train_targets[:, self.target_mask]
         device = torch.device('cpu')
-        state_dict = torch.load(path_to_statedict, map_location=device)
+        state_dict = torch.load(path_to_statedict, map_location=device, _use_new_zipfile_serialization=True)
         self._init_model(train_inputs, train_targets)
         if self.NORMALIZE:
             train_inputs = torch.from_numpy(self.scaler.transform(train_inputs.numpy()))
         self.model.load_state_dict(state_dict)
-        self.model.double()  # needed otherwise loads state_dict as float32
+        self.model = self.model.to(dtype=torch.float64)  # needed otherwise loads state_dict as float32
         self._compute_GP_covariances(train_inputs)
         self.casadi_predict = self.make_casadi_prediction_func(train_inputs, train_targets)
 
@@ -1045,8 +1045,8 @@ def train(self,
             test_y = test_y.cuda()
             self.model = self.model.cuda()
             self.likelihood = self.likelihood.cuda()
-        self.model.double()
-        self.likelihood.double()
+        self.model = self.model.to(dtype=torch.float64)
+        self.likelihood = self.likelihood.to(dtype=torch.float64)
         self.model.train()
         self.likelihood.train()
         self.optimizer = torch.optim.Adam(self.model.parameters(), lr=learning_rate)
@@ -1056,7 +1056,7 @@ def train(self,
         loss = torch.tensor(0)
         i = 0
         while i < n_train and torch.abs(loss - last_loss) > 1e-2:
-            with torch.no_grad():
+            with torch.inference_mode():
                 self.model.eval()
                 self.likelihood.eval()
                 test_output = self.model(test_x)
@@ -1074,7 +1074,7 @@ def train(self,
             if test_loss < best_loss:
                 best_loss = test_loss
                 state_dict = self.model.state_dict()
-                torch.save(state_dict, fname)
+                torch.save(state_dict, fname, _use_new_zipfile_serialization=True)
                 best_epoch = i
 
             i += 1
@@ -1085,7 +1085,7 @@ def train(self,
         self.likelihood = self.likelihood.cpu()
         train_x = train_x.cpu()
         train_y = train_y.cpu()
-        self.model.load_state_dict(torch.load(fname))
+        self.model.load_state_dict(torch.load(fname, weights_only=False))
         self._compute_GP_covariances(train_x)
         self.casadi_predict = self.make_casadi_prediction_func(train_x, train_y)
 
@@ -1109,7 +1109,7 @@ def predict(self,
         self.model.eval()
         self.likelihood.eval()
         if isinstance(x, np.ndarray):
-            x = torch.from_numpy(x).double()
+            x = torch.tensor(x, dtype=torch.float64)
         if self.input_mask is not None:
             x = x[:, self.input_mask]
         if self.NORMALIZE:

safe_control_gym/controllers/ppo/ppo.py

@@ -134,7 +134,7 @@ def load(self,
              path
              ):
         '''Restores model and experiment given checkpoint path.'''
-        state = torch.load(path)
+        state = torch.load(path, weights_only=False)  # Safe since we're loading our own models
         # Restore policy.
         self.agent.load_state_dict(state['agent'])
         self.obs_normalizer.load_state_dict(state['obs_normalizer'])
@@ -182,7 +182,7 @@ def learn(self,
                                                                                                             eval_results['ep_returns'].std()))
                 # Save best model.
                 eval_score = eval_results['ep_returns'].mean()
-                eval_best_score = getattr(self, 'eval_best_score', -np.infty)
+                eval_best_score = getattr(self, 'eval_best_score', -np.inf)
                 if self.eval_save_best and eval_best_score < eval_score:
                     self.eval_best_score = eval_score
                     self.save(os.path.join(self.output_dir, 'model_best.pt'))
@@ -201,7 +201,7 @@ def select_action(self, obs, info=None):
             action (ndarray): The action chosen by the controller.
         '''
 
-        with torch.no_grad():
+        with torch.inference_mode():
             obs = torch.FloatTensor(obs).to(self.device)
             action = self.agent.ac.act(obs)
 
@@ -264,7 +264,7 @@ def train_step(self):
         obs = self.obs
         start = time.time()
         for _ in range(self.rollout_steps):
-            with torch.no_grad():
+            with torch.inference_mode():
                 act, v, logp = self.agent.ac.step(torch.FloatTensor(obs).to(self.device))
             next_obs, rew, done, info = self.env.step(act)
             next_obs = self.obs_normalizer(next_obs)

safe_control_gym/controllers/ppo/ppo_utils.py

@@ -235,7 +235,7 @@ def act(self,
             ):
         dist, _ = self.actor(obs)
         action = dist.mode()
-        return action.cpu().numpy()
+        return action.cpu().numpy().astype(np.float32)
 
 
 class PPOBuffer(object):
@@ -303,7 +303,7 @@ def reset(self):
             vshape = info['vshape']
             dtype = info.get('dtype', np.float32)
             init = info.get('init', np.zeros)
-            self.__dict__[k] = init(vshape, dtype=dtype)
+            self.__dict__[k] = init(vshape).astype(dtype)
         self.t = 0
 
     def push(self,

safe_control_gym/controllers/rarl/rap.py

@@ -143,7 +143,7 @@ def save(self, path):
 
     def load(self, path):
         '''Restores model and experiment given checkpoint path.'''
-        state = torch.load(path)
+        state = torch.load(path, weights_only=False)  # Safe since we're loading our own models
 
         # restore pllicy
         self.agent.load_state_dict(state['agent'])
@@ -185,7 +185,7 @@ def learn(self, env=None, **kwargs):
                                                                                                             eval_results['ep_returns'].std()))
                 # save best model
                 eval_score = eval_results['ep_returns'].mean()
-                eval_best_score = getattr(self, 'eval_best_score', -np.infty)
+                eval_best_score = getattr(self, 'eval_best_score', -np.inf)
                 if self.eval_save_best and eval_best_score < eval_score:
                     self.eval_best_score = eval_score
                     self.save(os.path.join(self.output_dir, 'model_best.pt'))
@@ -215,14 +215,14 @@ def run(self, env=None, render=False, n_episodes=10, verbose=False, use_adv=Fals
         frames = []
 
         while len(ep_returns) < n_episodes:
-            with torch.no_grad():
+            with torch.inference_mode():
                 obs = torch.FloatTensor(obs).to(self.device)
                 action = self.agent.ac.act(obs)
 
             # no disturbance during testing
             if use_adv:
                 adv_idx = np.random.choice(self.num_adversaries)
-                with torch.no_grad():
+                with torch.inference_mode():
                     action_adv = self.adversaries[adv_idx].ac.act(obs)
             else:
                 action_adv = np.zeros(self.adv_act_space.shape[0])
@@ -362,7 +362,7 @@ def collect_rollouts(self):
 
         for _ in range(self.rollout_steps):
             # get actions
-            with torch.no_grad():
+            with torch.inference_mode():
                 act, v, logp = self.agent.ac.step(torch.FloatTensor(obs).to(self.device))
 
                 # adversary actions