Merge pull request #626 from FinlaySanders/5c-nethack

5c nethack

Author: jsuarez5341

config/default.ini

@@ -107,6 +107,11 @@ vtrace_c_clip = 1.0
 prio_alpha = 0.8
 prio_beta0 = 0.2
 
+# Verb-eps exploration floor (experimental, Nethack; see src/algo.cu). 0 = off.
+verb_eps = 0.0
+verb_eps_anneal_start = 0.4
+verb_eps_anneal_end = 1.0
+
 [sweep]
 metric = score
 metric_distribution = linear

config/nethack.ini

@@ -1,130 +1,114 @@
 [base]
 env_name = nethack
 reset_every_horizon = 0
-async = 1
+async = 0
 
 [vec]
-total_agents = 256
+total_agents = 512
 num_buffers = 4
-#num_threads = 8
 num_threads = 32
-action_mask_size = 690
+action_mask_size = 734
 
 [env]
-gold_coef = 1
-exp_coef = 0.0222526733
-descent_coef = 4.57267237
-xp_coef = 0.648124218
-scout_coef = 0.1
-hp_coef = 0.00617194036
-hunger_coef = 0.01
-illegal_penalty = -0.01
-death_penalty = -0.112676442
+gold_coef = 0.021659979757862095
+exp_coef = 0.08341832601491733
+descent_coef = 1.0
+xp_coef = 0.5926087261188399
+scout_coef = 0.025
 ac_coef = 1.0
-heal_coef = 0.001
-status_coef = 0.0116204191
+death_penalty = -0.8209825710553373
+
+# flags
+# scout reward scales with xp level relative to depth; <= 0 disables
+scout_ready = 1.0
+# fraction of protection-spell AC excluded from the ac reward; 1 = armor only
+ac_nospell = 1.0
+# 1 removes the verb from the action space
+mask_search20 = 0
+mask_run = 0
 
 [policy]
 hidden_size = 512
 num_layers = 3
 
 [train]
-#total_timesteps = 166275056
-total_timesteps = 4_000_000_000
-minibatch_size = 16384
-horizon = 512
-learning_rate = 0.00175767997
-gamma = 0.993521929
-gae_lambda = 0.611256123
-replay_ratio = 2.42882919
-clip_coef = 0.478926629
-vf_coef = 2.29309034
-vf_clip_coef = 3.19970345
-max_grad_norm = 1.43588173
-ent_coef = 0.0192690529
-beta1 = 0.9694917707305044
-beta2 = 0.9475724955192064
-eps = 0.0001
-prio_alpha = 0.931300879
-prio_beta0 = 0.885229468
+total_timesteps = 247765094
+minibatch_size = 32768
+horizon = 64
+learning_rate = 0.011854702184553156
+momentum = 0.805399553172738
+verb_eps = 0.01
+verb_eps_anneal_start = 0.4
+verb_eps_anneal_end = 1.0
+gamma = 0.9973714278257158
+gae_lambda = 0.7339946788354172
+replay_ratio = 1.036717410123433
+clip_coef = 1.0
+vf_coef = 1.8486010633248169
+vf_clip_coef = 4.624607731543118
+max_grad_norm = 0.902165688849037
+ent_coef = 0.008389210353641681
+beta1 = 0.805399553172738
+beta2 = 0.999139492311758
+eps = 1e-14
+vtrace_rho_clip = 2.770648671516198
+vtrace_c_clip = 4.5026502564349595
+prio_alpha = 0.6754855924309153
+prio_beta0 = 0.8672463883417074
 
 [sweep]
-metric = max_depth
+metric = score
 
 [sweep.train.total_timesteps]
 distribution = log_normal
 min = 1.5e8
 max = 4e8
 scale = time
 
-[sweep.env.gold_coef]
+[sweep.env.exp_coef]
 distribution = log_normal
-min = 0.001
-max = 1.0
+min = 0.01
+max = 0.1
 scale = auto
 
-[sweep.env.exp_coef]
+[sweep.env.gold_coef]
 distribution = log_normal
-min = 0.01
-max = 10.0
+min = 0.001
+max = 0.1
 scale = auto
 
 [sweep.env.descent_coef]
-distribution = log_normal
-min = 0.5
-max = 10.0
+distribution = uniform
+min = 0.0
+max = 1.0
 scale = auto
 
 [sweep.env.xp_coef]
-distribution = log_normal
-min = 0.05
+distribution = uniform
+min = 0.0
 max = 1.0
 scale = auto
 
 [sweep.env.scout_coef]
 distribution = log_normal
-min = 0.0001
+min = 0.001
 max = 0.1
 scale = auto
 
-[sweep.env.hp_coef]
-distribution = log_normal
-min = 0.0005
-max = 0.06
-scale = auto
-
-[sweep.env.hunger_coef]
-distribution = log_normal
-min = 0.01
-max = 0.25
-scale = auto
-
-[sweep.env.illegal_penalty]
-distribution = uniform
-min = -0.01
-max = 0.0
-scale = auto
-
 [sweep.env.death_penalty]
 distribution = uniform
 min = -1.0
 max = 0.0
 scale = auto
 
-[sweep.env.heal_coef]
-distribution = log_normal
-min = 0.001
-max = 0.1
-scale = auto
-
-[sweep.env.status_coef]
-distribution = log_normal
-min = 0.01
-max = 1.0
-scale = auto
-
-[sweep.env.ac_coef]
-distribution = log_normal
-min = 0.01
-max = 0.5
-scale = auto
+#[sweep.train.verb_eps]
+#distribution = log_normal
+#min = 0.0005
+#max = 0.02
+#scale = auto
+
+#[sweep.train.verb_eps_anneal_start]
+#distribution = uniform
+#min = 0.2
+#max = 0.8
+#scale = auto

ocean/nethack/README.md

@@ -1,8 +1,8 @@
 # NetHack
 
 PufferLib environment for NetHack 3.6.6 over
-[fast-nle](https://github.com/FinlaySanders/fast-nle): 22-verb factored
-action space (verb, item slot, direction), legality masking,
+[fast-nle](https://github.com/FinlaySanders/fast-nle): 26-verb factored
+action space (verb, item slot, direction, spell slot), legality masking,
 decomposed-score reward, custom CUDA encoder/decoder (`ocean/nethack/nethack.cu`).
 
 ## Setup
@@ -39,11 +39,8 @@ Interactive controls (default when stdin is a TTY):
 | `Shift+Space` | hold advances at **20 Hz** (fallback: hold `S`) |
 | `q` / `Esc` | quit |
 
-Weight resolution: `NH_WEIGHTS` if set, else the highest-step checkpoint in
-the **most recently written** run under `checkpoints/nethack/`, else
-`resources/nethack/nethack_weights.bin`. (Global max-step is wrong after a
-role switch — a mature monk 2B run outranks a fresh valk 200M run.) Set
-`NH_SEED` to replay a seed.
+Weight resolution: `NH_WEIGHTS` if set, else the checked-in
+`resources/nethack/nethack_weights.bin`. Set `NH_SEED` to replay a seed.
 
 ```bash
 ./nethack 10000 0            # headless 10k steps (prints avg_score / avg_max_depth)

ocean/nethack/depth.ini

@@ -0,0 +1,64 @@
+# async depth config: sweep-derived depth-seeking rewards (descent-dominated,
+# soft death penalty) on the async trainer. To use: copy to config/depth.ini,
+# then ./puffer train depth
+[base]
+env_name = nethack
+reset_every_horizon = 0
+async = 1
+
+[vec]
+total_agents = 256
+num_buffers = 4
+num_threads = 32
+action_mask_size = 734
+
+[env]
+gold_coef = 1.0
+exp_coef = 0.0222526733
+descent_coef = 4.57267237
+xp_coef = 0.648124218
+scout_coef = 0.1
+ac_coef = 0.0
+death_penalty = -0.112676442
+
+# flags
+# scout reward scales with xp level relative to depth; <= 0 disables
+scout_ready = 0.0
+# fraction of protection-spell AC excluded from the ac reward; 1 = armor only
+ac_nospell = 1.0
+# 1 removes the verb from the action space
+mask_search20 = 0
+mask_run = 0
+
+[policy]
+hidden_size = 512
+num_layers = 3
+
+[train]
+total_timesteps = 247765094
+minibatch_size = 16384
+horizon = 512
+learning_rate = 0.00175767997
+anneal_lr = 1
+min_lr_ratio = 0.0
+momentum = 0.9694917707305044
+verb_eps = 0.005
+verb_eps_anneal_start = 0.4
+verb_eps_anneal_end = 1.0
+gamma = 0.993521929
+gae_lambda = 0.611256123
+replay_ratio = 2.42882919
+clip_coef = 0.478926629
+vf_coef = 2.29309034
+vf_clip_coef = 3.19970345
+max_grad_norm = 1.43588173
+ent_coef = 0.0192690529
+anneal_ent_coef = 0
+min_ent_coef_ratio = 0.1
+beta1 = 0.9694917707305044
+beta2 = 0.9475724955192064
+eps = 0.0001
+vtrace_rho_clip = 1.0
+vtrace_c_clip = 1.0
+prio_alpha = 0.931300879
+prio_beta0 = 0.885229468

ocean/nethack/fs.h

@@ -19,7 +19,7 @@ static void nethack_touch(const char* path) {
 }
 
 static void nethack_rm_rf(const char* path, int depth) {
-    if (depth > 3) return;   // vardir trees are at most base/env/save/files
+    if (depth > 3) return; // vardir trees are at most base/env/save/files
     DIR* d = opendir(path);
     if (d) {
         struct dirent* e;
@@ -82,7 +82,7 @@ static int nethack_make_vardir(const char* source_hackdir, char* out_buf, size_t
 
     // fail fast on a dangling nhdat symlink: symlink(2) would succeed and the
     // error surface later as a cryptic init_dungeons panic
-    char resolved[4096];   // realpath(3) requires a PATH_MAX buffer
+    char resolved[4096]; // realpath(3) requires a PATH_MAX buffer
     if (realpath(src, resolved) == NULL || access(resolved, R_OK) != 0) {
         fprintf(stderr,
                 "nethack: NETHACKDIR misconfigured — no readable nhdat at %s (%s).\n"