Merge pull request #621 from FinlaySanders/nethack

Merging this here, will pull it on to 5c and test there

Author: jsuarez5341

.gitignore

@@ -172,3 +172,5 @@ pufferlib/ocean/impulse_wars/benchmark/
 resources/drive/data/*
 resources/drive/binaries/*
 
+vendor/nle/
+vendor/fast-nle/

build.sh

@@ -119,18 +119,20 @@ elif [ "$ENV" = "impulse_wars" ]; then
     LINK_ARCHIVES+=("./$BOX2D_NAME/libbox2d.a")
 elif [ "$ENV" = "nethack" ]; then
     SRC_DIR="ocean/$ENV"
-    NLE_DIR="vendor/nle"
-    NLE_REPO="https://github.com/liujonathan24/NetHack.git"
+    NLE_DIR="vendor/fast-nle"
+    NLE_REPO="https://github.com/FinlaySanders/fast-nle.git"
     if [ ! -d "$NLE_DIR/src" ]; then
-        echo "Cloning modified NLE from $NLE_REPO ..."
+        echo "Cloning fast-nle from $NLE_REPO ..."
         git clone --depth 1 "$NLE_REPO" "$NLE_DIR"
     fi
-    NETHACK_LIB_DIR="$(pwd)/$NLE_DIR/src/build"
+    NETHACK_LIB_DIR="$(pwd)/$NLE_DIR/build"
     if [ ! -f "$NETHACK_LIB_DIR/libnethack.so" ]; then
         echo "Building libnethack.so ..."
-        make -C "$NETHACK_LIB_DIR" nethack -j$(nproc)
+        cmake -S "$NLE_DIR" -B "$NETHACK_LIB_DIR" -DCMAKE_BUILD_TYPE=Release
+        cmake --build "$NETHACK_LIB_DIR" --target nethack -j$(nproc)
     fi
-    INCLUDES+=(-I./$NLE_DIR/include)
+    INCLUDES+=(-I./$NLE_DIR/include
+               -I./$NLE_DIR/build/_deps/deboost_context-src/include)
     EXTRA_LDFLAGS+=(-L"$NETHACK_LIB_DIR" -lnethack -Wl,-rpath,"$NETHACK_LIB_DIR" -ldl)
 elif [ -d "ocean/$ENV" ]; then
     SRC_DIR="ocean/$ENV"

config/nethack.ini

@@ -1,120 +1,129 @@
 [base]
 env_name = nethack
+cudagraphs = 2
 
 [vec]
-total_agents = 4096
-num_buffers = 2
-num_threads = 16
-seed = 73
+total_agents = 512
+num_buffers = 4
+num_threads = 32
 
 [env]
-# Reward shaping. Each coef is independently toggleable: set to 0 to disable
-# the term. See ocean/nethack/REWARDS.md for the recipe + future candidates.
-# CLI: --env.score-coef, --env.descent-coef, --env.scout-coef, --env.illegal-penalty
-score_coef = 0.01
-descent_coef = 10.0
-scout_coef = 0.1
-illegal_penalty = -0.1
+gold_coef = 0.03540707780083026
+exp_coef = 0.25886063598802184
+descent_coef = 1.8884272008758112
+xp_coef = 0.28073502137812123
+scout_coef = 0.01689042846824247
+hp_coef = 0.018973887012487405
+hunger_coef = 0.023586377138546992
+illegal_penalty = -0.006951422236819335
+death_penalty = -0.6335992931154887
+ac_coef = 0.01
+heal_coef = 0.001
+status_coef = 0.01
 
 [policy]
 hidden_size = 512
-num_layers = 4
-expansion_factor = 2
+num_layers = 3
+expansion_factor = 1
 
 [train]
-gpus = 1
-seed = 42
-total_timesteps = 1443360107
-learning_rate = 0.00224872
-anneal_lr = 1
-min_lr_ratio = 0
-gamma = 0.99816
-gae_lambda = 0.696214
-replay_ratio = 2.2381
-clip_coef = 0.20
-vf_coef = 0.1
-vf_clip_coef = 0.01
-max_grad_norm = 1.14364
-ent_coef = 0.10
-beta1 = 0.981226
-beta2 = 0.994178
-eps = 1.82223e-06
+total_timesteps = 179_453_112
 minibatch_size = 32768
-horizon = 256
-vtrace_rho_clip = 3.24229
-vtrace_c_clip = 3.95664
-prio_alpha = 1
-prio_beta0 = 0.555562
-env = 0
+horizon = 64
+learning_rate = 0.013102893141695623
+gamma = 0.995330546979216
+gae_lambda = 0.7296684290473575
+replay_ratio = 1.393745327111498
+clip_coef = 1
+vf_coef = 2.967784385898158
+vf_clip_coef = 4.084322349420867
+max_grad_norm = 1.0968230967915535
+ent_coef = 0.005217403062762164
+beta1 = 0.8250073190394559
+beta2 = 0.9935894811507382
+eps = 3.82673948478513e-13
+vtrace_rho_clip = 3.8767312776569813
+vtrace_c_clip = 4.60481789092722
+prio_alpha = 0.7459213056962047
+prio_beta0 = 0.7387749352151263
 
 [sweep]
-method = Random
-metric = depth
-goal = maximize
-downsample = 4
-max_runs = 2
-gpus = 1
-max_suggestion_cost = 1500
-
-[sweep.env.score_coef]
+metric = score
+
+[sweep.train.total_timesteps]
+distribution = log_normal
+min = 1.5e8
+max = 4e8
+scale = time
+
+[sweep.env.gold_coef]
 distribution = log_normal
 min = 0.001
-mean = 0.01
-max = 0.1
+max = 1.0
+scale = auto
+
+[sweep.env.exp_coef]
+distribution = log_normal
+min = 0.01
+max = 10.0
 scale = auto
 
 [sweep.env.descent_coef]
-distribution = uniform
-min = 0.0
-mean = 10.0
-max = 50.0
+distribution = log_normal
+min = 0.5
+max = 10.0
+scale = auto
+
+[sweep.env.xp_coef]
+distribution = log_normal
+min = 0.05
+max = 1.0
 scale = auto
 
 [sweep.env.scout_coef]
-distribution = uniform
-min = 0.0
-mean = 0.1
-max = 2.0
+distribution = log_normal
+min = 0.0001
+max = 0.1
 scale = auto
 
-[sweep.env.illegal_penalty]
-distribution = uniform
-min = -2.0
-mean = -0.5
-max = 0.0
+[sweep.env.hp_coef]
+distribution = log_normal
+min = 0.0005
+max = 0.06
 scale = auto
 
-[sweep.train.learning_rate]
+[sweep.env.hunger_coef]
 distribution = log_normal
-min = 0.0001
-mean = 0.002
-max = 0.01
+min = 0.01
+max = 0.25
 scale = auto
 
-[sweep.train.gamma]
+[sweep.env.illegal_penalty]
 distribution = uniform
-min = 0.99
-mean = 0.998
-max = 0.9999
+min = -0.01
+max = 0.0
 scale = auto
 
-[sweep.train.gae_lambda]
+[sweep.env.death_penalty]
 distribution = uniform
-min = 0.5
-mean = 0.7
-max = 0.99
+min = -1.0
+max = 0.0
 scale = auto
 
-[sweep.train.ent_coef]
+[sweep.env.heal_coef]
 distribution = log_normal
 min = 0.001
-mean = 0.1
-max = 0.5
+max = 0.1
 scale = auto
 
-[sweep.train.max_grad_norm]
-distribution = uniform
-min = 0.1
-mean = 1.0
-max = 3.0
+[sweep.env.status_coef]
+distribution = log_normal
+min = 0.01
+max = 1.0
+scale = auto
+
+[sweep.env.ac_coef]
+distribution = log_normal
+min = 0.01
+max = 0.5
 scale = auto