Route Phoenix case-opt pre-build through SLURM to avoid 4GB cgroup limit

Phoenix login nodes have a 4GB per-user cgroup memory limit that OOM-kills
case-optimized GPU builds (confirmed via dmesg: CONSTRAINT_MEMCG). Route
the pre-build through submit.sh on Phoenix so it runs on a compute node
with full memory. Frontier continues to pre-build on the login node.

Reverts retry/parallelism changes (max_attempts back to 3, -j back to 8)
since the root cause was the cgroup, not parallelism.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/prebuild-case-optimization.sh

@@ -1,14 +1,17 @@
 #!/bin/bash
 
-# Pre-builds all benchmark cases with --case-optimization on the login node.
-# Called by the case-optimization CI job before SLURM submission.
-# Usage: bash .github/scripts/prebuild-case-optimization.sh <cluster> <device> <interface>
+# Pre-builds all benchmark cases with --case-optimization.
+# Can run in two modes:
+#   1. Direct (Frontier login nodes): pass cluster/device/interface as args
+#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit.sh
+# Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
 
 set -e
 
-cluster=$1
-job_device=$2
-job_interface=$3
+# Support both positional args (direct invocation) and env vars (SLURM via submit.sh)
+cluster="${1:-${job_cluster:-phoenix}}"
+job_device="${2:-$job_device}"
+job_interface="${3:-$job_interface}"
 
 # Derive module flag from cluster name
 case "$cluster" in
@@ -21,10 +24,7 @@ esac
 . ./mfc.sh load -c "$flag" -m g
 source .github/scripts/gpu-opts.sh
 
-# Case-optimized GPU builds are memory-intensive (nvfortran/CCE + target offload).
-# Login nodes have per-user cgroup memory limits (e.g., 4GB on Phoenix) that
-# cause OOM kills at higher parallelism.
 for case in benchmarks/*/case.py; do
     echo "=== Pre-building: $case ==="
-    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 2
+    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
 done

.github/scripts/retry-build.sh

@@ -24,7 +24,7 @@ _retry_clean() {
 retry_build() {
     local clean_cmd="${RETRY_CLEAN_CMD:-rm -rf build/staging build/install build/lock.yaml}"
     local validate_cmd="${RETRY_VALIDATE_CMD:-}"
-    local max_attempts=1
+    local max_attempts=3
     local attempt=1
     while [ $attempt -le $max_attempts ]; do
         echo "Build attempt $attempt of $max_attempts..."

.github/workflows/bench.yml

@@ -106,7 +106,7 @@ jobs:
         if: matrix.build_script != ''
         uses: nick-fields/retry@v3
         with:
-          max_attempts: 1
+          max_attempts: 3
           retry_wait_seconds: 60
           timeout_minutes: 150
           command: |

.github/workflows/test.yml

@@ -249,7 +249,7 @@ jobs:
         if:   matrix.cluster != 'phoenix'
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
         with:
-          max_attempts: 1
+          max_attempts: 3
           retry_wait_seconds: 60
           timeout_minutes: 60
           command: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
@@ -323,10 +323,15 @@ jobs:
         with:
           clean: false
 
-      - name: Pre-Build
+      - name: Pre-Build (SLURM)
+        if:   matrix.cluster == 'phoenix'
+        run:  bash .github/workflows/phoenix/submit.sh .github/scripts/prebuild-case-optimization.sh ${{ matrix.device }} ${{ matrix.interface }}
+
+      - name: Pre-Build (login node)
+        if:   matrix.cluster != 'phoenix'
         uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
         with:
-          max_attempts: 1
+          max_attempts: 3
           retry_wait_seconds: 60
           timeout_minutes: 120
           command: bash .github/scripts/prebuild-case-optimization.sh ${{ matrix.cluster }} ${{ matrix.device }} ${{ matrix.interface }}
@@ -337,11 +342,17 @@ jobs:
 
       - name: Print Logs
         if:   always()
-        run:  cat run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out
+        run:  |
+          for f in prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out \
+                   run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out; do
+            [ -f "$f" ] && echo "=== $f ===" && cat "$f"
+          done
 
       - name: Archive Logs
         uses: actions/upload-artifact@v4
         if:   always()
         with:
           name: case-opt-${{ strategy.job-index }}-${{ matrix.cluster }}-${{ matrix.interface }}
-          path: run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out
+          path: |
+            prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out
+            run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out