FuncBenchGen/scripts/run_funcbenchgen.sh at main · megagonlabs/FuncBenchGen · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#!/usr/bin/env bash
set -euo pipefail

# ---------------- Models to run ----------------
MODELS=(
  "gpt-5-2025-08-07"
)

# ---------------- I/O & experiment naming ----------------
ROOT_SAVE_DIR="./data"
EXPERIMENT_NAME="main_experiment"

# ---------------- Repro / execution knobs ----------------
NUM_TRIALS_PER_GRAPH=1
NUM_GRAPHS_PER_CONFIG=5
OVERWRITE_GRAPHS=False
VERBOSE=False

# ---------------- Graph image & aggregate files ----------------
SAVE_AGGREGATE_OUTPUTS=True # write CSV/Parquet; plots are in the notebook

# ---------------- Sweep spaces ----------------
# Core nodes to sweep (extend as needed)
CORE_NODE_OPTIONS=(
  10
  # 20
  # 40
  )

# Pairs of [EXTRA_CONNECTED_NODES, NUM_DISCONNECTED_NODES]
# Interpretation:
#   NUM_TOTAL_NODES = CORE_NODES + NUM_DISCONNECTED_NODES + EXTRA_CONNECTED_NODES
EXTRA_DISCONNECTED_PAIRS=(
  # # no extra
  "0,0"
  # # # # # connected
  # "10,0"
  # "20,0"
  # "40,0"
  # # disconnected
  # "0,10"
  # "0,20"
  # "0,40"
  # # half & half
  # "5,5"
  # "10,10"
  # "20,20"
)

# ---------------- Type & variable naming ----------------
USE_TYPES=True
SHOW_VARNAMES_IN_DESC=False
RENAME_VARIABLES=False

# ---------------- Type subtype controls ----------------
USE_SUBTYPES=True
NUM_SUPERTYPES=5

# ---------------- GPT-5 controls (defaults 'medium') ----------------
GPT5_THINKING_BUDGET="medium"  # minimal | low | medium | high
GPT5_VERBOSITY="medium"        # low | medium | high

# ---------------- Noise inputs ----------------
NUM_NOISE_INPUTS=0

# ---------------- Launcher ----------------
SCRIPT_DIR="./src"
PYTHON_BIN="python"

for MODEL in "${MODELS[@]}"; do
  echo "[run] model=${MODEL}"

  for CORE_NODES in "${CORE_NODE_OPTIONS[@]}"; do
    # Derive MCP range from CORE_NODES
    MCP_START=1
    MCP_STOP=$(( CORE_NODES - 1 ))

    # MCP_STEP = CORE_NODES // 10
    if (( CORE_NODES >= 10 )); then
      MCP_STEP=$(( CORE_NODES / 10 ))
    else
      MCP_STEP=1
    fi

    # Skip invalid MCP range (e.g., CORE_NODES < 2)
    if (( MCP_STOP < MCP_START )); then
      echo "[skip] core_nodes=${CORE_NODES} -> invalid MCP range (${MCP_START}..${MCP_STOP})"
      continue
    fi

    for PAIR in "${EXTRA_DISCONNECTED_PAIRS[@]}"; do
      # Parse "EXTRA_CONNECTED,DISCONNECTED"
      IFS=',' read -r EXTRA_CONNECTED_NODES NUM_DISCONNECTED_NODES <<< "${PAIR}"

      # Compute total nodes from the rule:
      # extra connected nodes = NUM_TOTAL_NODES - (CORE_NODES + NUM_DISCONNECTED_NODES)
      NUM_TOTAL_NODES=$(( CORE_NODES + NUM_DISCONNECTED_NODES + EXTRA_CONNECTED_NODES ))

      # Sanity check
      if (( NUM_TOTAL_NODES < CORE_NODES )); then
        echo "[skip] total nodes < core nodes (should not happen) core=${CORE_NODES} total=${NUM_TOTAL_NODES}"
        continue
      fi

      echo "[run] core=${CORE_NODES} mcp=[${MCP_START}:${MCP_STOP}:${MCP_STEP}] extra_connected=${EXTRA_CONNECTED_NODES} disconnected=${NUM_DISCONNECTED_NODES} total=${NUM_TOTAL_NODES}"

      "${PYTHON_BIN}" "${SCRIPT_DIR}/funcbenchgen.py" run \
        --model "${MODEL}" \
        --root_save_dir "${ROOT_SAVE_DIR}" \
        --experiment_name "${EXPERIMENT_NAME}" \
        --num_graphs_per_config "${NUM_GRAPHS_PER_CONFIG}" \
        --save_aggregate_outputs "${SAVE_AGGREGATE_OUTPUTS}" \
        --num_trials_per_graph "${NUM_TRIALS_PER_GRAPH}" \
        --use_types "${USE_TYPES}" \
        --verbose "${VERBOSE}" \
        --mcp_start "${MCP_START}" \
        --mcp_stop "${MCP_STOP}" \
        --mcp_step "${MCP_STEP}" \
        --num_total_nodes "${NUM_TOTAL_NODES}" \
        --core_nodes "${CORE_NODES}" \
        --num_disconnected_nodes "${NUM_DISCONNECTED_NODES}" \
        --show_variable_names_in_description "${SHOW_VARNAMES_IN_DESC}" \
        --rename_variables "${RENAME_VARIABLES}" \
        --use_subtypes "${USE_SUBTYPES}" \
        --num_supertypes "${NUM_SUPERTYPES}" \
        --thinking_budget "${GPT5_THINKING_BUDGET}" \
        --verbosity "${GPT5_VERBOSITY}" \
        --num_noise_inputs "${NUM_NOISE_INPUTS}" \
        --overwrite_graphs "${OVERWRITE_GRAPHS}" \
        --on_wrong_inputs "Execute" \
        --overwrite_results False
    done
  done
done