Skip to content

Commit 40e43e8

Browse files
musicofhelclaude
andcommitted
Stabilization sweep: centralize paths, TB-2 timeout retry, per-persona timeout
Centralize all hardcoded /tmp/dev-loop/ paths into paths.py with DEVLOOP_TMP_DIR env var override (14 refs across 8 source files, justfile, and stress-test script). Feed TB-2 spawn timeouts into the retry loop instead of early-returning — the 303s timeout from stress tests will now trigger a retry with diagnostic context. Add per-persona timeout_seconds to agents.yaml (120-600s by type), threaded through select_persona → spawn_agent in all TB pipelines. Align stress test subprocess timeout (600→1320s) with pipeline cap. Close 219 stress test beads issues. 3 new tests. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent e0c86ef commit 40e43e8

20 files changed

Lines changed: 452 additions & 295 deletions

.beads/issues.jsonl

Lines changed: 219 additions & 219 deletions
Large diffs are not rendered by default.

config/agents.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ personas:
1313
model: sonnet
1414
max_turns_default: 10
1515
max_context_pct: 75
16+
timeout_seconds: 300
1617

1718
feature:
1819
labels: [feature]
@@ -24,6 +25,7 @@ personas:
2425
model: opus
2526
max_turns_default: 25
2627
max_context_pct: 75
28+
timeout_seconds: 600
2729

2830
refactor:
2931
labels: [refactor]
@@ -35,6 +37,7 @@ personas:
3537
model: opus
3638
max_turns_default: 20
3739
max_context_pct: 75
40+
timeout_seconds: 600
3841

3942
security-fix:
4043
labels: [security]
@@ -47,6 +50,7 @@ personas:
4750
model: opus
4851
max_turns_default: 15
4952
max_context_pct: 75
53+
timeout_seconds: 600
5054

5155
docs:
5256
labels: [docs]
@@ -57,6 +61,7 @@ personas:
5761
model: haiku
5862
max_turns_default: 10
5963
max_context_pct: 75
64+
timeout_seconds: 120
6065

6166
chore:
6267
labels: [chore]
@@ -67,6 +72,7 @@ personas:
6772
model: haiku
6873
max_turns_default: 5
6974
max_context_pct: 75
75+
timeout_seconds: 120
7076

7177
performance:
7278
labels: [performance, perf]
@@ -77,6 +83,7 @@ personas:
7783
model: sonnet
7884
max_turns_default: 15
7985
max_context_pct: 75
86+
timeout_seconds: 300
8087

8188
infrastructure:
8289
labels: [infrastructure, ci, ci-cd, devops]
@@ -88,6 +95,7 @@ personas:
8895
model: sonnet
8996
max_turns_default: 10
9097
max_context_pct: 75
98+
timeout_seconds: 300
9199

92100
test:
93101
labels: [test, testing]
@@ -99,3 +107,4 @@ personas:
99107
model: sonnet
100108
max_turns_default: 15
101109
max_context_pct: 75
110+
timeout_seconds: 300

justfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ emergency-stop:
232232
recover:
233233
@echo "=== Recovery scan ==="
234234
@echo "Checking for orphaned worktrees..."
235-
@find /tmp/dev-loop/worktrees -name ".dev-loop-metadata.json" -mmin +60 2>/dev/null || echo " No worktree directory found"
235+
@find "${DEVLOOP_TMP_DIR:-/tmp/dev-loop}"/worktrees -name ".dev-loop-metadata.json" -mmin +60 2>/dev/null || echo " No worktree directory found"
236236
@echo "Checking for stuck issues..."
237237
@br stale --days 1 2>/dev/null || echo " No stale issues"
238238
@echo "Run 'just worktree-gc' to clean up orphaned worktrees"
@@ -242,7 +242,7 @@ worktree-gc:
242242
#!/usr/bin/env bash
243243
set -euo pipefail
244244
echo "Scanning for orphaned worktrees..."
245-
orphans=$(find /tmp/dev-loop/worktrees -maxdepth 1 -mmin +1440 -type d 2>/dev/null || true)
245+
orphans=$(find "${DEVLOOP_TMP_DIR:-/tmp/dev-loop}"/worktrees -maxdepth 1 -mmin +1440 -type d 2>/dev/null || true)
246246
if [ -z "$orphans" ]; then
247247
echo " No orphans found"
248248
exit 0
@@ -278,7 +278,7 @@ sessions-list *ARGS:
278278
#!/usr/bin/env bash
279279
sessions_dir="${HOME}/.local/share/dev-loop/sessions"
280280
if [ ! -d "$sessions_dir" ]; then
281-
sessions_dir="/tmp/dev-loop/sessions"
281+
sessions_dir="${DEVLOOP_TMP_DIR:-/tmp/dev-loop}/sessions"
282282
fi
283283
if [ ! -d "$sessions_dir" ]; then
284284
echo "No sessions directory found"

scripts/stress-test.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,16 @@
2222
from pathlib import Path
2323

2424
REPO = Path(__file__).parent.parent
25+
26+
# Add project src to path so we can import devloop.paths
27+
import sys
28+
sys.path.insert(0, str(REPO / "src"))
29+
from devloop.paths import RESULTS_DIR, SESSIONS_DIR, WORKTREE_BASE
30+
2531
PROMPT_BENCH = Path.home() / "prompt-bench"
2632
OOTESTPROJECT1 = Path.home() / "OOTestProject1"
27-
RESULTS_BASE = Path("/tmp/dev-loop/stress-test")
28-
WORKTREE_BASE = Path("/tmp/dev-loop/worktrees")
29-
SESSION_BASE = Path("/tmp/dev-loop/sessions")
33+
RESULTS_BASE = RESULTS_DIR
34+
SESSION_BASE = SESSIONS_DIR
3035

3136

3237
def ts() -> str:
@@ -127,7 +132,7 @@ def run_pipeline(func_name: str, *args) -> dict:
127132
result = subprocess.run(
128133
["uv", "run", "python", "-c", code],
129134
capture_output=True, text=True, check=False,
130-
timeout=600, cwd=str(REPO),
135+
timeout=1320, cwd=str(REPO), # 1200s pipeline cap + 120s grace
131136
env={**os.environ, "CLAUDECODE": ""}, # unset CLAUDECODE
132137
)
133138
# Find the JSON line in stdout (skip log noise)

src/devloop/config_schemas.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class PersonaConfig(BaseModel):
3232
model: str
3333
max_turns_default: int = Field(ge=1)
3434
max_context_pct: int = Field(ge=1, le=100)
35+
timeout_seconds: int = Field(default=300, ge=30)
3536

3637
@field_validator("model")
3738
@classmethod

src/devloop/feedback/cost_monitor.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,20 @@
2222

2323
from opentelemetry import trace
2424

25+
from devloop.paths import SESSIONS_DIR as _SESSIONS_DIR
26+
2527
logger = logging.getLogger(__name__)
2628
tracer = trace.get_tracer("feedback.cost_monitor", "0.1.0")
2729

2830
# ---------------------------------------------------------------------------
2931
# Session-based usage tracking (reads from TB-6 session metadata)
3032
# ---------------------------------------------------------------------------
3133

32-
_SESSIONS_DIR = Path("/tmp/dev-loop/sessions")
33-
3434

3535
def get_usage_summary(hours: int = 24) -> dict:
3636
"""Aggregate usage stats from recent session metadata files.
3737
38-
Scans /tmp/dev-loop/sessions/*.meta.json for sessions within the
38+
Scans ``SESSIONS_DIR/*.meta.json`` for sessions within the
3939
time window and sums up turn/token counts.
4040
4141
Args:

src/devloop/feedback/pattern_detector.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@
2323

2424
from opentelemetry import trace
2525

26+
from devloop.paths import SESSIONS_DIR as _SESSIONS_DIR
27+
2628
logger = logging.getLogger(__name__)
2729
tracer = trace.get_tracer("feedback.pattern_detector", "0.1.0")
2830

29-
_SESSIONS_DIR = Path("/tmp/dev-loop/sessions")
30-
3131
# ---------------------------------------------------------------------------
3232
# Known failure patterns and their suggested fixes
3333
# ---------------------------------------------------------------------------

src/devloop/feedback/server.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ def retry_agent(
311311
max_retries: int = 2,
312312
model: str = "sonnet",
313313
max_turns: int | None = None,
314+
timeout_seconds: float = 300.0,
314315
) -> dict:
315316
"""Re-spawn agent with failure context and re-run gates."""
316317
with tracer.start_as_current_span(
@@ -364,6 +365,7 @@ def retry_agent(
364365
task_prompt=prompt_text,
365366
model=model,
366367
max_turns=max_turns,
368+
timeout_seconds=timeout_seconds,
367369
)
368370

369371
exit_code = agent_result.get("exit_code", -1)

src/devloop/feedback/tb1_golden_path.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,7 @@ def run_tb1(issue_id: str, repo_path: str) -> dict:
341341
model=persona_result.get("model", "sonnet"),
342342
allowed_tools=allowed_tools,
343343
max_context_pct=max_context_pct,
344+
timeout_seconds=persona_result.get("timeout_seconds", 300),
344345
)
345346

346347
agent_exit = agent_result.get("exit_code", -1)
@@ -431,6 +432,7 @@ def run_tb1(issue_id: str, repo_path: str) -> dict:
431432
model=persona_result.get("model", "sonnet"),
432433
allowed_tools=allowed_tools,
433434
max_context_pct=max_context_pct,
435+
timeout_seconds=persona_result.get("timeout_seconds", 300),
434436
)
435437

436438
agent_exit = agent_result.get("exit_code", -1)

src/devloop/feedback/tb2_retry.py

Lines changed: 97 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -348,13 +348,26 @@ def run_tb2(
348348
task_prompt=task_prompt,
349349
model=persona_result.get("model", "sonnet"),
350350
allowed_tools=allowed_tools,
351+
timeout_seconds=persona_result.get("timeout_seconds", 300),
351352
)
352353

353354
agent_exit = agent_result.get("exit_code", -1)
354355
agent_span.set_attribute("tb2.agent_exit_code", agent_exit)
355356
attempt_span_ids.append(_span_id_hex(agent_span))
356357

357-
if agent_exit != 0:
358+
# If the agent timed out, feed the timeout into the retry
359+
# loop instead of returning early. This lets TB-2 retry
360+
# after a slow spawn (the 303s timeout seen in stress tests).
361+
initial_timed_out = agent_result.get("timed_out", False)
362+
if initial_timed_out:
363+
duration = agent_result.get("duration_seconds", 0)
364+
agent_span.set_attribute("tb2.agent_timed_out", True)
365+
agent_span.set_attribute("tb2.agent_duration", duration)
366+
logger.warning(
367+
"TB-2: Initial agent timed out after %.0fs — entering retry loop",
368+
duration,
369+
)
370+
elif agent_exit != 0:
358371
elapsed = time.monotonic() - pipeline_start
359372
return TB2Result(
360373
issue_id=issue_id,
@@ -372,59 +385,95 @@ def run_tb2(
372385
).model_dump()
373386

374387
# ----------------------------------------------------------
375-
# Phase 8: Run quality gates (or force-fail on first attempt)
388+
# Phase 8: Run quality gates (or force-fail, or skip if timed out)
376389
# ----------------------------------------------------------
377-
with tracer_tb2.start_as_current_span(
378-
"tb2.phase.gates",
379-
attributes={
380-
"tb2.phase": "gates",
381-
"tb2.attempt": 0,
382-
"tb2.force_fail": force_gate_fail,
383-
},
384-
) as gates_span:
385-
if force_gate_fail:
386-
logger.info("TB-2: FORCED FAILURE on initial gate run")
387-
gate_raw = _make_forced_failure()
388-
else:
389-
gate_raw = run_all_gates(
390-
worktree_path=worktree_path,
391-
issue_title=issue_title,
392-
issue_description=issue_description,
393-
)
394-
395-
try:
396-
gate_suite = GateSuiteResult(**gate_raw)
397-
except Exception as exc:
398-
elapsed = time.monotonic() - pipeline_start
399-
error_msg = f"Malformed gate result: {exc}"
400-
return TB2Result(
401-
issue_id=issue_id,
402-
repo_path=repo_path,
403-
success=False,
404-
phase="gates",
405-
worktree_path=worktree_path,
406-
persona=persona_name,
407-
error=error_msg,
408-
duration_seconds=round(elapsed, 2),
409-
trace_id=root_trace_id,
410-
attempt_span_ids=attempt_span_ids,
411-
force_gate_fail_used=force_gate_fail,
412-
).model_dump()
413-
414-
gates_span.set_attribute("tb2.gates_passed", gate_suite.overall_passed)
415-
if gate_suite.first_failure:
416-
gates_span.set_attribute("tb2.first_failure", gate_suite.first_failure)
417-
418-
# Record initial attempt
390+
gate_suite: GateSuiteResult | None = None
391+
gate_raw: dict | None = None
392+
gates_span_context = None
393+
394+
if initial_timed_out:
395+
# Agent timed out — no output to gate. Seed a timeout
396+
# failure record and jump straight to the retry loop.
397+
duration = agent_result.get("duration_seconds", 0)
398+
gate_raw = GateSuiteResult(
399+
overall_passed=False,
400+
first_failure="agent_timeout",
401+
gate_results=[
402+
GateResult(
403+
gate_name="agent_timeout",
404+
passed=False,
405+
findings=[
406+
Finding(
407+
severity="critical",
408+
message=f"Agent timed out after {duration:.0f}s (no output to gate)",
409+
),
410+
],
411+
),
412+
],
413+
).model_dump()
414+
gate_suite = GateSuiteResult(**gate_raw)
419415
retry_history.append(
420416
RetryAttempt(
421417
attempt=0,
422418
agent_exit_code=agent_exit,
423-
gates_passed=gate_suite.overall_passed,
424-
first_failure=gate_suite.first_failure,
419+
gates_passed=False,
420+
first_failure="agent_timeout",
425421
span_id=attempt_span_ids[0] if attempt_span_ids else None,
426422
)
427423
)
424+
else:
425+
with tracer_tb2.start_as_current_span(
426+
"tb2.phase.gates",
427+
attributes={
428+
"tb2.phase": "gates",
429+
"tb2.attempt": 0,
430+
"tb2.force_fail": force_gate_fail,
431+
},
432+
) as gates_span:
433+
if force_gate_fail:
434+
logger.info("TB-2: FORCED FAILURE on initial gate run")
435+
gate_raw = _make_forced_failure()
436+
else:
437+
gate_raw = run_all_gates(
438+
worktree_path=worktree_path,
439+
issue_title=issue_title,
440+
issue_description=issue_description,
441+
)
442+
443+
try:
444+
gate_suite = GateSuiteResult(**gate_raw)
445+
except Exception as exc:
446+
elapsed = time.monotonic() - pipeline_start
447+
error_msg = f"Malformed gate result: {exc}"
448+
return TB2Result(
449+
issue_id=issue_id,
450+
repo_path=repo_path,
451+
success=False,
452+
phase="gates",
453+
worktree_path=worktree_path,
454+
persona=persona_name,
455+
error=error_msg,
456+
duration_seconds=round(elapsed, 2),
457+
trace_id=root_trace_id,
458+
attempt_span_ids=attempt_span_ids,
459+
force_gate_fail_used=force_gate_fail,
460+
).model_dump()
461+
462+
gates_span.set_attribute("tb2.gates_passed", gate_suite.overall_passed)
463+
if gate_suite.first_failure:
464+
gates_span.set_attribute("tb2.first_failure", gate_suite.first_failure)
465+
gates_span_context = gates_span.get_span_context()
466+
467+
# Record initial attempt
468+
retry_history.append(
469+
RetryAttempt(
470+
attempt=0,
471+
agent_exit_code=agent_exit,
472+
gates_passed=gate_suite.overall_passed,
473+
first_failure=gate_suite.first_failure,
474+
span_id=attempt_span_ids[0] if attempt_span_ids else None,
475+
)
476+
)
428477

429478
# ----------------------------------------------------------
430479
# Phase 9: Gates passed on first try -> success
@@ -485,7 +534,7 @@ def run_tb2(
485534
# Phase 10: Gates failed -> retry loop with span linking
486535
# ----------------------------------------------------------
487536
all_gate_failures: list[dict] = [gate_raw]
488-
previous_span_context = gates_span.get_span_context()
537+
previous_span_context = gates_span_context
489538

490539
for attempt in range(1, max_retries + 1):
491540
retries_used = attempt
@@ -526,6 +575,7 @@ def run_tb2(
526575
gate_failures=all_gate_failures,
527576
attempt=attempt,
528577
max_retries=max_retries,
578+
timeout_seconds=persona_result.get("timeout_seconds", 300),
529579
)
530580

531581
retry_success = retry_raw.get("success", False)

0 commit comments

Comments
 (0)