Makefile

# Agent Smith workflow shortcuts

# Provider presets. Override with PROVIDER=groq, PROVIDER=google,
# PROVIDER=openrouter, PROVIDER=cohere, PROVIDER=sarvam, or a full endpoint URL.
PROVIDER ?= groq
PROVIDER_openrouter := https://openrouter.ai/api/v1
PROVIDER_google := https://generativelanguage.googleapis.com/v1beta/openai
PROVIDER_groq := https://api.groq.com/openai/v1
PROVIDER_cohere := https://api.cohere.ai/completions/v1/text
PROVIDER_together := https://api.together.ai/v1/chat/completions
PROVIDER_mistral := https://api.mistral.ai/v1/chat/completions
PROVIDER_sarvam := https://api.sarvam.ai/v1
# Backward-compatible local alias: Artificial Analysis is a benchmark/data API,
# not the chat completion endpoint used by the agent.
PROVIDER_artificial := $(PROVIDER_sarvam)

PROVIDER_URL := $(or $(PROVIDER_$(PROVIDER)),$(PROVIDER))

# Model presets (override with LLM=<name>, MODEL_PRESET=<name>, or explicit MODEL=...)
LLM ?= qwen3
MODEL_PRESET ?= $(LLM)
#MISTRAL
MODEL_mistral := mistral-large-2512
#TOGETHER
MODEL_qwentogether := Qwen/Qwen3-235B-A22B-fp8
#GROQ
MODEL_qwen3 := qwen/qwen3-32b
MODEL_llama2 := meta-llama/llama-prompt-guard-2-86m
MODEL_llama3 := llama-3.3-70b-versatile
MODEL_llama3.1 := llama-3.1-8b-instant
MODEL_gpt := openai/gpt-oss-120b
MODEL_gpt_safeguard := openai/gpt-oss-safeguard-20b
MODEL_compound := groq/compound
MODEL_llama4 := meta-llama/llama-4-scout-17b-16e-instruct
#OPEN ROUTER
MODEL_llama := meta-llama/llama-3.3-70b-instruct:free
MODEL_qwen := qwen/qwen3-next-80b-a3b-instruct:free
MODEL_tencent := tencent/hy3-preview:free
MODEL_minimax := minimax/minimax-m2.5:free
MODEL_hermes := nousresearch/hermes-3-llama-3.1-405b:free
MODEL_ling := inclusionai/ling-2.6-1t:free
#GOOGLE
MODEL_gemini := gemini-3.1-flash-lite-preview
#COHERE
MODEL_cohere := command-a-03-2025

#SARVAM
MODEL_sarvam := sarvam-30b
MODEL_sarvam30 := sarvam-30b
MODEL_sarvam105 := sarvam-105b

MODEL ?= $(or $(MODEL_$(LLM)),$(MODEL_$(MODEL_PRESET)),$(MODEL_qwen))

MBPP_TASK ?= cache/mbpp_task.json
MBPP_SOLUTION ?= cache/mbpp_solution.json

SWEBENCH_TASK ?= cache/swebench_task.json
SWEBENCH_SOLUTION ?= cache/swebench_solution.json
SWEBENCH_INSTANCE ?= sympy__sympy-18189
SWEBENCH_REPLAY_DIR ?= evaluations/swebench/replay/$(SWEBENCH_INSTANCE)
SWEBENCH_REPLAY_TASK ?= $(SWEBENCH_REPLAY_DIR)/task.json
SWEBENCH_REPLAY_SOLUTION ?= $(SWEBENCH_REPLAY_DIR)/solution.json

STUDENT_PATH ?= ./student
MOULINETTE_PATH ?= ./moulinette
ENV_FILE ?= ./.env

.PHONY: help models docker-check clean clean-cache clean-py mbpp-dump mbpp-run mbpp-validate mbpp-all swebench-dump swebench-run swebench-validate swebench-all swebench-dump-task swebench-replay swebench-triage swebench-overview swebench-overview-run swebench-backfill exam-mbpp exam-swebench exam-sandbox exam-anticheat exam-all

help:
	@echo "Available targets:"
	@echo "  make mbpp-dump       # Dump one MBPP task into $(MBPP_TASK)"
	@echo "  make mbpp-run        # Run MBPP agent"
	@echo "  make mbpp-validate   # Validate MBPP solution"
	@echo "  make mbpp-all        # Dump + run + validate MBPP"
	@echo "  make swebench-dump   # Dump one SWE-bench task into $(SWEBENCH_TASK)"
	@echo "  make swebench-run    # Run SWE-bench agent"
	@echo "  make swebench-validate # Validate SWE-bench solution"
	@echo "  make swebench-all    # Dump + run + validate SWE-bench"
	@echo "  make swebench-replay SWEBENCH_INSTANCE=... # Run one known SWE-bench task"
	@echo "  make swebench-triage SWEBENCH_REPLAY_DIR=... # Summarize SWE-bench artifacts"
	@echo "  make swebench-overview # Update evaluation overview from new runs"
	@echo "  make swebench-backfill # Backfill overview with all existing runs"
	@echo "  make exam-mbpp       # Official MBPP exam script (5 tasks, pass >= 4)"
	@echo "  make exam-swebench   # Official SWE-bench exam script"
	@echo "  make exam-sandbox    # Official sandbox exam script"
	@echo "  make exam-anticheat  # Official anti-cheat checks"
	@echo "  make exam-all        # Run all exam scripts in sequence"
	@echo "  make models          # Show preset model names and IDs"
	@echo "  make docker-check    # Verify Docker daemon is reachable"
	@echo "  make clean           # Remove generated task/solution cache files"
	@echo "  make clean-py        # Remove Python cache artifacts"
	@echo "  make clean-cache     # Same as clean (generated MBPP/SWE files)"
	@echo ""
	@echo "Default preset: LLM=$(LLM) (MODEL_PRESET=$(MODEL_PRESET)) => $(MODEL)"
	@echo "Provider: PROVIDER=$(PROVIDER) => $(PROVIDER_URL)"
	@echo ""
	@echo "Override model/provider like this:"
	@echo "  make mbpp-run LLM=ling"
	@echo "  make mbpp-run MODEL_PRESET=ling"
	@echo "  make mbpp-run MODEL='inclusionai/ling-2.6-1t:free'"
	@echo "  make mbpp-run PROVIDER=groq"
	@echo "  make mbpp-run PROVIDER=sarvam LLM=sarvam"
	@echo "  make mbpp-run PROVIDER='https://openrouter.ai/api/v1'"
	@echo ""
	@echo "Exam overrides:"
	@echo "  make exam-mbpp LLM=minimax"
	@echo "  make exam-mbpp STUDENT_PATH=./student MOULINETTE_PATH=./moulinette ENV_FILE=./.env"

models:
	@echo "Available LLM / MODEL_PRESET values:"
	@echo "  qwen    -> $(MODEL_qwen)"
	@echo "  mistral -> $(MODEL_mistral)"
	@echo "  llama   -> $(MODEL_llama)"
	@echo "  ling    -> $(MODEL_ling)"
	@echo "  hermes  -> $(MODEL_hermes)"
	@echo "  minimax -> $(MODEL_minimax)"
	@echo "  sarvam  -> $(MODEL_sarvam)"
	@echo "  sarvam105 -> $(MODEL_sarvam105)"
	@echo ""
	@echo "Current selection: LLM=$(LLM)"
	@echo "Alias selection: MODEL_PRESET=$(MODEL_PRESET)"
	@echo "Resolved MODEL: $(MODEL)"
	@echo ""
	@echo "Available PROVIDER values:"
	@echo "  openrouter -> $(PROVIDER_openrouter)"
	@echo "  google     -> $(PROVIDER_google)"
	@echo "  groq       -> $(PROVIDER_groq)"
	@echo "  cohere     -> $(PROVIDER_cohere)"
	@echo "  sarvam     -> $(PROVIDER_sarvam)"
	@echo ""
	@echo "Current provider: PROVIDER=$(PROVIDER)"
	@echo "Resolved URL: $(PROVIDER_URL)"

docker-check:
	@docker info --format 'server={{.ServerVersion}}'
	@docker ps --format 'table {{.Names}}\t{{.Status}}'

clean-cache:
	rm -f cache/mbpp_task.json cache/mbpp_solution.json
	rm -f cache/swebench_task.json cache/swebench_solution.json

clean-py:
	rm -rf .pytest_cache
	find . -type d -name '__pycache__' -prune -exec rm -rf {} +

clean: clean-cache clean-py

mbpp-dump:
	cd moulinette && uv run moulinette_eval dump mbpp --output ../$(MBPP_TASK)

mbpp-run:
	cd student && uv run python -m agent_mbpp \
		--task-file ../$(MBPP_TASK) \
		--output ../$(MBPP_SOLUTION) \
		--model-name "$(MODEL)" \
		--provider-url "$(PROVIDER_URL)"

mbpp-validate:
	cd moulinette && uv run moulinette_eval validate mbpp ../$(MBPP_TASK) ../$(MBPP_SOLUTION)

mbpp-all: mbpp-dump mbpp-run mbpp-validate

swebench-dump:
	cd moulinette && uv run moulinette_eval dump swebench --output ../$(SWEBENCH_TASK)

swebench-run:
	cd student && uv run python -m agent_swebench \
		--task-file ../$(SWEBENCH_TASK) \
		--output ../$(SWEBENCH_SOLUTION) \
		--model-name "$(MODEL)" \
		--provider-url "$(PROVIDER_URL)"

swebench-validate:
	cd moulinette && uv run moulinette_eval validate swebench ../$(SWEBENCH_TASK) ../$(SWEBENCH_SOLUTION)

swebench-all: swebench-dump swebench-run swebench-validate
	@echo "Updating evaluation overview..."
	@uv run python scripts/update_swebench_overview.py
	@echo "Overview updated: SWEBENCH_EVALUATION_OVERVIEW.md"

swebench-dump-task:
	mkdir -p "$(SWEBENCH_REPLAY_DIR)"
	cd moulinette && uv run moulinette_eval dump swebench \
		--task-id "$(SWEBENCH_INSTANCE)" \
		--output "../$(SWEBENCH_REPLAY_TASK)"

swebench-replay: swebench-dump-task
	cd student && uv run python -m agent_swebench \
		--task-file ../$(SWEBENCH_REPLAY_TASK) \
		--output ../$(SWEBENCH_REPLAY_SOLUTION) \
		--model-name "$(MODEL)" \
		--provider-url "$(PROVIDER_URL)"
	-cd moulinette && uv run moulinette_eval validate swebench \
		../$(SWEBENCH_REPLAY_TASK) \
		../$(SWEBENCH_REPLAY_SOLUTION)
	python3 tools/swebench_triage.py "$(SWEBENCH_REPLAY_DIR)"

swebench-triage:
	python3 tools/swebench_triage.py "$(SWEBENCH_REPLAY_DIR)"

swebench-overview:
	uv run python scripts/update_swebench_overview.py

swebench-overview-run:
	uv run python scripts/update_swebench_overview.py $(RUN)

swebench-backfill:
	uv run python scripts/update_swebench_overview.py
	@echo "Backfill complete."

exam-mbpp:
	PATH="$(CURDIR)/tools:$$PATH" bash exams/exam_mbpp.sh \
		--student-path "$(STUDENT_PATH)" \
		--moulinette-path "$(MOULINETTE_PATH)" \
		--env-file "$(ENV_FILE)" \

exam-swebench:
	PATH="$(CURDIR)/tools:$$PATH" bash exams/exam_swebench.sh \
		--student-path "$(STUDENT_PATH)" \
		--moulinette-path "$(MOULINETTE_PATH)" \
		--env-file "$(ENV_FILE)" \
		--model-name "$(MODEL)" \
		--provider-url "$(PROVIDER_URL)"

exam-sandbox:
	bash exams/exam_sandbox.sh \
		--student-path "$(STUDENT_PATH)" \
		--moulinette-path "$(MOULINETTE_PATH)" \
		--env-file "$(ENV_FILE)"

exam-anticheat:
	bash exams/exam_anticheat.sh \
		--student-path "$(STUDENT_PATH)"

benchmark-run:
	cd student && uv run python -m agent_swebench \
	  --task-file ../benchmark/tasks/$(TASK).json \
	  --output ../benchmark/results/$(TASK)/$(MODEL)_solution.json \
	  --model-name "$(MODEL)" \
	  --provider-url "$(PROVIDER_URL)"

exam-all: exam-anticheat exam-sandbox exam-mbpp exam-swebench