Build in modal sandboxes (#226)

* New buildroot image

* Build in modal sandboxes

* Fix modal PR CI failures

* Update frontend visual snapshots

* Allow Buildroot data partitions to grow

* Normalize build provider settings on partial updates

* Support build host SD image generation

* Executor abstraction

* Route command execution through build executor

* Clarify Modal sandbox builds

* Log Modal sandbox timeouts

* Forward Modal image build logs

* Run arm64 Modal builds through nested Docker

* Add SD card build action to frame menu

* Run Modal arm builds with cross compilers

* Tune Modal sandbox build resources

* better text

* Update frontend visual snapshots

* Quiet QuickJS cross-compile logging

* Show SD image build progress updates

* Bake target cross toolchains into Modal images

---------

Co-authored-by: FrameOS Bot <git@frameos.net>

Author: mariusandra

.github/workflows/frameos-cross-toolchain.yml

@@ -56,8 +56,15 @@ jobs:
           import os
           import re
           import subprocess
+          import sys
           from pathlib import Path
 
+          sys.path.insert(0, str(Path("backend").resolve()))
+          from app.utils.cross_toolchain_packages import (  # noqa: E402
+              TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS,
+              TARGET_CROSS_TOOLCHAIN_PACKAGES,
+          )
+
           def sanitize(value: str) -> str:
               return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)
 
@@ -72,6 +79,11 @@ jobs:
               safe_platform = sanitize(target["platform"].replace("/", "_"))
               image = f"{image_repo}:{safe_base}-{safe_platform}-{image_tag}"
               metadata_file = Path(f"/tmp/toolchain-metadata-{index}.json")
+              target_cross_dpkg_archs = ""
+              target_cross_packages = ""
+              if target["platform"] == "linux/amd64":
+                  target_cross_dpkg_archs = " ".join(TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS)
+                  target_cross_packages = " ".join(TARGET_CROSS_TOOLCHAIN_PACKAGES)
 
               subprocess.run(
                   [
@@ -82,6 +94,10 @@ jobs:
                       target["platform"],
                       "--build-arg",
                       f"BASE_IMAGE={base_image}",
+                      "--build-arg",
+                      f"TARGET_CROSS_DPKG_ARCHS={target_cross_dpkg_archs}",
+                      "--build-arg",
+                      f"TARGET_CROSS_PACKAGES={target_cross_packages}",
                       "--tag",
                       image,
                       "--push",

backend/app/api/frames.py

@@ -112,6 +112,9 @@
 )
 from app.models.assets import copy_custom_fonts_to_local_source_folder
 from app.models.settings import get_settings_dict
+from app.utils.build_environment import selected_build_environment_provider
+from app.utils.build_host import get_build_executor_config
+from app.utils.build_executor import build_environment_requires_executor_config
 from app.utils.ssh_key_utils import default_ssh_key_ids
 from app.utils.timezone import frame_timezone, normalize_timezone, stored_timezone
 from app.utils.tls import generate_frame_tls_material, parse_certificate_not_valid_after
@@ -2436,6 +2439,15 @@ async def api_frame_buildroot_sd_image(
     if (frame.mode or "rpios") != "buildroot":
         _bad_request("SD card image generation is only available for Buildroot frames")
 
+    settings = get_settings_dict(db, project_id=frame.project_id)
+    build_environment_provider = selected_build_environment_provider(settings)
+    if build_environment_provider == "none":
+        _bad_request(
+            "Buildroot SD card image generation requires Docker, build host, or Modal sandboxes as the global build environment."
+        )
+    if build_environment_requires_executor_config(build_environment_provider) and get_build_executor_config(db, frame.project_id) is None:
+        _bad_request(f"Selected build environment '{build_environment_provider}' is not configured")
+
     try:
         ensure_buildroot_frame_defaults(frame)
     except ValueError as exc:

backend/app/api/settings.py

@@ -1,4 +1,6 @@
 from http import HTTPStatus
+from types import SimpleNamespace
+
 from fastapi import Depends, HTTPException
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
@@ -7,6 +9,10 @@
 from app.models.settings import get_settings_dict, Settings
 from app.schemas.settings import SettingsResponse, SettingsUpdateRequest
 from app.tenancy import current_project_id
+from app.utils.build_environment import selected_build_environment_provider
+from app.utils.build_executor import create_build_executor
+from app.utils.build_host import BuildHostConfig
+from app.utils.modal_sandbox import ModalSandboxConfig
 from app.utils.posthog import initialize_posthog
 from . import api_project
 
@@ -23,6 +29,13 @@ async def set_settings(data: SettingsUpdateRequest, db: Session = Depends(get_db
 
     try:
         current_settings = get_settings_dict(db, project_id=project_id)
+        merged_settings = {**current_settings, **payload}
+        provider = selected_build_environment_provider(merged_settings)
+        if isinstance(payload.get("buildHost"), dict):
+            payload["buildHost"] = {**payload["buildHost"], "enabled": provider == "buildHost"}
+        if isinstance(payload.get("modalSandbox"), dict):
+            payload["modalSandbox"] = {**payload["modalSandbox"], "enabled": provider == "modal"}
+
         for key, value in payload.items():
             if value != current_settings.get(key):
                 setting = db.query(Settings).filter_by(project_id=project_id, key=key).first()
@@ -39,3 +52,76 @@ async def set_settings(data: SettingsUpdateRequest, db: Session = Depends(get_db
     if "posthog" in payload:
         initialize_posthog(updated_settings, project_id=project_id)
     return updated_settings
+
+
+@api_project.post("/settings/test_build_host")
+async def test_build_host(data: SettingsUpdateRequest):
+    payload = data.to_dict()
+    raw_build_host_settings = payload.get("buildHost") if isinstance(payload, dict) else None
+    build_host_config = BuildHostConfig.from_settings(
+        {**raw_build_host_settings, "enabled": True} if isinstance(raw_build_host_settings, dict) else raw_build_host_settings
+    )
+    if build_host_config is None:
+        raise HTTPException(
+            status_code=HTTPStatus.BAD_REQUEST,
+            detail="Select build host via SSH and enter a host, user, and private SSH key first",
+    )
+
+    try:
+        async with create_build_executor(
+            build_host_config,
+            db=None,
+            redis=None,
+            frame=SimpleNamespace(id=0),
+            workspace_prefix="frameos-build-host-test-",
+        ) as executor:
+            status, out, err = await executor.run(
+                "echo frameos-build-host-ok && command -v docker >/dev/null && docker buildx version >/dev/null",
+                log_command=False,
+                log_output=False,
+            )
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(status_code=HTTPStatus.BAD_GATEWAY, detail=f"Build host connection failed: {exc}") from exc
+
+    if status != 0:
+        raise HTTPException(
+            status_code=HTTPStatus.BAD_GATEWAY,
+            detail=err or out or "Build host is missing Docker or the Docker Buildx plugin",
+        )
+
+    return {"ok": True, "output": (out or "").strip()}
+
+
+@api_project.post("/settings/test_modal_sandbox")
+async def test_modal_sandbox(data: SettingsUpdateRequest):
+    payload = data.to_dict()
+    raw_modal_settings = payload.get("modalSandbox") if isinstance(payload, dict) else None
+    modal_config = ModalSandboxConfig.from_settings(raw_modal_settings)
+    if modal_config is None:
+        raise HTTPException(
+            status_code=HTTPStatus.BAD_REQUEST,
+            detail="Select Modal sandboxes and enter a token ID and token secret first",
+    )
+
+    try:
+        async with create_build_executor(
+            modal_config,
+            db=None,
+            redis=None,
+            frame=SimpleNamespace(id=0),
+        ) as executor:
+            status, out, err = await executor.run(
+                "command -v nimble && nimble --version >/dev/null && echo frameos-modal-sandbox-ok",
+                log_command=False,
+                log_output=False,
+            )
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(status_code=HTTPStatus.BAD_GATEWAY, detail=f"Modal sandbox test failed: {exc}") from exc
+
+    if status != 0:
+        raise HTTPException(
+            status_code=HTTPStatus.BAD_GATEWAY,
+            detail=err or out or "Modal sandbox image is missing the FrameOS Nim toolchain",
+        )
+
+    return {"ok": True, "output": (out or "").strip()}

backend/app/api/system.py

@@ -1,5 +1,5 @@
 from app.api import api_user
-from app.schemas.system import CacheInfo, DatabaseInfo, DiskInfo, LoadInfo, MemoryInfo, SystemInfoResponse, SystemMetricsResponse
+from app.schemas.system import CacheInfo, DatabaseInfo, DiskInfo, DockerInfo, LoadInfo, MemoryInfo, SystemInfoResponse, SystemMetricsResponse
 from app.utils.system_info import get_system_info, get_system_metrics
 
 
@@ -27,15 +27,24 @@ def _database_to_schema(database) -> DatabaseInfo:
     )
 
 
+def _docker_to_schema(docker) -> DockerInfo:
+    return DockerInfo(
+        cliAvailable=docker.cli_available,
+        daemonAvailable=docker.daemon_available,
+        error=docker.error,
+    )
+
+
 @api_user.get("/system/info", response_model=SystemInfoResponse)
 def system_info():
-    disk, caches, database, memory, load = get_system_info()
+    disk, caches, database, memory, load, docker = get_system_info()
     return SystemInfoResponse(
         disk=_disk_to_schema(disk),
         caches=_cache_to_schema(caches),
         database=_database_to_schema(database),
         memory=_memory_to_schema(memory),
         load=_load_to_schema(load),
+        docker=_docker_to_schema(docker),
     )
 
 

backend/app/api/tests/test_frames.py

@@ -17,6 +17,7 @@
 from app.models.frame import Frame
 from app.models.log import Log
 from app.models.metrics import Metrics
+from app.models.settings import Settings
 from app.models.user import User
 from app.tenancy import ensure_default_project_for_user
 from app.tasks.buildroot_image import BUILDROOT_SD_IMAGE_CUSTOMIZATION_VERSION, buildroot_sd_image_config_fingerprint
@@ -1167,6 +1168,47 @@ async def fake_buildroot_sd_image(id, _redis, *, request_id=None, queue_job_id=N
     assert frame.agent['deployWithAgent'] is True
 
 
+@pytest.mark.asyncio
+async def test_api_frame_buildroot_sd_image_accepts_configured_build_host(async_client, db, redis, monkeypatch):
+    import app.tasks.buildroot_image as buildroot_image_module
+
+    frame = await new_frame(db, redis, 'BuildrootFrame', 'frame.local', 'backend.local')
+    frame.mode = 'buildroot'
+    frame.network = {
+        **(frame.network or {}),
+        'wifiSSID': 'Test WiFi',
+        'wifiPassword': 'secret1234',
+    }
+    frame.buildroot = {'platform': 'raspberry-pi-zero-2-w'}
+    db.add(Settings(project_id=frame.project_id, key='buildEnvironment', value={'provider': 'buildHost'}))
+    db.add(
+        Settings(
+            project_id=frame.project_id,
+            key='buildHost',
+            value={
+                'enabled': True,
+                'host': 'builder.local',
+                'user': 'ubuntu',
+                'sshKey': 'dummy-key',
+            },
+        )
+    )
+    db.add(frame)
+    db.commit()
+    captured: list[int] = []
+
+    async def fake_buildroot_sd_image(id, _redis, *, request_id=None, queue_job_id=None):
+        captured.append(id)
+
+    monkeypatch.setattr(buildroot_image_module, "buildroot_sd_image", fake_buildroot_sd_image)
+
+    response = await async_client.post(f'/api/frames/{frame.id}/buildroot/sd_image')
+
+    assert response.status_code == 200
+    assert response.json()['message'] == 'Buildroot SD card image preparation started'
+    assert captured == [frame.id]
+
+
 @pytest.mark.asyncio
 async def test_api_frame_buildroot_sd_image_does_not_publish_previous_error(async_client, db, redis, monkeypatch):
     import app.api.frames as frames_api_module