Skip to content

Commit 4a8a130

Browse files
mariusandramariusandra
committed
Modernize scene compilation and cross-build handling
1 parent 71d2bd0 commit 4a8a130

32 files changed

Lines changed: 1088 additions & 83 deletions

‎.github/workflows/frameos-cross-toolchain.yml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: Publish FrameOS cross-toolchain images
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- backend/bin/cross
9+
- backend/tools/cross-toolchain.Dockerfile
10+
- .github/workflows/frameos-cross-toolchain.yml
11+
workflow_dispatch:
12+
inputs:
13+
image_tag:
14+
description: "Toolchain image tag to publish"
15+
required: false
16+
default: latest
17+
type: string
18+
19+
permissions:
20+
contents: write
21+
22+
jobs:
23+
determine-matrix:
24+
runs-on: ubuntu-24.04
25+
outputs:
26+
targets: ${{ steps.set-matrix.outputs.targets }}
27+
steps:
28+
- uses: actions/checkout@v4
29+
- id: set-matrix
30+
run: |
31+
MATRIX=$(python3 backend/bin/cross matrix)
32+
echo "targets=$MATRIX" >> "$GITHUB_OUTPUT"
33+
34+
publish:
35+
needs: determine-matrix
36+
runs-on: ubuntu-24.04
37+
steps:
38+
- uses: actions/checkout@v4
39+
- uses: docker/setup-qemu-action@v3
40+
- uses: docker/setup-buildx-action@v3
41+
- name: Login to Docker Hub
42+
uses: docker/login-action@v3
43+
with:
44+
username: ${{ secrets.DOCKERHUB_USERNAME }}
45+
password: ${{ secrets.DOCKERHUB_TOKEN }}
46+
- name: Build and push cross-toolchain images
47+
env:
48+
IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}
49+
MATRIX: ${{ needs.determine-matrix.outputs.targets }}
50+
run: |
51+
set -euo pipefail
52+
python3 - <<'PY'
53+
from __future__ import annotations
54+
55+
import json
56+
import os
57+
import re
58+
import subprocess
59+
from pathlib import Path
60+
61+
def sanitize(value: str) -> str:
62+
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)
63+
64+
matrix = json.loads(os.environ["MATRIX"])
65+
image_repo = "frameos/frameos-cross-toolchain"
66+
image_tag = os.environ.get("IMAGE_TAG", "latest").strip() or "latest"
67+
digest_data = {"images": {}}
68+
69+
for index, target in enumerate(matrix["include"]):
70+
base_image = target["image"]
71+
safe_base = sanitize(base_image.replace("/", "_"))
72+
safe_platform = sanitize(target["platform"].replace("/", "_"))
73+
image = f"{image_repo}:{safe_base}-{safe_platform}-{image_tag}"
74+
metadata_file = Path(f"/tmp/toolchain-metadata-{index}.json")
75+
76+
subprocess.run(
77+
[
78+
"docker",
79+
"buildx",
80+
"build",
81+
"--platform",
82+
target["platform"],
83+
"--build-arg",
84+
f"BASE_IMAGE={base_image}",
85+
"--tag",
86+
image,
87+
"--push",
88+
"--metadata-file",
89+
str(metadata_file),
90+
"-f",
91+
"backend/tools/cross-toolchain.Dockerfile",
92+
".",
93+
],
94+
check=True,
95+
)
96+
97+
metadata = json.loads(metadata_file.read_text(encoding="utf-8"))
98+
digest = (metadata.get("containerimage") or {}).get("digest")
99+
if not digest:
100+
raise RuntimeError(f"Missing container image digest for {image}")
101+
102+
digest_data["images"][image] = {
103+
"digest": digest,
104+
"platform": target["platform"],
105+
"base_image": base_image,
106+
"tag": image_tag,
107+
}
108+
109+
Path("cross-toolchain-images.json").write_text(
110+
json.dumps(digest_data, indent=2, sort_keys=True) + "\n",
111+
encoding="utf-8",
112+
)
113+
PY
114+
- name: Commit updated cross-toolchain digests
115+
if: github.event_name == 'push'
116+
run: |
117+
set -euo pipefail
118+
git add cross-toolchain-images.json
119+
if git diff --cached --quiet; then
120+
echo "No cross-toolchain digest updates to commit."
121+
exit 0
122+
fi
123+
git config user.name "github-actions[bot]"
124+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
125+
git commit -m "chore: refresh cross-toolchain image digests [skip ci]"
126+
git push

‎.github/workflows/frameos-cross.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,17 @@ name: FrameOS cross compilation
22

33
on:
44
workflow_dispatch:
5+
inputs:
6+
toolchain_image_tag:
7+
description: "Cross-toolchain image tag (optional override)"
8+
required: false
9+
default: ""
10+
type: string
11+
toolchain_image:
12+
description: "Cross-toolchain image override (optional)"
13+
required: false
14+
default: ""
15+
type: string
516
pull_request:
617
paths:
718
- frameos/**
@@ -28,6 +39,9 @@ jobs:
2839
cross-build:
2940
needs: determine-matrix
3041
runs-on: ${{ matrix.runner }}
42+
env:
43+
FRAMEOS_CROSS_TOOLCHAIN_IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.toolchain_image_tag || '' }}
44+
FRAMEOS_CROSS_TOOLCHAIN_IMAGE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.toolchain_image || '' }}
3145
strategy:
3246
fail-fast: false
3347
matrix: ${{ fromJson(needs.determine-matrix.outputs.targets) }}
@@ -80,6 +94,8 @@ jobs:
8094
needs: determine-matrix
8195
runs-on: ubuntu-24.04
8296
env:
97+
FRAMEOS_CROSS_TOOLCHAIN_IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.toolchain_image_tag || '' }}
98+
FRAMEOS_CROSS_TOOLCHAIN_IMAGE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.toolchain_image || '' }}
8399
TARGET_SLUG: debian-trixie-amd64
84100
TARGET_PLATFORM: linux/amd64
85101
TARGET_IMAGE: debian:trixie

‎README.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,32 @@ docker run -d -p 8989:8989 \
121121
-e TMPDIR=/tmp/frameos-cross \
122122
frameos
123123
```
124+
125+
### Cross-toolchain build container images
126+
127+
Cross-compilation uses prebuilt toolchain containers from Docker Hub at `frameos/frameos-cross-toolchain` when possible, which avoids rebuilding the toolchain image for every target.
128+
129+
The workflow `.github/workflows/frameos-cross-toolchain.yml` builds and publishes these images.
130+
131+
The image name is resolved as:
132+
133+
- `{repo}:{base}_{version}-{platform}-{tag}`
134+
- `base` is the Linux distro (`debian`, `ubuntu`, ...)
135+
- `version` is the distro version (for example `bookworm` or `26.04`)
136+
- `platform` is the docker platform with `/` replaced by `_` (for example `linux_amd64`, `linux_arm64`, `linux_arm_v7`)
137+
- `tag` defaults to `latest`
138+
139+
You can override the default behavior with environment variables:
140+
141+
- `FRAMEOS_CROSS_TOOLCHAIN_IMAGE`: full Docker image override (can be a Python format template using `slug`, `base`, `platform`, and `tag`)
142+
- `FRAMEOS_CROSS_TOOLCHAIN_IMAGE_REPO`: image repository (default `frameos/frameos-cross-toolchain`)
143+
- `FRAMEOS_CROSS_TOOLCHAIN_IMAGE_TAG`: image tag used by the default resolver (default `latest`)
144+
- `FRAMEOS_CROSS_TOOLCHAIN_FORCE_LOCAL_BUILD=1`: force rebuilding the toolchain image locally, even if a remote tag exists
145+
- `FRAMEOS_CROSS_TOOLCHAIN_SKIP_PULL=1`: skip pulling remote images and only use local/locally built images
146+
147+
Example for local iteration on a new toolchain image:
148+
149+
```bash
150+
export FRAMEOS_CROSS_TOOLCHAIN_IMAGE=frameos/frameos-cross-toolchain:debian_trixie-linux_arm_v7-my-wip
151+
export FRAMEOS_CROSS_TOOLCHAIN_FORCE_LOCAL_BUILD=1
152+
```

‎backend/app/codegen/drivers_nim.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77

88
COMPILATION_MODE_STATIC = "static"
99
COMPILATION_MODE_SHARED = "shared"
10+
COMPILATION_MODE_SHARED_SCENES = "shared-scenes"
1011
COMPILATION_MODE_PRECOMPILED = "precompiled"
1112
DEFAULT_COMPILATION_MODE = COMPILATION_MODE_PRECOMPILED
1213
VALID_COMPILATION_MODES = {
1314
COMPILATION_MODE_STATIC,
1415
COMPILATION_MODE_SHARED,
16+
COMPILATION_MODE_SHARED_SCENES,
1517
COMPILATION_MODE_PRECOMPILED,
1618
}
1719

@@ -31,6 +33,7 @@ def frame_compilation_mode(frame) -> str:
3133
def compilation_mode_uses_shared_libraries(value: str | None) -> bool:
3234
return normalize_compilation_mode(value) in {
3335
COMPILATION_MODE_SHARED,
36+
COMPILATION_MODE_SHARED_SCENES,
3437
COMPILATION_MODE_PRECOMPILED,
3538
}
3639

‎backend/app/codegen/release_drivers_nim.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,14 @@ def write_release_shared_drivers_nim(drivers: dict[str, Driver]) -> str:
8383
if spec_lines:
8484
spec_lines = newline + " " + spec_lines + newline
8585

86+
setup_imports, setup_local_code, _setup_names = setup_helpers_nim(
87+
drivers,
88+
include_compiled_drivers=False,
89+
setup_proc_name="setupLocalDrivers",
90+
setup_proc_exported=False,
91+
include_setup_driver_names=False,
92+
)
93+
8694
return f"""
8795
import std/[dynlib, json, options, os, strutils]
8896
import pixie
@@ -91,6 +99,7 @@ def write_release_shared_drivers_nim(drivers: dict[str, Driver]) -> str:
9199
import frameos/device_setup
92100
import frameos/channels as hostChannels
93101
import frameos/driver_abi
102+
{newline.join(setup_imports)}
94103
95104
type
96105
DriverSpec = object
@@ -267,8 +276,11 @@ def write_release_shared_drivers_nim(drivers: dict[str, Driver]) -> str:
267276
result = @[]
268277
269278
proc setup*(frameOS: FrameOS): SetupResult =
279+
addSetupResult(result, setupLocalDrivers(frameOS))
270280
addSetupResult(result, setupSharedDrivers(frameOS))
271281
282+
{setup_local_code}
283+
272284
proc init*(frameOS: FrameOS) =
273285
loadedDrivers = @[]
274286
let driverCtx = buildDriverContext(frameOS)

0 commit comments

Comments
 (0)