Add shared scene bundle mode and precompiled fallback

Author: mariusandra

backend/app/codegen/drivers_nim.py

@@ -7,11 +7,13 @@
 
 COMPILATION_MODE_STATIC = "static"
 COMPILATION_MODE_SHARED = "shared"
+COMPILATION_MODE_SHARED_SCENES = "shared-scenes"
 COMPILATION_MODE_PRECOMPILED = "precompiled"
 DEFAULT_COMPILATION_MODE = COMPILATION_MODE_PRECOMPILED
 VALID_COMPILATION_MODES = {
     COMPILATION_MODE_STATIC,
     COMPILATION_MODE_SHARED,
+    COMPILATION_MODE_SHARED_SCENES,
     COMPILATION_MODE_PRECOMPILED,
 }
 
@@ -31,6 +33,7 @@ def frame_compilation_mode(frame) -> str:
 def compilation_mode_uses_shared_libraries(value: str | None) -> bool:
     return normalize_compilation_mode(value) in {
         COMPILATION_MODE_SHARED,
+        COMPILATION_MODE_SHARED_SCENES,
         COMPILATION_MODE_PRECOMPILED,
     }
 

backend/app/codegen/scene_nim.py

@@ -9,6 +9,7 @@
 from app.models.apps import get_local_frame_apps, get_local_app_path, get_scene_app_id
 from app.codegen.drivers_nim import (
     DEFAULT_COMPILATION_MODE,
+    COMPILATION_MODE_SHARED_SCENES,
     compilation_mode_uses_shared_libraries,
 )
 from app.codegen.utils import sanitize_nim_string, natural_keys
@@ -68,6 +69,22 @@ def scene_library_filename(scene: dict) -> str:
     return f"scene_{scene_module_suffix(scene)}.so"
 
 
+def scene_bundle_library_filename() -> str:
+    return "scenes.so"
+
+
+def _scene_symbol_suffix(scene: dict) -> str:
+    return scene_module_suffix(scene).replace(".", "_").replace("-", "_")
+
+
+def _scene_bundle_init_symbol(scene: dict) -> str:
+    return f"frameos_scene_init_{_scene_symbol_suffix(scene)}"
+
+
+def _scene_bundle_export_symbol(scene: dict) -> str:
+    return f"frameos_scene_export_{_scene_symbol_suffix(scene)}"
+
+
 def write_scene_library_nim(scene: dict) -> str:
     scene_module = f"scene_{scene_module_suffix(scene)}"
     return f"""# This file is autogenerated
@@ -1842,7 +1859,141 @@ def write_shared_scenes_nim(frame: Frame) -> str:
     return scenes_source
 
 
+def write_shared_scenes_bundle_nim(frame: Frame) -> str:
+    compiled_scenes = compiled_frame_scenes(frame)
+    scene_specs = [
+        "SceneBundleSpec("
+        f'id: "{scene_registry_id(scene)}".SceneId, '
+        f'name: "{sanitize_nim_string(scene.get("name", "Default"))}", '
+        f'libraryName: "{scene_bundle_library_filename()}", '
+        f'initSymbol: "{_scene_bundle_init_symbol(scene)}", '
+        f'exportSymbol: "{_scene_bundle_export_symbol(scene)}"'
+        ")"
+        for scene in compiled_scenes
+    ]
+    default_scene = next((scene for scene in compiled_scenes if scene.get("default", False)), None)
+    sceneOptionTuples = [
+        f'  ("{scene_registry_id(scene)}".SceneId, "{sanitize_nim_string(scene.get("name", "Default"))}"),'
+        for scene in compiled_scenes
+    ]
+    spec_lines = ("," + "\n" + "  ").join(scene_specs)
+    if spec_lines:
+        spec_lines = "\n  " + spec_lines + "\n"
+    newline = "\n"
+
+    bundle_imports = "\n".join(
+        f'import scenes.scene_{scene_module_suffix(scene)} as scene_{scene_module_suffix(scene)}'
+        for scene in compiled_scenes
+    )
+    if not bundle_imports:
+        bundle_imports = "# no compiled scenes"
+
+    wrapper_exports = ""
+    for scene in compiled_scenes:
+        wrapper_exports += f"""
+proc {_scene_bundle_init_symbol(scene)}*(logHook: HostLogProc, sendEventHook: HostSendEventProc) {{.cdecl, exportc, dynlib.}} =
+  scene_{scene_module_suffix(scene)}.frameos_scene_init(logHook, sendEventHook)
+
+proc {_scene_bundle_export_symbol(scene)}*(): pointer {{.cdecl, exportc, dynlib.}} =
+  result = cast[pointer](scene_{scene_module_suffix(scene)}.exportedScene)
+"""
+
+    scenes_source = f"""
+import std/[dynlib, json, options, os, tables]
+import frameos/types
+import frameos/channels as hostChannels
+import frameos/driver_abi
+
+{bundle_imports}
+
+type
+  SceneBundleSpec = object
+    id: SceneId
+    name: string
+    libraryName: string
+    initSymbol: string
+    exportSymbol: string
+  LoadedSceneLibrary = object
+    spec: SceneBundleSpec
+    library: LibHandle
+    exportedScene: ExportedScene
+
+const sceneSpecs*: seq[SceneBundleSpec] = @[{spec_lines}]
+
+{_default_scene_line(default_scene)}
+
+const sceneOptions*: array[{len(sceneOptionTuples)}, tuple[id: SceneId, name: string]] = [
+{newline.join(sorted(sceneOptionTuples))}
+]
+var loadedSceneLibraries: seq[LoadedSceneLibrary] = @[]
+
+type
+  SceneInitProc = proc(logHook: HostLogProc, sendEventHook: HostSendEventProc) {{.cdecl.}}
+  SceneExportProc = proc(): pointer {{.cdecl.}}
+
+proc hostLog(event: JsonNode) {{.cdecl, gcsafe.}} =
+  hostChannels.log(event)
+
+proc hostSendEvent(scene: Option[SceneId], event: string, payload: JsonNode) {{.cdecl, gcsafe.}} =
+  hostChannels.sendEvent(scene, event, payload)
+
+proc sceneLibraryPath(spec: SceneBundleSpec): string =
+  getAppDir() / "scenes" / spec.libraryName
+
+proc loadRequiredSymbol[T](library: LibHandle, sceneId: SceneId, symbol: string): T =
+  let address = symAddr(library, symbol)
+  if address.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": sceneId.string,
+        "error": "Missing symbol", "symbol": symbol}})
+    return nil
+  cast[T](address)
+
+proc loadSharedScene(spec: SceneBundleSpec): Option[ExportedScene] =
+  let path = sceneLibraryPath(spec)
+  let library = loadLib(path)
+  if library.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": spec.id.string,
+        "error": "Unable to load scene library", "path": path}})
+    return none(ExportedScene)
+
+  let initProc = loadRequiredSymbol[SceneInitProc](library, spec.id, spec.initSymbol)
+  if initProc.isNil:
+    unloadLib(library)
+    return none(ExportedScene)
+  initProc(hostLog, hostSendEvent)
+
+  let exportProc = loadRequiredSymbol[SceneExportProc](library, spec.id, spec.exportSymbol)
+  if exportProc.isNil:
+    unloadLib(library)
+    return none(ExportedScene)
+
+  let exportedScene = cast[ExportedScene](exportProc())
+  if exportedScene.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": spec.id.string,
+        "error": "Scene library returned nil export", "path": path}})
+    unloadLib(library)
+    return none(ExportedScene)
+
+  loadedSceneLibraries.add(LoadedSceneLibrary(spec: spec, library: library, exportedScene: exportedScene))
+  hostChannels.log(%*{{"event": "scene:shared", "sceneId": spec.id.string, "path": path, "loaded": true}})
+  return some(exportedScene)
+
+proc getExportedScenes*(): Table[SceneId, ExportedScene] =
+  result = initTable[SceneId, ExportedScene]()
+  for spec in sceneSpecs:
+    let exportedScene = loadSharedScene(spec)
+    if exportedScene.isSome:
+      result[spec.id] = exportedScene.get()
+
+{wrapper_exports}
+"""
+
+    return scenes_source
+
+
 def write_scenes_nim(frame: Frame, compilation_mode: str = DEFAULT_COMPILATION_MODE) -> str:
     if compilation_mode_uses_shared_libraries(compilation_mode):
+        if compilation_mode == COMPILATION_MODE_SHARED_SCENES:
+            return write_shared_scenes_bundle_nim(frame)
         return write_shared_scenes_nim(frame)
     return write_static_scenes_nim(frame)

backend/app/tasks/_frame_deployer.py

@@ -29,6 +29,7 @@
 from app.codegen.drivers_nim import (
     DEFAULT_COMPILATION_MODE,
     COMPILATION_MODE_STATIC,
+    COMPILATION_MODE_SHARED_SCENES,
     compiled_drivers,
     compilation_mode_uses_shared_libraries,
     driver_library_filename,
@@ -38,6 +39,7 @@
 )
 from app.codegen.scene_nim import (
     compiled_frame_scenes,
+    scene_bundle_library_filename,
     scene_library_filename,
     scene_module_filename,
     scene_module_suffix,
@@ -230,9 +232,14 @@ def scene_library_paths(
     ) -> list[str]:
         if not compilation_mode_uses_shared_libraries(compilation_mode):
             return []
+        compiled_scenes = compiled_frame_scenes(frame)
+        if not compiled_scenes:
+            return []
+        if compilation_mode == COMPILATION_MODE_SHARED_SCENES:
+            return [os.path.join(build_dir, "scenes", scene_bundle_library_filename())]
         return [
             os.path.join(build_dir, "scenes", scene_module_suffix(scene), scene_library_filename(scene))
-            for scene in compiled_frame_scenes(frame)
+            for scene in compiled_scenes
         ]
 
     @staticmethod
@@ -242,6 +249,10 @@ def scene_library_names(
     ) -> list[str]:
         if not compilation_mode_uses_shared_libraries(compilation_mode):
             return []
+        if not compiled_frame_scenes(frame):
+            return []
+        if compilation_mode == COMPILATION_MODE_SHARED_SCENES:
+            return [scene_bundle_library_filename()]
         return [scene_library_filename(scene) for scene in compiled_frame_scenes(frame)]
 
     async def _upload_frame_json(self, path: str) -> None:
@@ -438,6 +449,9 @@ async def make_local_modifications(
             for scene in compiled_frame_scenes(frame):
                 with open(os.path.join(shared_scene_dir, scene_module_filename(scene)), "w") as sf:
                     sf.write(write_scene_library_nim(scene))
+            if compilation_mode == COMPILATION_MODE_SHARED_SCENES:
+                with open(os.path.join(source_dir, "src", "scenes", "scenes_bundle.nim"), "w") as bf:
+                    bf.write(write_scenes_nim(frame, compilation_mode=COMPILATION_MODE_SHARED_SCENES))
 
         with open(os.path.join(source_dir, "src", "scenes", "scenes.nim"), "w") as f:
             source = write_scenes_nim(frame, compilation_mode=compilation_mode)
@@ -828,6 +842,7 @@ async def create_local_build_archive(
         )
 
         if compilation_mode_uses_shared_libraries(compilation_mode):
+            compiled_scenes = compiled_frame_scenes(frame)
             for driver in compiled_drivers(drivers):
                 driver_dir = os.path.join(build_dir, "drivers", driver.name)
                 os.makedirs(driver_dir, exist_ok=True)
@@ -866,23 +881,57 @@ async def create_local_build_archive(
                 )
                 driver_make_dirs.append(os.path.join("drivers", driver.name))
 
-            for scene in compiled_frame_scenes(frame):
-                scene_dir_name = scene_module_suffix(scene)
-                scene_dir = os.path.join(build_dir, "scenes", scene_dir_name)
+            if compilation_mode == COMPILATION_MODE_SHARED:
+                for scene in compiled_scenes:
+                    scene_dir_name = scene_module_suffix(scene)
+                    scene_dir = os.path.join(build_dir, "scenes", scene_dir_name)
+                    os.makedirs(scene_dir, exist_ok=True)
+                    output_name = scene_library_filename(scene)
+                    await self.log("stdout", f"🔥 Generating C sources for scene {scene.get('id', 'default')}.")
+                    scene_cmd = (
+                        f"cd {source_dir} && {nim_path} compile --app:lib --os:linux --cpu:{cpu} "
+                        f"--define:frameosSharedLibrary {' '.join(SHARED_LIBRARY_NIM_FLAGS)} "
+                        f"--compileOnly --genScript --nimcache:{scene_dir} --out:{output_name} "
+                        f"{debug_options} src/scenes/shared/{scene_module_filename(scene)} 2>&1"
+                    )
+                    scene_status, scene_out, scene_err = await exec_local_command(db, redis, frame, scene_cmd)
+                    if scene_status != 0:
+                        raise Exception(
+                            f"Failed to generate scene library sources for {scene.get('id', 'default')}: "
+                            f"{scene_err or scene_out or 'see logs'}"
+                        )
+                    shutil.copy(nimbase_path, os.path.join(scene_dir, "nimbase.h"))
+
+                    scene_script_path = self._find_compile_script(scene_dir)
+                    scene_linker_flags, scene_compiler_flags = self._extract_compile_flags(
+                        scene_script_path, output_name
+                    )
+                    scene_linker_flags = self._dedupe_preserve_order(
+                        scene_linker_flags + ["../../quickjs/libquickjs.a"]
+                    )
+                    self._write_driver_makefile(
+                        makefile_path=os.path.join(scene_dir, "Makefile"),
+                        output_name=output_name,
+                        linker_flags=scene_linker_flags,
+                        compiler_flags=scene_compiler_flags,
+                        library_kind="scene",
+                    )
+                    scene_make_dirs.append(os.path.join("scenes", scene_dir_name))
+            elif compilation_mode == COMPILATION_MODE_SHARED_SCENES and compiled_scenes:
+                scene_dir = os.path.join(build_dir, "scenes")
                 os.makedirs(scene_dir, exist_ok=True)
-                output_name = scene_library_filename(scene)
-                await self.log("stdout", f"🔥 Generating C sources for scene {scene.get('id', 'default')}.")
+                output_name = scene_bundle_library_filename()
+                await self.log("stdout", "🔥 Generating C sources for bundled shared scenes.")
                 scene_cmd = (
                     f"cd {source_dir} && {nim_path} compile --app:lib --os:linux --cpu:{cpu} "
                     f"--define:frameosSharedLibrary {' '.join(SHARED_LIBRARY_NIM_FLAGS)} "
                     f"--compileOnly --genScript --nimcache:{scene_dir} --out:{output_name} "
-                    f"{debug_options} src/scenes/shared/{scene_module_filename(scene)} 2>&1"
+                    f"{debug_options} src/scenes/scenes_bundle.nim 2>&1"
                 )
                 scene_status, scene_out, scene_err = await exec_local_command(db, redis, frame, scene_cmd)
                 if scene_status != 0:
                     raise Exception(
-                        f"Failed to generate scene library sources for {scene.get('id', 'default')}: "
-                        f"{scene_err or scene_out or 'see logs'}"
+                        f"Failed to generate bundled scene library sources: {scene_err or scene_out or 'see logs'}"
                     )
                 shutil.copy(nimbase_path, os.path.join(scene_dir, "nimbase.h"))
 
@@ -891,7 +940,7 @@ async def create_local_build_archive(
                     scene_script_path, output_name
                 )
                 scene_linker_flags = self._dedupe_preserve_order(
-                    scene_linker_flags + ["../../quickjs/libquickjs.a"]
+                    scene_linker_flags + ["../quickjs/libquickjs.a"]
                 )
                 self._write_driver_makefile(
                     makefile_path=os.path.join(scene_dir, "Makefile"),
@@ -900,7 +949,7 @@ async def create_local_build_archive(
                     compiler_flags=scene_compiler_flags,
                     library_kind="scene",
                 )
-                scene_make_dirs.append(os.path.join("scenes", scene_dir_name))
+                scene_make_dirs.append("scenes")
 
         self._write_c_makefile(
             makefile_path=os.path.join(build_dir, "Makefile"),

backend/app/tasks/binary_builder.py

@@ -16,6 +16,7 @@
 from app.codegen.drivers_nim import (
     COMPILATION_MODE_PRECOMPILED,
     COMPILATION_MODE_STATIC,
+    COMPILATION_MODE_SHARED_SCENES,
     frame_compilation_mode,
     normalize_compilation_mode,
 )
@@ -211,7 +212,9 @@ async def plan_build(
             else:
                 will_attempt_precompiled = True
             if not will_attempt_precompiled:
-                resolved_compilation_mode = COMPILATION_MODE_STATIC
+                resolved_compilation_mode = (
+                    COMPILATION_MODE_SHARED_SCENES if compiled_scene_count > 0 else COMPILATION_MODE_STATIC
+                )
 
         build_host = get_build_host_config(self.db)
         cross_compile_supported = can_cross_compile_target(target.arch)

backend/app/tasks/frame_deploy_workflow.py

@@ -495,14 +495,19 @@ async def _plan_full(self, *, frame_dict: dict[str, Any], previous_frameos_versi
             if binary_plan.will_attempt_precompiled:
                 notes.append("Precompiled FrameOS release will be used because all scenes are interpreted.")
             else:
+                fallback = "single executable"
+                if binary_plan.compilation_mode == "shared-scenes":
+                    fallback = "compiled scenes library"
+                elif binary_plan.compilation_mode == "shared":
+                    fallback = "shared libraries"
                 notes.append(
                     "Precompiled FrameOS release will be skipped"
                     + (
                         f": {binary_plan.precompiled_skip_reason}."
                         if binary_plan.precompiled_skip_reason
                         else "."
                     )
-                    + " Falling back to single executable."
+                    + f" Falling back to {fallback}."
                 )
         if low_memory and not binary_plan.will_attempt_precompiled:
             notes.append("Device is low memory; on-device build path will stop FrameOS before compilation.")