Rework model storage directory management (for external storage) (#1765)

## Motivation

Replace confusing EXO_MODELS_DIR/EXO_MODELS_PATH with clearer
multi-directory support, enabling automatic download spillover across
volumes.

## Changes

- EXO_MODELS_DIRS: colon-separated writable dirs (default always
prepended, first with enough space wins)
- EXO_MODELS_READ_ONLY_DIRS: colon-separated read-only dirs (protected
from deletion)
- select_download_dir(): picks writable dir by free space
- resolve_existing_model(): unified lookup across all dirs
- is_read_only_model_dir(): path-based read-only detection instead of
hardcoded flag
- Updated coordinator, worker, model cards, tests

## Why It Works

Default dir always included so zero-config behavior is unchanged. Disk
space checked at download time for automatic spillover. Read-only status
derived from path, not hardcoded.

## Test Plan

### Manual Testing

- No env vars set → identical behavior
- EXO_MODELS_DIRS=/Volumes/SSD/models → downloads to external storage
- EXO_MODELS_READ_ONLY_DIRS=/mnt/nfs → models found, deletion blocked

### Automated Testing

- 4 new tests in test_xdg_paths.py (prepend, default-only, overlap,
empty read-only)
- Existing tests updated to patch new constants

Author: ciaranbor

README.md

@@ -295,8 +295,9 @@ exo supports several environment variables for configuration:
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `EXO_MODELS_PATH` | Colon-separated paths to search for pre-downloaded models (e.g., on NFS mounts or shared storage) | None |
-| `EXO_MODELS_DIR` | Directory where exo downloads and stores models | `~/.local/share/exo/models` (Linux) or `~/.exo/models` (macOS) |
+| `EXO_DEFAULT_MODELS_DIR` | Default directory for model downloads and caches. Always first in the writable dirs list. | `~/.local/share/exo/models` (Linux) or `~/.exo/models` (macOS) |
+| `EXO_MODELS_DIRS` | Colon-separated additional writable directories for model downloads. Checked in order after the default; first with enough free space is used. | None |
+| `EXO_MODELS_READ_ONLY_DIRS` | Colon-separated read-only directories to search for pre-downloaded models (e.g., NFS mounts, shared storage). Models here cannot be deleted. | None |
 | `EXO_OFFLINE` | Run without internet connection (uses only local models) | `false` |
 | `EXO_ENABLE_IMAGE_MODELS` | Enable image model support | `false` |
 | `EXO_LIBP2P_NAMESPACE` | Custom namespace for cluster isolation | None |
@@ -306,8 +307,11 @@ exo supports several environment variables for configuration:
 **Example usage:**
 
 ```bash
-# Use pre-downloaded models from NFS mount
-EXO_MODELS_PATH=/mnt/nfs/models:/opt/ai-models uv run exo
+# Use pre-downloaded models from NFS mount (read-only)
+EXO_MODELS_READ_ONLY_DIRS=/mnt/nfs/models:/opt/ai-models uv run exo
+
+# Download models to an external SSD (falls back to default dir if full)
+EXO_MODELS_DIRS=/Volumes/ExternalSSD/exo-models uv run exo
 
 # Run in offline mode
 EXO_OFFLINE=true uv run exo

bench/harness.py

@@ -377,7 +377,7 @@ def run_planning_phase(
                 f"have {avail // (1024**3)}GB. Use --danger-delete-downloads to free space."
             )
 
-        # Delete from smallest to largest (skip read-only models from EXO_MODELS_PATH)
+        # Delete from smallest to largest (skip read-only models)
         completed = [
             (
                 unwrap_instance(p["DownloadCompleted"]["shardMetadata"])["modelCard"][

src/exo/download/coordinator.py

@@ -1,17 +1,21 @@
+from __future__ import annotations
+
 from dataclasses import dataclass, field
+from pathlib import Path
 
 import anyio
-from anyio import current_time
+from anyio import current_time, to_thread
 from loguru import logger
 
 from exo.download.download_utils import (
     RepoDownloadProgress,
     delete_model,
+    is_read_only_model_dir,
     map_repo_download_progress_to_download_progress_data,
-    resolve_model_in_path,
+    resolve_existing_model,
 )
 from exo.download.shard_downloader import ShardDownloader
-from exo.shared.constants import EXO_MODELS_DIR, EXO_MODELS_PATH
+from exo.shared.constants import EXO_DEFAULT_MODELS_DIR, EXO_MODELS_READ_ONLY_DIRS
 from exo.shared.models.model_cards import ModelId, get_model_cards
 from exo.shared.types.commands import (
     CancelDownload,
@@ -24,6 +28,7 @@
     Event,
     NodeDownloadProgress,
 )
+from exo.shared.types.memory import Memory
 from exo.shared.types.worker.downloads import (
     DownloadCompleted,
     DownloadFailed,
@@ -57,8 +62,23 @@ class DownloadCoordinator:
     def __post_init__(self) -> None:
         self.shard_downloader.on_progress(self._download_progress_callback)
 
-    def _model_dir(self, model_id: ModelId) -> str:
-        return str(EXO_MODELS_DIR / model_id.normalize())
+    @staticmethod
+    def _default_model_dir(model_id: ModelId) -> str:
+        return str(EXO_DEFAULT_MODELS_DIR / model_id.normalize())
+
+    def _completed_from_path(
+        self,
+        shard: ShardMetadata,
+        found: Path,
+        total: Memory,
+    ) -> DownloadCompleted:
+        return DownloadCompleted(
+            shard_metadata=shard,
+            node_id=self.node_id,
+            total=total,
+            model_directory=str(found),
+            read_only=is_read_only_model_dir(found),
+        )
 
     async def _download_progress_callback(
         self, callback_shard: ShardMetadata, progress: RepoDownloadProgress
@@ -67,12 +87,18 @@ async def _download_progress_callback(
         throttle_interval_secs = 1.0
 
         if progress.status == "complete":
-            completed = DownloadCompleted(
-                shard_metadata=callback_shard,
-                node_id=self.node_id,
-                total=progress.total,
-                model_directory=self._model_dir(model_id),
-            )
+            found = await to_thread.run_sync(resolve_existing_model, model_id)
+            if found is not None:
+                completed = self._completed_from_path(
+                    callback_shard, found, progress.total
+                )
+            else:
+                completed = DownloadCompleted(
+                    shard_metadata=callback_shard,
+                    node_id=self.node_id,
+                    total=progress.total,
+                    model_directory=self._default_model_dir(model_id),
+                )
             self.download_status[model_id] = completed
             await self.event_sender.send(
                 NodeDownloadProgress(download_progress=completed)
@@ -89,7 +115,7 @@ async def _download_progress_callback(
                 download_progress=map_repo_download_progress_to_download_progress_data(
                     progress
                 ),
-                model_directory=self._model_dir(model_id),
+                model_directory=self._default_model_dir(model_id),
             )
             self.download_status[model_id] = ongoing
             await self.event_sender.send(
@@ -135,7 +161,7 @@ async def _cancel_download(self, model_id: ModelId) -> None:
             pending = DownloadPending(
                 shard_metadata=current_status.shard_metadata,
                 node_id=self.node_id,
-                model_directory=self._model_dir(model_id),
+                model_directory=self._default_model_dir(model_id),
             )
             self.download_status[model_id] = pending
             await self.event_sender.send(
@@ -154,18 +180,12 @@ async def _start_download(self, shard: ShardMetadata) -> None:
                 )
                 return
 
-        # Check EXO_MODELS_PATH for pre-downloaded models
-        found_path = resolve_model_in_path(model_id)
+        # Check all model directories for pre-existing complete models
+        found_path = await to_thread.run_sync(resolve_existing_model, model_id)
         if found_path is not None:
-            logger.info(
-                f"DownloadCoordinator: Model {model_id} found in EXO_MODELS_PATH at {found_path}"
-            )
-            completed = DownloadCompleted(
-                shard_metadata=shard,
-                node_id=self.node_id,
-                total=shard.model_card.storage_size,
-                model_directory=str(found_path),
-                read_only=True,
+            logger.info(f"DownloadCoordinator: Model {model_id} found at {found_path}")
+            completed = self._completed_from_path(
+                shard, found_path, shard.model_card.storage_size
             )
             self.download_status[model_id] = completed
             await self.event_sender.send(
@@ -177,7 +197,7 @@ async def _start_download(self, shard: ShardMetadata) -> None:
         progress = DownloadPending(
             shard_metadata=shard,
             node_id=self.node_id,
-            model_directory=self._model_dir(model_id),
+            model_directory=self._default_model_dir(model_id),
         )
         self.download_status[model_id] = progress
         await self.event_sender.send(NodeDownloadProgress(download_progress=progress))
@@ -188,12 +208,18 @@ async def _start_download(self, shard: ShardMetadata) -> None:
         )
 
         if initial_progress.status == "complete":
-            completed = DownloadCompleted(
-                shard_metadata=shard,
-                node_id=self.node_id,
-                total=initial_progress.total,
-                model_directory=self._model_dir(model_id),
-            )
+            found = await to_thread.run_sync(resolve_existing_model, model_id)
+            if found is not None:
+                completed = self._completed_from_path(
+                    shard, found, initial_progress.total
+                )
+            else:
+                completed = DownloadCompleted(
+                    shard_metadata=shard,
+                    node_id=self.node_id,
+                    total=initial_progress.total,
+                    model_directory=self._default_model_dir(model_id),
+                )
             self.download_status[model_id] = completed
             await self.event_sender.send(
                 NodeDownloadProgress(download_progress=completed)
@@ -208,7 +234,7 @@ async def _start_download(self, shard: ShardMetadata) -> None:
                 shard_metadata=shard,
                 node_id=self.node_id,
                 error_message=f"Model files not found locally in offline mode: {model_id}",
-                model_directory=self._model_dir(model_id),
+                model_directory=self._default_model_dir(model_id),
             )
             self.download_status[model_id] = failed
             await self.event_sender.send(NodeDownloadProgress(download_progress=failed))
@@ -229,7 +255,7 @@ def _start_download_task(
             download_progress=map_repo_download_progress_to_download_progress_data(
                 initial_progress
             ),
-            model_directory=self._model_dir(model_id),
+            model_directory=self._default_model_dir(model_id),
         )
         self.download_status[model_id] = status
         self.event_sender.send_nowait(NodeDownloadProgress(download_progress=status))
@@ -244,7 +270,7 @@ async def download_wrapper(cancel_scope: anyio.CancelScope) -> None:
                     shard_metadata=shard,
                     node_id=self.node_id,
                     error_message=str(e),
-                    model_directory=self._model_dir(model_id),
+                    model_directory=self._default_model_dir(model_id),
                 )
                 self.download_status[model_id] = failed
                 await self.event_sender.send(
@@ -261,13 +287,11 @@ async def download_wrapper(cancel_scope: anyio.CancelScope) -> None:
         self.active_downloads[model_id] = scope
 
     async def _delete_download(self, model_id: ModelId) -> None:
-        # Protect read-only models (from EXO_MODELS_PATH) from deletion
+        # Protect read-only models from deletion
         if model_id in self.download_status:
             current = self.download_status[model_id]
             if isinstance(current, DownloadCompleted) and current.read_only:
-                logger.warning(
-                    f"Refusing to delete read-only model {model_id} (from EXO_MODELS_PATH)"
-                )
+                logger.warning(f"Refusing to delete read-only model {model_id}")
                 return
 
         # Cancel if active
@@ -290,7 +314,7 @@ async def _delete_download(self, model_id: ModelId) -> None:
             pending = DownloadPending(
                 shard_metadata=current_status.shard_metadata,
                 node_id=self.node_id,
-                model_directory=self._model_dir(model_id),
+                model_directory=self._default_model_dir(model_id),
             )
             await self.event_sender.send(
                 NodeDownloadProgress(download_progress=pending)
@@ -314,22 +338,26 @@ async def _emit_existing_download_progress(self) -> None:
                         continue
 
                     if progress.status == "complete":
-                        status: DownloadProgress = DownloadCompleted(
-                            node_id=self.node_id,
-                            shard_metadata=progress.shard,
-                            total=progress.total,
-                            model_directory=self._model_dir(
-                                progress.shard.model_card.model_id
-                            ),
+                        found = await to_thread.run_sync(
+                            resolve_existing_model, model_id
                         )
+                        if found is not None:
+                            status: DownloadProgress = self._completed_from_path(
+                                progress.shard, found, progress.total
+                            )
+                        else:
+                            status = DownloadCompleted(
+                                node_id=self.node_id,
+                                shard_metadata=progress.shard,
+                                total=progress.total,
+                                model_directory=self._default_model_dir(model_id),
+                            )
                     elif progress.status in ["in_progress", "not_started"]:
                         if progress.downloaded_this_session.in_bytes == 0:
                             status = DownloadPending(
                                 node_id=self.node_id,
                                 shard_metadata=progress.shard,
-                                model_directory=self._model_dir(
-                                    progress.shard.model_card.model_id
-                                ),
+                                model_directory=self._default_model_dir(model_id),
                                 downloaded=progress.downloaded,
                                 total=progress.total,
                             )
@@ -340,9 +368,7 @@ async def _emit_existing_download_progress(self) -> None:
                                 download_progress=map_repo_download_progress_to_download_progress_data(
                                     progress
                                 ),
-                                model_directory=self._model_dir(
-                                    progress.shard.model_card.model_id
-                                ),
+                                model_directory=self._default_model_dir(model_id),
                             )
                     else:
                         continue
@@ -351,8 +377,8 @@ async def _emit_existing_download_progress(self) -> None:
                     await self.event_sender.send(
                         NodeDownloadProgress(download_progress=status)
                     )
-                # Scan EXO_MODELS_PATH for pre-downloaded models
-                if EXO_MODELS_PATH is not None:
+                # Scan read-only directories for pre-downloaded models
+                if EXO_MODELS_READ_ONLY_DIRS:
                     for card in await get_model_cards():
                         mid = card.model_id
                         if mid in self.active_downloads:
@@ -362,8 +388,8 @@ async def _emit_existing_download_progress(self) -> None:
                             (DownloadCompleted, DownloadOngoing, DownloadFailed),
                         ):
                             continue
-                        found = resolve_model_in_path(mid)
-                        if found is not None:
+                        found = await to_thread.run_sync(resolve_existing_model, mid)
+                        if found is not None and is_read_only_model_dir(found):
                             path_shard = PipelineShardMetadata(
                                 model_card=card,
                                 device_rank=0,
@@ -372,12 +398,10 @@ async def _emit_existing_download_progress(self) -> None:
                                 end_layer=card.n_layers,
                                 n_layers=card.n_layers,
                             )
-                            path_completed: DownloadProgress = DownloadCompleted(
-                                node_id=self.node_id,
-                                shard_metadata=path_shard,
-                                total=card.storage_size,
-                                model_directory=str(found),
-                                read_only=True,
+                            path_completed: DownloadProgress = (
+                                self._completed_from_path(
+                                    path_shard, found, card.storage_size
+                                )
                             )
                             self.download_status[mid] = path_completed
                             await self.event_sender.send(