feat: URL-to-idea ingestion + resource tracking (TDD, 31 tests)

New feature: generate project ideas from URLs. Paste a URL (article,
RFC, newsletter), Project Forge fetches it, extracts content, and
generates a scored project idea. Source domains are tracked as
"resources" for future idea generation.

New endpoints:
- POST /api/ideas/from-url — generate idea from URL content
- GET /api/resources — list tracked source domains
- POST /api/resources — add a domain as an idea source

New modules:
- engine/url_ingest.py — URL fetching, HTML parsing, domain extraction,
  UTM param stripping, URL validation
- Resource model + resources DB table with domain dedup
- source_url field on Idea model for provenance tracking

TDD cycle: 31 tests written first (RED), then implemented (GREEN),
lint-fixed (REFACTOR). 185/185 total tests passing, 0 regressions.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: rayketcham

src/project_forge/engine/url_ingest.py

@@ -0,0 +1,101 @@
+"""URL ingestion engine — fetch URLs, extract content, generate ideas."""
+
+import re
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urlencode, urlparse
+
+import httpx
+
+# Tracking parameters to strip from URLs
+TRACKING_PARAMS = {
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_content",
+    "utm_term",
+    "ref",
+    "fbclid",
+    "gclid",
+}
+
+
+class UrlFetchError(Exception):
+    """Raised when a URL cannot be fetched successfully."""
+
+
+@dataclass
+class UrlContent:
+    url: str
+    domain: str
+    title: str
+    text: str
+
+
+def validate_url(url: str) -> bool:
+    """Check if URL is valid http(s)."""
+    if not url:
+        return False
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def extract_domain(url: str) -> str:
+    """Extract clean domain from URL (strip www.)."""
+    parsed = urlparse(url)
+    domain = parsed.netloc
+    if domain.startswith("www."):
+        domain = domain[4:]
+    return domain
+
+
+def clean_url(url: str) -> str:
+    """Remove tracking parameters from URL."""
+    parsed = urlparse(url)
+    params = parse_qs(parsed.query)
+    clean_params = {k: v for k, v in params.items() if k not in TRACKING_PARAMS}
+    if clean_params:
+        clean_query = urlencode(clean_params, doseq=True)
+        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{clean_query}"
+    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+
+
+async def fetch_url_content(url: str) -> UrlContent:
+    """Fetch URL and extract content."""
+    async with httpx.AsyncClient(follow_redirects=True, timeout=30.0) as client:
+        response = await client.get(url)
+
+    if response.status_code >= 400:
+        raise UrlFetchError(f"HTTP {response.status_code} fetching {url}")
+
+    text = response.text
+    domain = extract_domain(url)
+
+    # Extract title from HTML and strip tags for text content
+    title = ""
+    content_type = response.headers.get("content-type", "")
+    if "html" in content_type:
+        title_match = re.search(r"<title[^>]*>(.*?)</title>", text, re.DOTALL | re.IGNORECASE)
+        if title_match:
+            title = title_match.group(1).strip()
+        # Strip script/style blocks first, then all other tags
+        text = re.sub(r"<script[^>]*>.*?</script>", "", text, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+
+    if not title:
+        title = domain  # Fallback to domain when no HTML title found
+
+    return UrlContent(url=url, domain=domain, title=title, text=text[:5000])
+
+
+async def generate_idea_from_url(content: UrlContent, category_hint=None):
+    """Generate an idea from URL content via IdeaGenerator."""
+    from project_forge.engine.generator import IdeaGenerator
+
+    generator = IdeaGenerator()
+    idea = await generator.generate_from_content(content, category_hint=category_hint)
+    return idea

src/project_forge/models.py

@@ -3,9 +3,10 @@
 from datetime import UTC, datetime
 from enum import StrEnum
 from typing import Literal
+from urllib.parse import urlparse
 from uuid import uuid4
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 
 class IdeaCategory(StrEnum):
@@ -41,6 +42,32 @@ class Idea(BaseModel):
     github_issue_url: str | None = None
     project_repo_url: str | None = None
     content_hash: str | None = None
+    source_url: str | None = None
+
+
+class Resource(BaseModel):
+    id: str = Field(default_factory=lambda: uuid4().hex[:12])
+    domain: str
+    name: str
+    description: str
+    url: str | None = None
+    categories: list[str] = Field(default_factory=list)
+    idea_count: int = 0
+    added_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class UrlIngestRequest(BaseModel):
+    url: str
+    category: str | None = None
+    notes: str | None = None
+
+    @field_validator("url")
+    @classmethod
+    def validate_url_format(cls, v: str) -> str:
+        parsed = urlparse(v)
+        if parsed.scheme not in ("http", "https") or not parsed.netloc:
+            raise ValueError(f"Invalid URL: {v!r} — must be http(s)")
+        return v
 
 
 class ScaffoldSpec(BaseModel):

src/project_forge/storage/db.py

@@ -10,7 +10,7 @@
 
 import aiosqlite
 
-from project_forge.models import GenerationRun, Idea, IdeaCategory, IdeaStatus
+from project_forge.models import GenerationRun, Idea, IdeaCategory, IdeaStatus, Resource
 
 SCHEMA = """
 CREATE TABLE IF NOT EXISTS ideas (
@@ -53,6 +53,17 @@
     used_at TEXT NOT NULL,
     PRIMARY KEY (category, concept_idx, domain_idx, direction)
 );
+
+CREATE TABLE IF NOT EXISTS resources (
+    id TEXT PRIMARY KEY,
+    domain TEXT NOT NULL UNIQUE,
+    name TEXT NOT NULL,
+    description TEXT NOT NULL,
+    url TEXT,
+    categories TEXT NOT NULL DEFAULT '[]',
+    idea_count INTEGER NOT NULL DEFAULT 0,
+    added_at TEXT NOT NULL
+);
 """
 
 
@@ -74,6 +85,11 @@ async def connect(self):
             await self._db.execute("ALTER TABLE ideas ADD COLUMN content_hash TEXT")
         except Exception:  # noqa: S110
             pass  # Column already exists on migrated DBs
+        # Migration: add source_url column if missing
+        try:
+            await self._db.execute("ALTER TABLE ideas ADD COLUMN source_url TEXT")
+        except Exception:  # noqa: S110
+            pass  # Column already exists on migrated DBs
         # Add indexes (safe to re-run)
         await self._db.execute(
             "CREATE UNIQUE INDEX IF NOT EXISTS idx_ideas_content_hash "
@@ -107,8 +123,8 @@ async def save_idea(self, idea: Idea) -> Idea:
             """INSERT OR REPLACE INTO ideas
             (id, name, tagline, description, category, market_analysis,
              feasibility_score, mvp_scope, tech_stack, generated_at, status,
-             github_issue_url, project_repo_url, content_hash)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+             github_issue_url, project_repo_url, content_hash, source_url)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
             (
                 idea.id,
                 idea.name,
@@ -124,6 +140,7 @@ async def save_idea(self, idea: Idea) -> Idea:
                 idea.github_issue_url,
                 idea.project_repo_url,
                 content_hash,
+                idea.source_url,
             ),
         )
         await self.db.commit()
@@ -300,6 +317,7 @@ async def get_stats(self) -> dict:
 
     @staticmethod
     def _row_to_idea(row) -> Idea:
+        keys = row.keys() if hasattr(row, "keys") else []
         return Idea(
             id=row["id"],
             name=row["name"],
@@ -316,4 +334,67 @@ def _row_to_idea(row) -> Idea:
             status=row["status"],
             github_issue_url=row["github_issue_url"],
             project_repo_url=row["project_repo_url"],
+            source_url=row["source_url"] if "source_url" in keys else None,
+        )
+
+    # === RESOURCE CRUD ===
+
+    async def save_resource(self, resource: Resource) -> Resource:
+        await self.db.execute(
+            """INSERT OR REPLACE INTO resources
+            (id, domain, name, description, url, categories, idea_count, added_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                resource.id,
+                resource.domain,
+                resource.name,
+                resource.description,
+                resource.url,
+                json.dumps(resource.categories),
+                resource.idea_count,
+                resource.added_at.isoformat(),
+            ),
+        )
+        await self.db.commit()
+        return resource
+
+    async def get_resource(self, resource_id: str) -> Resource | None:
+        cursor = await self.db.execute("SELECT * FROM resources WHERE id = ?", (resource_id,))
+        row = await cursor.fetchone()
+        if not row:
+            return None
+        return self._row_to_resource(row)
+
+    async def get_resource_by_domain(self, domain: str) -> Resource | None:
+        cursor = await self.db.execute("SELECT * FROM resources WHERE domain = ?", (domain,))
+        row = await cursor.fetchone()
+        if not row:
+            return None
+        return self._row_to_resource(row)
+
+    async def list_resources(self) -> list[Resource]:
+        cursor = await self.db.execute("SELECT * FROM resources ORDER BY added_at DESC")
+        rows = await cursor.fetchall()
+        return [self._row_to_resource(row) for row in rows]
+
+    async def increment_resource_idea_count(self, domain: str) -> None:
+        await self.db.execute(
+            "UPDATE resources SET idea_count = idea_count + 1 WHERE domain = ?",
+            (domain,),
+        )
+        await self.db.commit()
+
+    @staticmethod
+    def _row_to_resource(row) -> Resource:
+        return Resource(
+            id=row["id"],
+            domain=row["domain"],
+            name=row["name"],
+            description=row["description"],
+            url=row["url"],
+            categories=json.loads(row["categories"]),
+            idea_count=row["idea_count"],
+            added_at=datetime.fromisoformat(row["added_at"]).replace(tzinfo=UTC)
+            if "+" not in row["added_at"]
+            else datetime.fromisoformat(row["added_at"]),
         )

src/project_forge/web/app.py

@@ -66,6 +66,38 @@ async def lifespan(app: FastAPI):
 app.include_router(router)
 
 
+def create_app(db_path=None):
+    """Create a test-friendly app instance with an isolated database."""
+    from project_forge.storage.db import Database as DB
+
+    test_db = DB(db_path or settings.db_path)
+
+    @asynccontextmanager
+    async def test_lifespan(application: FastAPI):
+        await test_db.connect()
+        # Swap the module-level db reference so routes use the test DB
+        import project_forge.web.app as app_mod
+
+        old_db = app_mod.db
+        app_mod.db = test_db
+        # Also patch into routes module (imported from app)
+        import project_forge.web.routes as routes_mod
+
+        old_routes_db = routes_mod.db
+        routes_mod.db = test_db
+        yield
+        await test_db.close()
+        app_mod.db = old_db
+        routes_mod.db = old_routes_db
+
+    test_app = FastAPI(lifespan=test_lifespan)
+    test_app.add_middleware(CSPMiddleware)
+    from project_forge.web.routes import router as r
+
+    test_app.include_router(r)
+    return test_app
+
+
 def run():
     """Entry point for forge-serve command."""
     import uvicorn

src/project_forge/web/routes.py

@@ -4,7 +4,7 @@
 from fastapi.responses import HTMLResponse
 
 from project_forge.engine.scorer import score_summary
-from project_forge.models import IdeaCategory, IdeaStatus
+from project_forge.models import IdeaCategory, IdeaStatus, Resource, UrlIngestRequest
 from project_forge.web.app import db, templates
 
 router = APIRouter()
@@ -216,3 +216,37 @@ async def api_ideas(
 async def api_search(q: str = Query(min_length=1), limit: int = Query(default=20, ge=1, le=100)):
     ideas = await db.search_ideas(q, limit=limit)
     return {"ideas": [i.model_dump() for i in ideas], "total": len(ideas)}
+
+
+# === URL INGESTION & RESOURCE ROUTES ===
+
+
+async def ingest_idea_from_url(request_body: UrlIngestRequest):
+    """Fetch URL, extract content, and generate an idea. Module-level for patching in tests."""
+    from project_forge.engine.url_ingest import fetch_url_content, generate_idea_from_url
+
+    content = await fetch_url_content(request_body.url)
+    idea = await generate_idea_from_url(content, category_hint=request_body.category)
+    return idea
+
+
+@router.post("/api/ideas/from-url")
+async def ingest_url(request_body: UrlIngestRequest):
+    """Generate a project idea from a URL."""
+    idea = await ingest_idea_from_url(request_body)
+    await db.save_idea(idea)
+    return idea.model_dump()
+
+
+@router.get("/api/resources")
+async def list_resources():
+    """List all tracked source resources."""
+    resources = await db.list_resources()
+    return {"resources": [r.model_dump() for r in resources]}
+
+
+@router.post("/api/resources")
+async def add_resource(resource: Resource):
+    """Add or update a source resource."""
+    saved = await db.save_resource(resource)
+    return saved.model_dump()