Add artwork self-hosting: local-disk download worker + storage

Section 10: every image API responses reference has pointed straight
at TMDb's CDN since day one -- Aurora never actually downloaded or
served artwork itself, despite storing dimensions/checksum/local_path
columns for exactly that. Closes the gap:

- is_primary was declared on the Artwork model but never actually set
  anywhere -- every row defaulted to False. Now the normalizer marks
  the one image each provider designates as *the* image for an entity
  (TMDb's poster_path/backdrop_path/profile_path) as primary,
  distinct from the dozens of alternates also ingested, including
  correctly marking an existing row primary (not inserting a
  duplicate) when that image already appears in the alternates list.
- app/services/artwork/: an ArtworkStorage interface with a
  LocalDiskStorage implementation (checksum-named files, single-host
  deployment via a Docker volume shared between api and worker -- see
  docs/architecture.md for the S3/R2 upgrade path), and a download
  pipeline with content-type validation, a size cap, and path-traversal
  protection.
- After movie/series/person ingestion succeeds, tasks.py cascades one
  download_artwork_task per primary artwork id via arq's ctx['redis']
  (same fan-out pattern as the season cascade) -- fire-and-forget, not
  tracked in ingestion_jobs, since a failed image cache is a
  known-safe fallback to the provider URL, not a user-facing failure.
- app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is
  preferred over original_url once set (already-existing fallback
  logic in app/services/common.py:artwork_url, now actually exercised).

Also fixes a real interaction bug this surfaced: every ingestion
pipeline deleted and recreated all of an entity's Artwork rows on each
re-sync, which would have silently wiped local_path/checksum on every
re-ingestion, forcing a full re-download every time and defeating the
point of caching locally at all. app/repositories/polymorphic.py's new
sync_artwork upserts by provider_asset_id (TMDb's stable per-image
path) instead, preserving downloaded copies across re-sync, and
replaces three near-identical delete+recreate helpers that used to
live separately in movie_ingestion.py, series_ingestion.py, and
person_ingestion.py.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/services/artwork/download.py

@@ -0,0 +1,110 @@
+"""Downloads one Artwork row's original_url and stores it locally
+(section 10; the "Artwork processing" background worker from section 1's
+architecture diagram).
+
+Deliberately narrow: only ever called for artwork already marked
+`is_primary` (see `app/services/ingestion/runner.py:get_primary_artwork_ids`
+and `app/workers/tasks.py`, which cascade a download job per primary image
+after each ingestion succeeds) — TMDb returns dozens of backdrop/logo
+alternates per title, and downloading every one of them automatically would
+be a lot of unnecessary storage and bandwidth for images Aurora's own
+clients are unlikely to ever show. Downloading on request for anything
+beyond the primary image is a reasonable follow-up if that's ever needed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+
+import httpx
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import get_settings
+from app.core.logging import get_logger
+from app.models.artwork import Artwork
+from app.services.artwork.storage import ArtworkStorage
+
+logger = get_logger(__name__)
+
+_EXTENSION_BY_CONTENT_TYPE = {
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/png": "png",
+    "image/webp": "webp",
+    "image/gif": "gif",
+    "image/svg+xml": "svg",
+}
+
+
+class ArtworkDownloadError(Exception):
+    """Raised for any download-side failure (bad content-type, too large,
+    network error, ...). Callers (the arq task) let this propagate so the
+    job gets retried like any other ingestion failure — see
+    app/services/ingestion/runner.py."""
+
+
+async def download_artwork(
+    session: AsyncSession,
+    storage: ArtworkStorage,
+    artwork_id: uuid.UUID,
+    *,
+    client: httpx.AsyncClient | None = None,
+    force: bool = False,
+) -> bool:
+    """Fetch `artwork.original_url`, store it, and set `local_path` +
+    `checksum`. Returns False (no-op) if the row is gone or already has a
+    local copy (unless `force=True`); True if it downloaded something.
+    """
+
+    artwork = await session.get(Artwork, artwork_id)
+    if artwork is None:
+        return False
+    if artwork.local_path and not force:
+        return False
+
+    settings = get_settings()
+    owns_client = client is None
+    http_client = client or httpx.AsyncClient(timeout=20.0, follow_redirects=True)
+    try:
+        content, content_type = await _fetch(http_client, artwork.original_url, settings.ARTWORK_DOWNLOAD_MAX_BYTES)
+    finally:
+        if owns_client:
+            await http_client.aclose()
+
+    checksum = hashlib.sha256(content).hexdigest()
+    extension = _EXTENSION_BY_CONTENT_TYPE.get(content_type, "jpg")
+    relative_path = f"{artwork.artwork_type.value}/{checksum[:2]}/{checksum}.{extension}"
+
+    local_url = await storage.save(relative_path, content)
+
+    artwork.local_path = local_url
+    artwork.checksum = checksum
+    await session.flush()
+    logger.info(
+        "artwork_downloaded",
+        artwork_id=str(artwork_id),
+        checksum=checksum,
+        bytes=len(content),
+    )
+    return True
+
+
+async def _fetch(client: httpx.AsyncClient, url: str, max_bytes: int) -> tuple[bytes, str]:
+    try:
+        async with client.stream("GET", url) as response:
+            response.raise_for_status()
+            content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
+            if not content_type.startswith("image/"):
+                raise ArtworkDownloadError(f"expected an image, got content-type={content_type!r}")
+
+            body = bytearray()
+            async for chunk in response.aiter_bytes():
+                body.extend(chunk)
+                if len(body) > max_bytes:
+                    raise ArtworkDownloadError(
+                        f"artwork at {url} exceeds ARTWORK_DOWNLOAD_MAX_BYTES ({max_bytes} bytes)"
+                    )
+            return bytes(body), content_type
+    except httpx.HTTPError as exc:
+        raise ArtworkDownloadError(f"failed to download {url}: {exc}") from exc

app/services/artwork/storage.py

@@ -0,0 +1,66 @@
+"""Artwork storage abstraction (section 10).
+
+Downloaded artwork bytes need somewhere to live that both the worker
+(writing) and the API (serving, via app/main.py's static mount) can reach.
+`ArtworkStorage` is the seam: today there's only `LocalDiskStorage` — fine
+for a single Docker host where the api and worker containers share one
+volume (see docker-compose.yml), which is this project's actual deployment
+shape (section 1: modular monolith, not a multi-host fleet). A deployment
+that outgrows a single host would swap in an S3/R2-backed implementation
+behind this same interface without touching the download pipeline that
+calls it (app/services/artwork/download.py).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from pathlib import Path
+
+from app.core.config import get_settings
+
+
+class ArtworkStorage(ABC):
+    @abstractmethod
+    async def save(self, relative_path: str, content: bytes) -> str:
+        """Persist `content` at `relative_path` (already includes an
+        extension, e.g. "poster/ab/ab34...jpg"). Returns the root-relative
+        URL clients should use to fetch it."""
+
+    @abstractmethod
+    async def exists(self, relative_path: str) -> bool: ...
+
+
+class LocalDiskStorage(ArtworkStorage):
+    def __init__(self, root: str, url_prefix: str) -> None:
+        self._root = Path(root)
+        self._url_prefix = url_prefix.rstrip("/")
+
+    def _full_path(self, relative_path: str) -> Path:
+        # relative_path is always built by us from a sha256 checksum, never
+        # from unsanitized external input — but guard anyway against ever
+        # writing outside MEDIA_ROOT.
+        root = self._root.resolve()
+        full = (self._root / relative_path).resolve()
+        if full != root and root not in full.parents:
+            raise ValueError(f"unsafe artwork path escapes MEDIA_ROOT: {relative_path!r}")
+        return full
+
+    async def save(self, relative_path: str, content: bytes) -> str:
+        full = self._full_path(relative_path)
+        full.parent.mkdir(parents=True, exist_ok=True)
+        # Blocking I/O, but artwork files are small (capped by
+        # ARTWORK_DOWNLOAD_MAX_BYTES) and this only ever runs inside the
+        # worker process, not a request handler — not worth wiring aiofiles
+        # for.
+        full.write_bytes(content)
+        return f"{self._url_prefix}/{relative_path}"
+
+    async def exists(self, relative_path: str) -> bool:
+        return self._full_path(relative_path).is_file()
+
+
+@lru_cache
+def get_artwork_storage() -> ArtworkStorage:
+    settings = get_settings()
+    return LocalDiskStorage(settings.MEDIA_ROOT, settings.MEDIA_URL_PREFIX)

app/services/ingestion/movie_ingestion.py

@@ -20,15 +20,15 @@ from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.logging import get_logger
-from app.models.artwork import Artwork
 from app.models.company import MovieCompany
 from app.models.credit import Credit
-from app.models.enums import ArtworkType, MatchConfidence, MovieStatus, VideoType
+from app.models.enums import MatchConfidence, MovieStatus, VideoType
 from app.models.genre import MovieGenre
 from app.models.keyword import MovieKeyword
 from app.models.movie import Movie, MovieRecommendation
 from app.models.video import Video
 from app.providers.base import MetadataProvider
+from app.repositories import polymorphic
 from app.repositories import reference_repository as refs
 from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
 from app.services.metadata.matcher import match_entity
@@ -115,36 +115,9 @@ async def _replace_movie_credits(
         )
 
 
-_ARTWORK_TYPES = {t.value for t in ArtworkType}
 _VIDEO_TYPES = {t.value for t in VideoType}
 
 
-async def _replace_movie_artwork(
-    session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
-) -> None:
-    await session.execute(
-        delete(Artwork).where(Artwork.entity_type == "movie", Artwork.entity_id == movie_id)
-    )
-    for art in artwork_list:
-        art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
-        session.add(
-            Artwork(
-                entity_type="movie",
-                entity_id=movie_id,
-                artwork_type=ArtworkType(art_type),
-                language_code=None,
-                width=art.width,
-                height=art.height,
-                aspect_ratio=art.aspect_ratio,
-                provider_id=provider_id,
-                provider_asset_id=art.provider_asset_id,
-                original_url=art.url,
-                vote_average=art.vote_average,
-                is_primary=False,
-            )
-        )
-
-
 async def _replace_movie_videos(
     session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, videos
 ) -> None:
@@ -259,7 +232,13 @@ async def ingest_movie(
     await _replace_movie_keywords(session, movie.id, normalized.keywords)
     await _replace_movie_companies(session, movie.id, tmdb_provider.id, normalized.companies)
     await _replace_movie_credits(session, movie.id, tmdb_provider.id, normalized.cast, normalized.crew)
-    await _replace_movie_artwork(session, movie.id, tmdb_provider.id, normalized.artwork)
+    await polymorphic.sync_artwork(
+        session,
+        entity_type="movie",
+        entity_id=movie.id,
+        provider_id=tmdb_provider.id,
+        artwork_list=normalized.artwork,
+    )
     await _replace_movie_videos(session, movie.id, tmdb_provider.id, normalized.videos)
     await _link_external_ids(session, movie.id, tmdb_provider.id, normalized)
 

app/services/ingestion/person_ingestion.py

@@ -10,14 +10,13 @@ from __future__ import annotations
 
 import uuid
 
-from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.logging import get_logger
-from app.models.artwork import Artwork
-from app.models.enums import ArtworkType, MatchConfidence
+from app.models.enums import MatchConfidence
 from app.models.person import Person
 from app.providers.base import MetadataProvider
+from app.repositories import polymorphic
 from app.repositories import reference_repository as refs
 from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
 from app.services.metadata.matcher import match_entity
@@ -64,24 +63,13 @@ async def ingest_person(
 
     await session.flush()
 
-    await session.execute(
-        delete(Artwork).where(Artwork.entity_type == "person", Artwork.entity_id == person.id)
+    await polymorphic.sync_artwork(
+        session,
+        entity_type="person",
+        entity_id=person.id,
+        provider_id=tmdb_provider.id,
+        artwork_list=normalized.artwork,
     )
-    for art in normalized.artwork:
-        session.add(
-            Artwork(
-                entity_type="person",
-                entity_id=person.id,
-                artwork_type=ArtworkType.PROFILE,
-                width=art.width,
-                height=art.height,
-                aspect_ratio=art.aspect_ratio,
-                provider_id=tmdb_provider.id,
-                provider_asset_id=art.provider_asset_id,
-                original_url=art.url,
-                vote_average=art.vote_average,
-            )
-        )
 
     await refs.link_external_id(
         session,

app/services/ingestion/runner.py

@@ -19,6 +19,7 @@ from app.models.enums import IngestionJobStatus
 from app.models.provider import IngestionJob
 from app.providers.base import MetadataProvider
 from app.providers.tmdb import TMDbProvider
+from app.repositories.polymorphic import get_primary_artwork_ids
 from app.repositories.provider_repository import (
     create_ingestion_job,
     get_ingestion_job,
@@ -165,3 +166,22 @@ async def get_series_season_numbers(series_id: uuid.UUID) -> list[int]:
     async with session_scope() as session:
         seasons = await get_seasons_for_series(session, series_id)
         return [season.season_number for season in seasons]
+
+
+async def get_series_season_ids(series_id: uuid.UUID) -> list[uuid.UUID]:
+    async with session_scope() as session:
+        seasons = await get_seasons_for_series(session, series_id)
+        return [season.id for season in seasons]
+
+
+async def get_primary_artwork_for_download(
+    entity_type: str, entity_ids: list[uuid.UUID]
+) -> list[uuid.UUID]:
+    """Artwork ids worth auto-downloading for local hosting after an
+    ingestion succeeds — see app/services/artwork/download.py and
+    app/workers/tasks.py, which enqueue one download_artwork_task per id
+    this returns. A short-lived session of its own, same reasoning as
+    get_series_season_numbers above."""
+
+    async with session_scope() as session:
+        return await get_primary_artwork_ids(session, entity_type, entity_ids)

app/services/ingestion/series_ingestion.py

@@ -20,17 +20,16 @@ from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.logging import get_logger
-from app.models.artwork import Artwork
 from app.models.company import SeriesCompany
 from app.models.credit import Credit
-from app.models.enums import ArtworkType, CompanyType, MatchConfidence, SeriesStatus, VideoType
+from app.models.enums import CompanyType, MatchConfidence, SeriesStatus, VideoType
 from app.models.genre import SeriesGenre
 from app.models.keyword import SeriesKeyword
 from app.models.tv import Series
 from app.models.video import Video
 from app.providers.base import MetadataProvider, ProviderNotFoundError
+from app.repositories import polymorphic, tv_repository
 from app.repositories import reference_repository as refs
-from app.repositories import tv_repository
 from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
 from app.services.metadata.matcher import match_entity
 from app.services.metadata.normalizer import (
@@ -41,7 +40,6 @@ from app.services.metadata.normalizer import (
 
 logger = get_logger(__name__)
 
-_ARTWORK_TYPES = {t.value for t in ArtworkType}
 _VIDEO_TYPES = {t.value for t in VideoType}
 
 _LEADING_ARTICLES = ("the ", "a ", "an ")
@@ -140,30 +138,6 @@ async def _replace_series_credits(
         )
 
 
-async def _replace_entity_artwork(
-    session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
-) -> None:
-    await session.execute(
-        delete(Artwork).where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
-    )
-    for art in artwork_list:
-        art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
-        session.add(
-            Artwork(
-                entity_type=entity_type,
-                entity_id=entity_id,
-                artwork_type=ArtworkType(art_type),
-                width=art.width,
-                height=art.height,
-                aspect_ratio=art.aspect_ratio,
-                provider_id=provider_id,
-                provider_asset_id=art.provider_asset_id,
-                original_url=art.url,
-                vote_average=art.vote_average,
-            )
-        )
-
-
 async def _replace_entity_videos(
     session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, videos
 ) -> None:
@@ -252,7 +226,13 @@ async def ingest_series(
     await _replace_series_credits(
         session, series.id, tmdb_provider.id, normalized.cast, normalized.crew, normalized.creators
     )
-    await _replace_entity_artwork(session, "series", series.id, tmdb_provider.id, normalized.artwork)
+    await polymorphic.sync_artwork(
+        session,
+        entity_type="series",
+        entity_id=series.id,
+        provider_id=tmdb_provider.id,
+        artwork_list=normalized.artwork,
+    )
     await _replace_entity_videos(session, "series", series.id, tmdb_provider.id, normalized.videos)
 
     for stub in normalized.seasons:
@@ -262,12 +242,12 @@ async def ingest_series(
         season.air_date = stub.air_date
         season.episode_count = stub.episode_count
         await session.flush()
-        await _replace_entity_artwork(
+        await polymorphic.sync_artwork(
             session,
-            "season",
-            season.id,
-            tmdb_provider.id,
-            [stub.poster] if stub.poster else [],
+            entity_type="season",
+            entity_id=season.id,
+            provider_id=tmdb_provider.id,
+            artwork_list=[stub.poster] if stub.poster else [],
         )
 
     await refs.link_external_id(
@@ -373,7 +353,13 @@ async def ingest_season(
     season.episode_count = normalized.episode_count
     await session.flush()
 
-    await _replace_entity_artwork(session, "season", season.id, tmdb_provider.id, normalized.artwork)
+    await polymorphic.sync_artwork(
+        session,
+        entity_type="season",
+        entity_id=season.id,
+        provider_id=tmdb_provider.id,
+        artwork_list=normalized.artwork,
+    )
 
     for ep in normalized.episodes:
         episode = await tv_repository.get_or_create_episode(
@@ -390,7 +376,13 @@ async def ingest_season(
         await session.flush()
 
         await _replace_episode_credits(session, episode.id, tmdb_provider.id, ep.guest_stars, ep.crew)
-        await _replace_entity_artwork(session, "episode", episode.id, tmdb_provider.id, ep.stills)
+        await polymorphic.sync_artwork(
+            session,
+            entity_type="episode",
+            entity_id=episode.id,
+            provider_id=tmdb_provider.id,
+            artwork_list=ep.stills,
+        )
         await refs.link_external_id(
             session,
             entity_type="episode",

app/services/metadata/normalizer.py

@@ -66,6 +66,12 @@ class NormalizedArtwork:
     aspect_ratio: float | None
     language: str | None
     vote_average: float | None = None
+    # True for the single image the provider designates as *the* image for
+    # this entity (TMDb's top-level poster_path/backdrop_path/profile_path),
+    # as opposed to the full alternates list — this is what gets
+    # auto-downloaded for local hosting (app/services/artwork/), so it's
+    # deliberately narrow rather than "the first one in the list".
+    is_primary: bool = False
 
 
 @dataclass
@@ -182,6 +188,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
                 None,
                 None,
                 None,
+                is_primary=True,
             )
         )
     if raw.get("backdrop_path"):
@@ -194,6 +201,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
                 None,
                 None,
                 None,
+                is_primary=True,
             )
         )
     for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
@@ -309,19 +317,26 @@ def normalize_person(raw: dict) -> NormalizedPerson:
         )
         for img in images
     ]
-    if raw.get("profile_path") and not any(a.provider_asset_id == raw["profile_path"] for a in artwork):
-        artwork.insert(
-            0,
-            NormalizedArtwork(
-                "profile",
-                _image_url_required(raw["profile_path"]),
-                raw["profile_path"],
-                None,
-                None,
-                None,
-                None,
-            ),
+    if raw.get("profile_path"):
+        existing = next(
+            (a for a in artwork if a.provider_asset_id == raw["profile_path"]), None
         )
+        if existing is not None:
+            existing.is_primary = True
+        else:
+            artwork.insert(
+                0,
+                NormalizedArtwork(
+                    "profile",
+                    _image_url_required(raw["profile_path"]),
+                    raw["profile_path"],
+                    None,
+                    None,
+                    None,
+                    None,
+                    is_primary=True,
+                ),
+            )
 
     also_known_as = raw.get("also_known_as") or []
 
@@ -473,6 +488,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
                 None,
                 None,
                 None,
+                is_primary=True,
             )
         )
     if raw.get("backdrop_path"):
@@ -485,6 +501,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
                 None,
                 None,
                 None,
+                is_primary=True,
             )
         )
     for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
@@ -526,7 +543,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
             poster=(
                 NormalizedArtwork(
                     "poster", _image_url_required(s["poster_path"]), s["poster_path"],
-                    None, None, None, None,
+                    None, None, None, None, is_primary=True,
                 )
                 if s.get("poster_path")
                 else None
@@ -621,7 +638,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
         artwork.append(
             NormalizedArtwork(
                 "poster", _image_url_required(raw["poster_path"]), raw["poster_path"],
-                None, None, None, None,
+                None, None, None, None, is_primary=True,
             )
         )
     for img in images.get("posters", []):
@@ -656,7 +673,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
             stills.append(
                 NormalizedArtwork(
                     "still", _image_url_required(ep["still_path"]), ep["still_path"],
-                    None, None, None, None,
+                    None, None, None, None, is_primary=True,
                 )
             )
         episodes.append(