Add collection/franchise ingestion

Collections were bare stubs (name only) lazily created the moment a
movie's belongs_to_collection reference was seen. Adds a real
ingest_collection pipeline: TMDb's /collection/{id} for overview and
poster/backdrop artwork, plus ordered CollectionMovie membership --
linking only movies already ingested into Aurora, the same
never-fabricate-a-stub rule movie recommendations already follow.

get_or_create_collection now refreshes name/overview on an existing
stub instead of only creating, guarded so a bare movie-stub call
(overview=None) never clobbers real data a full ingestion already
filled in.

New POST /api/v1/admin/ingest/collection endpoint and
ingest_collection_task; GET /api/v1/collections/{id} needed no changes
since its read path (collection_service.py) already assembled the full
detail shape.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/api/v1/admin.py

@@ -49,6 +49,10 @@ class IngestSeasonRequest(BaseModel):
     season_number: int
 
 
+class IngestCollectionRequest(BaseModel):
+    tmdb_id: str
+
+
 class IngestionEnqueuedOut(BaseModel):
     job_id: str
     arq_job_id: str
@@ -146,6 +150,24 @@ async def enqueue_season_ingestion(
     )
 
 
+@router.post("/ingest/collection", response_model=IngestionEnqueuedOut)
+async def enqueue_collection_ingestion(
+    body: IngestCollectionRequest, queue: ArqRedis = Depends(get_queue)
+) -> IngestionEnqueuedOut:
+    # Membership only links movies already ingested into Aurora (never a
+    # stub) -- see app/services/ingestion/collection_ingestion.py -- so
+    # ingesting the franchise's movies first (or re-running this afterward)
+    # is what fills the list in.
+    return await _enqueue(
+        queue,
+        job_type="collection_sync",
+        entity_type="collection",
+        external_id=body.tmdb_id,
+        task_name="ingest_collection_task",
+        task_args=(body.tmdb_id,),
+    )
+
+
 @router.get("/ingest/jobs/{job_id}", response_model=IngestionJobOut)
 async def get_ingestion_job_status(
     job_id: uuid.UUID, session: AsyncSession = Depends(get_db)

app/providers/base.py

@@ -49,6 +49,11 @@ class MetadataProvider(ABC):
     async def get_person(self, external_id: str) -> dict[str, Any]:
         """Fetch raw person data, including external IDs and images."""
 
+    @abstractmethod
+    async def get_collection(self, external_id: str) -> dict[str, Any]:
+        """Fetch raw collection/franchise data: name, overview, poster/
+        backdrop, and its member movies (TMDb's "parts")."""
+
     @abstractmethod
     async def search_movie(self, query: str) -> list[dict[str, Any]]:
         """Search this provider's catalog for movies matching `query`. Used

app/providers/tmdb.py

@@ -80,6 +80,9 @@ class TMDbProvider(MetadataProvider):
     async def get_person(self, external_id: str) -> dict[str, Any]:
         return await self._get(f"/person/{external_id}", {"append_to_response": _PERSON_APPEND})
 
+    async def get_collection(self, external_id: str) -> dict[str, Any]:
+        return await self._get(f"/collection/{external_id}")
+
     async def search_movie(self, query: str) -> list[dict[str, Any]]:
         data = await self._get("/search/movie", {"query": query})
         return data.get("results", [])

app/repositories/reference_repository.py

@@ -208,6 +208,15 @@ async def get_or_create_collection(
 
     existing = await _lookup()
     if existing is not None:
+        existing.name = name
+        # Only overwrite overview when the caller actually has one -- a
+        # movie's bare `belongs_to_collection` reference passes no overview
+        # at all, and must not blank out real data a full `ingest_collection`
+        # run already filled in (see get_or_create_person for the same
+        # partial-data guard).
+        if overview is not None:
+            existing.overview = overview
+        await session.flush()
         return existing
 
     async def _create() -> Collection:

app/services/ingestion/collection_ingestion.py

@@ -0,0 +1,95 @@
+"""Collection/franchise ingestion (section 15).
+
+Movie ingestion already creates a bare `Collection` stub (name only) the
+moment it sees a `belongs_to_collection` reference (see
+`movie_ingestion.py`). This module is the real sync: fetching TMDb's
+`/collection/{id}` payload for overview, poster/backdrop artwork, and the
+franchise's member movies.
+
+Membership (`CollectionMovie`) is only ever linked for movies Aurora has
+already ingested -- the same "never fabricate an entity from a bare
+reference" rule `ingest_movie` applies to recommendations. A newly-ingested
+collection can list zero movies if none of its parts have been ingested yet;
+re-running `ingest_collection` after ingesting more of them fills the list
+in, since it's a full re-sync, not an append.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.logging import get_logger
+from app.models.collection import CollectionMovie
+from app.models.enums import MatchConfidence
+from app.providers.base import MetadataProvider
+from app.repositories import polymorphic
+from app.repositories import reference_repository as refs
+from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
+from app.services.metadata.matcher import match_entity
+from app.services.metadata.normalizer import normalize_collection
+
+logger = get_logger(__name__)
+
+
+async def ingest_collection(
+    session: AsyncSession,
+    provider: MetadataProvider,
+    external_id: str,
+    *,
+    ingestion_job_id: uuid.UUID | None = None,
+) -> uuid.UUID:
+    """Fetch, normalize, and upsert a single collection. Returns the Aurora
+    collection UUID (new or existing)."""
+
+    tmdb_provider = await get_or_create_provider(session, provider.slug, provider.name)
+
+    aurora_id_existing, confidence = await match_entity(
+        session, provider_id=tmdb_provider.id, entity_type="collection", external_id=external_id
+    )
+
+    raw = await provider.get_collection(external_id)
+    normalized = normalize_collection(raw)
+
+    collection = await refs.get_or_create_collection(
+        session,
+        provider_id=tmdb_provider.id,
+        external_id=external_id,
+        name=normalized.name,
+        overview=normalized.overview,
+    )
+
+    await polymorphic.sync_artwork(
+        session,
+        entity_type="collection",
+        entity_id=collection.id,
+        provider_id=tmdb_provider.id,
+        artwork_list=normalized.artwork,
+    )
+
+    await session.execute(
+        delete(CollectionMovie).where(CollectionMovie.collection_id == collection.id)
+    )
+    movie_ids = await refs.find_entities_by_external_ids(
+        session, tmdb_provider.id, "movie", normalized.part_external_ids
+    )
+    for sort_order, movie_id in enumerate(movie_ids):
+        session.add(
+            CollectionMovie(collection_id=collection.id, movie_id=movie_id, sort_order=sort_order)
+        )
+
+    await upsert_provider_record(
+        session,
+        provider_id=tmdb_provider.id,
+        entity_type="collection",
+        external_id=external_id,
+        raw_data=raw,
+        aurora_entity_id=collection.id,
+        match_confidence=confidence if aurora_id_existing else MatchConfidence.NEW,
+        ingestion_job_id=ingestion_job_id,
+    )
+
+    logger.info("collection_ingested", collection_id=str(collection.id), external_id=external_id)
+    return collection.id

app/services/ingestion/runner.py

@@ -33,6 +33,7 @@ from app.repositories.provider_repository import (
 )
 from app.repositories.tv_repository import get_seasons_for_series, get_series_companies
 from app.services.cache import build_key, cache_delete
+from app.services.ingestion.collection_ingestion import ingest_collection
 from app.services.ingestion.movie_ingestion import ingest_movie
 from app.services.ingestion.person_ingestion import ingest_person
 from app.services.ingestion.series_ingestion import ingest_season, ingest_series
@@ -164,6 +165,16 @@ async def run_series_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID
     )
 
 
+async def run_collection_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
+    return await _run_ingestion(
+        job_id,
+        "collection_ingestion",
+        lambda session, provider: ingest_collection(
+            session, provider, external_id, ingestion_job_id=job_id
+        ),
+    )
+
+
 async def run_season_ingestion(
     job_id: uuid.UUID, series_external_id: str, season_number: int
 ) -> uuid.UUID:

app/services/metadata/normalizer.py

@@ -138,6 +138,61 @@ class NormalizedMovie:
     recommendation_external_ids: list[str] = field(default_factory=list)
 
 
+@dataclass
+class NormalizedCollection:
+    external_id: str
+    name: str
+    overview: str | None
+    artwork: list[NormalizedArtwork] = field(default_factory=list)
+    # TMDb's "parts", ordered by release date rather than TMDb's own array
+    # order (which isn't guaranteed to be chronological) -- see
+    # normalize_collection.
+    part_external_ids: list[str] = field(default_factory=list)
+
+
+def normalize_collection(raw: dict) -> NormalizedCollection:
+    artwork: list[NormalizedArtwork] = []
+    if raw.get("poster_path"):
+        artwork.append(
+            NormalizedArtwork(
+                "poster",
+                _image_url_required(raw["poster_path"]),
+                raw["poster_path"],
+                None,
+                None,
+                None,
+                None,
+                is_primary=True,
+            )
+        )
+    if raw.get("backdrop_path"):
+        artwork.append(
+            NormalizedArtwork(
+                "backdrop",
+                _image_url_required(raw["backdrop_path"]),
+                raw["backdrop_path"],
+                None,
+                None,
+                None,
+                None,
+                is_primary=True,
+            )
+        )
+
+    parts = sorted(
+        raw.get("parts") or [],
+        key=lambda p: _parse_date(p.get("release_date")) or date.max,
+    )
+
+    return NormalizedCollection(
+        external_id=str(raw["id"]),
+        name=raw["name"],
+        overview=raw.get("overview") or None,
+        artwork=artwork,
+        part_external_ids=[str(p["id"]) for p in parts],
+    )
+
+
 _STATUS_MAP = {
     "Rumored": "rumored",
     "Planned": "planned",

app/workers/tasks.py

@@ -28,6 +28,7 @@ from app.services.ingestion.runner import (
     get_series_company_ids,
     get_series_season_ids,
     get_series_season_numbers,
+    run_collection_ingestion,
     run_movie_ingestion,
     run_person_ingestion,
     run_season_ingestion,
@@ -102,6 +103,12 @@ async def ingest_season_task(
     return str(season_id)
 
 
+async def ingest_collection_task(ctx: dict[str, Any], job_id: str, external_id: str) -> str:
+    collection_id = await run_collection_ingestion(uuid.UUID(job_id), external_id)
+    await _cascade_artwork_downloads(ctx, "collection", [collection_id])
+    return str(collection_id)
+
+
 async def download_artwork_task(ctx: dict[str, Any], artwork_id: str) -> bool:
     async with session_scope() as session:
         return await download_artwork(session, get_artwork_storage(), uuid.UUID(artwork_id))

app/workers/worker.py

@@ -15,6 +15,7 @@ from app.core.config import get_settings
 from app.core.logging import configure_logging, get_logger
 from app.workers.tasks import (
     download_artwork_task,
+    ingest_collection_task,
     ingest_movie_task,
     ingest_person_task,
     ingest_season_task,
@@ -39,6 +40,7 @@ class WorkerSettings:
         ingest_person_task,
         ingest_series_task,
         ingest_season_task,
+        ingest_collection_task,
         download_artwork_task,
     ]
     on_startup = startup