Initial Aurora Core 0.1 implementation

Production-grade media metadata backend per docs/aurora-core-spec.md: Aurora-owned UUIDv7 identity system, normalized movie/TV/people/credits schema, provider abstraction with a TMDb adapter, an idempotent async ingestion pipeline (normalize -> match -> upsert -> provenance), a Postgres FTS/trigram search layer, a stateless FastAPI v1 API with Redis caching/rate-limiting, arq workers, dev/prod Docker Compose topologies, unit + integration + load tests, CI, and architecture/ database/API/ERD documentation. Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-09-04 17:34:03 -05:00
commit c20df1c503
111 changed files with 10041 additions and 0 deletions
@@ -0,0 +1,37 @@
+"""Entity matching (section 14): decides whether an incoming provider
+record corresponds to an existing Aurora entity.
+
+Core 0.1 policy — deliberately conservative:
+
+- An exact external-ID match (this provider has already been linked to an
+  Aurora entity of this type) is the only case treated as MatchConfidence
+  EXACT and reused.
+- Anything else creates a *new* Aurora entity (MatchConfidence NEW) rather
+  than guessing via fuzzy title matching. Silently merging two providers'
+  differing descriptions of "the same" title is exactly what section 14
+  prohibits.
+
+Fuzzy candidate matching (title + year similarity, surfaced for human
+review) is genuinely deferred — see docs/architecture.md — until Core has
+an admin review workflow to act on AMBIGUOUS matches.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.enums import MatchConfidence
+from app.repositories.reference_repository import find_entity_by_external_id
+
+
+async def match_entity(
+    session: AsyncSession, *, provider_id: uuid.UUID, entity_type: str, external_id: str
+) -> tuple[uuid.UUID | None, MatchConfidence]:
+    """Return (existing_aurora_id_or_None, confidence)."""
+
+    existing = await find_entity_by_external_id(session, provider_id, entity_type, external_id)
+    if existing is not None:
+        return existing, MatchConfidence.EXACT
+    return None, MatchConfidence.NEW
@@ -0,0 +1,341 @@
+"""Normalizes raw TMDb payloads into provider-agnostic dataclasses.
+
+This is the seam between "provider shape" and "Aurora shape" (section 13).
+A future provider adapter only needs its own `normalize_movie`-equivalent
+that produces these same dataclasses; nothing downstream (matcher,
+ingestion) needs to know which provider the data came from.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import date
+
+TMDB_IMAGE_BASE = "https://image.tmdb.org/t/p/original"
+
+
+def _parse_date(value: str | None) -> date | None:
+    if not value:
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def _image_url(path: str | None) -> str | None:
+    if not path:
+        return None
+    return f"{TMDB_IMAGE_BASE}{path}"
+
+
+def _image_url_required(path: str) -> str:
+    """Same as `_image_url`, for call sites already guarded by a truthy
+    check on `path` — keeps `NormalizedArtwork.url: str` honest."""
+
+    return f"{TMDB_IMAGE_BASE}{path}"
+
+
+@dataclass
+class NormalizedPersonRef:
+    external_id: str
+    name: str
+    profile_path: str | None = None
+    department: str | None = None
+    job: str | None = None
+    character: str | None = None
+    billing_order: int | None = None
+    gender: str | None = None
+
+
+@dataclass
+class NormalizedCompanyRef:
+    external_id: str
+    name: str
+    country_code: str | None = None
+    role: str = "production"
+
+
+@dataclass
+class NormalizedArtwork:
+    artwork_type: str  # "poster" | "backdrop" | "logo" | "profile" | "still"
+    url: str
+    provider_asset_id: str | None
+    width: int | None
+    height: int | None
+    aspect_ratio: float | None
+    language: str | None
+    vote_average: float | None = None
+
+
+@dataclass
+class NormalizedVideo:
+    video_type: str
+    provider_video_id: str
+    site: str | None
+    url: str
+    language: str | None
+    country: str | None
+    is_official: bool
+    published_at: date | None
+
+
+@dataclass
+class NormalizedMovie:
+    external_id: str
+    title: str
+    original_title: str | None
+    overview: str | None
+    tagline: str | None
+    status: str
+    original_language: str | None
+    release_date: date | None
+    runtime_minutes: int | None
+    budget: int | None
+    revenue: int | None
+    popularity: float | None
+    vote_average: float | None
+    vote_count: int | None
+    adult: bool
+    genres: list[str] = field(default_factory=list)
+    keywords: list[str] = field(default_factory=list)
+    alternate_titles: list[str] = field(default_factory=list)
+    companies: list[NormalizedCompanyRef] = field(default_factory=list)
+    cast: list[NormalizedPersonRef] = field(default_factory=list)
+    crew: list[NormalizedPersonRef] = field(default_factory=list)
+    artwork: list[NormalizedArtwork] = field(default_factory=list)
+    videos: list[NormalizedVideo] = field(default_factory=list)
+    external_ids: dict[str, str] = field(default_factory=dict)
+    collection: tuple[str, str] | None = None  # (external_id, name)
+    recommendation_external_ids: list[str] = field(default_factory=list)
+
+
+_STATUS_MAP = {
+    "Rumored": "rumored",
+    "Planned": "planned",
+    "In Production": "in_production",
+    "Post Production": "post_production",
+    "Released": "released",
+    "Canceled": "canceled",
+}
+
+
+def normalize_movie(raw: dict) -> NormalizedMovie:
+    credits_ = raw.get("credits") or {}
+    images = raw.get("images") or {}
+    videos = (raw.get("videos") or {}).get("results") or []
+    keywords = (raw.get("keywords") or {}).get("keywords") or []
+    alt_titles = (raw.get("alternative_titles") or {}).get("titles") or []
+    ext_ids = raw.get("external_ids") or {}
+    collection = raw.get("belongs_to_collection")
+    recommendations = (raw.get("recommendations") or {}).get("results") or []
+
+    external_ids: dict[str, str] = {}
+    if raw.get("imdb_id"):
+        external_ids["imdb"] = raw["imdb_id"]
+    elif ext_ids.get("imdb_id"):
+        external_ids["imdb"] = ext_ids["imdb_id"]
+    if ext_ids.get("wikidata_id"):
+        external_ids["wikidata"] = ext_ids["wikidata_id"]
+
+    cast = [
+        NormalizedPersonRef(
+            external_id=str(c["id"]),
+            name=c["name"],
+            profile_path=_image_url(c.get("profile_path")),
+            department="Acting",
+            character=c.get("character"),
+            billing_order=c.get("order"),
+            gender=_gender_label(c.get("gender")),
+        )
+        for c in credits_.get("cast", [])
+    ]
+    crew = [
+        NormalizedPersonRef(
+            external_id=str(c["id"]),
+            name=c["name"],
+            profile_path=_image_url(c.get("profile_path")),
+            department=c.get("department") or "Crew",
+            job=c.get("job"),
+            gender=_gender_label(c.get("gender")),
+        )
+        for c in credits_.get("crew", [])
+    ]
+
+    companies = [
+        NormalizedCompanyRef(
+            external_id=str(pc["id"]),
+            name=pc["name"],
+            country_code=pc.get("origin_country") or None,
+        )
+        for pc in raw.get("production_companies", [])
+    ]
+
+    artwork: list[NormalizedArtwork] = []
+    if raw.get("poster_path"):
+        artwork.append(
+            NormalizedArtwork(
+                "poster",
+                _image_url_required(raw["poster_path"]),
+                raw["poster_path"],
+                None,
+                None,
+                None,
+                None,
+            )
+        )
+    if raw.get("backdrop_path"):
+        artwork.append(
+            NormalizedArtwork(
+                "backdrop",
+                _image_url_required(raw["backdrop_path"]),
+                raw["backdrop_path"],
+                None,
+                None,
+                None,
+                None,
+            )
+        )
+    for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
+        for img in images.get(tmdb_key, []):
+            artwork.append(
+                NormalizedArtwork(
+                    kind,
+                    _image_url_required(img["file_path"]),
+                    img["file_path"],
+                    img.get("width"),
+                    img.get("height"),
+                    img.get("aspect_ratio"),
+                    img.get("iso_639_1"),
+                    img.get("vote_average"),
+                )
+            )
+
+    normalized_videos = [
+        NormalizedVideo(
+            video_type=(v.get("type") or "other").lower().replace(" ", "_"),
+            provider_video_id=v["id"],
+            site=v.get("site"),
+            url=_video_url(v),
+            language=v.get("iso_639_1"),
+            country=v.get("iso_3166_1"),
+            is_official=bool(v.get("official")),
+            published_at=_parse_date(v.get("published_at", "")[:10] if v.get("published_at") else None),
+        )
+        for v in videos
+    ]
+
+    return NormalizedMovie(
+        external_id=str(raw["id"]),
+        title=raw.get("title") or raw.get("original_title") or "",
+        original_title=raw.get("original_title"),
+        overview=raw.get("overview") or None,
+        tagline=raw.get("tagline") or None,
+        status=_STATUS_MAP.get(raw.get("status", "Released"), "released"),
+        original_language=raw.get("original_language"),
+        release_date=_parse_date(raw.get("release_date")),
+        runtime_minutes=raw.get("runtime"),
+        budget=raw.get("budget") or None,
+        revenue=raw.get("revenue") or None,
+        popularity=raw.get("popularity"),
+        vote_average=raw.get("vote_average"),
+        vote_count=raw.get("vote_count"),
+        adult=bool(raw.get("adult", False)),
+        genres=[g["name"] for g in raw.get("genres", [])],
+        keywords=[k["name"] for k in keywords],
+        alternate_titles=[t["title"] for t in alt_titles if t.get("title")],
+        companies=companies,
+        cast=cast,
+        crew=crew,
+        artwork=artwork,
+        videos=normalized_videos,
+        external_ids=external_ids,
+        collection=(str(collection["id"]), collection["name"]) if collection else None,
+        recommendation_external_ids=[str(m["id"]) for m in recommendations],
+    )
+
+
+@dataclass
+class NormalizedPerson:
+    external_id: str
+    name: str
+    biography: str | None
+    birthday: date | None
+    deathday: date | None
+    birthplace: str | None
+    gender: str | None
+    known_for_department: str | None
+    popularity: float | None
+    aliases: list[str]
+    artwork: list[NormalizedArtwork]
+    external_ids: dict[str, str]
+
+
+def _gender_label(tmdb_gender: int | None) -> str | None:
+    return {1: "female", 2: "male", 3: "non_binary"}.get(tmdb_gender or 0)
+
+
+def _video_url(video: dict) -> str:
+    site = (video.get("site") or "").lower()
+    key = video.get("key", "")
+    if site == "youtube":
+        return f"https://www.youtube.com/watch?v={key}"
+    if site == "vimeo":
+        return f"https://vimeo.com/{key}"
+    return key
+
+
+def normalize_person(raw: dict) -> NormalizedPerson:
+    images = (raw.get("images") or {}).get("profiles") or []
+    ext_ids = raw.get("external_ids") or {}
+    external_ids: dict[str, str] = {}
+    if raw.get("imdb_id"):
+        external_ids["imdb"] = raw["imdb_id"]
+    elif ext_ids.get("imdb_id"):
+        external_ids["imdb"] = ext_ids["imdb_id"]
+    if ext_ids.get("wikidata_id"):
+        external_ids["wikidata"] = ext_ids["wikidata_id"]
+
+    artwork = [
+        NormalizedArtwork(
+            "profile",
+            _image_url_required(img["file_path"]),
+            img["file_path"],
+            img.get("width"),
+            img.get("height"),
+            img.get("aspect_ratio"),
+            img.get("iso_639_1"),
+            img.get("vote_average"),
+        )
+        for img in images
+    ]
+    if raw.get("profile_path") and not any(a.provider_asset_id == raw["profile_path"] for a in artwork):
+        artwork.insert(
+            0,
+            NormalizedArtwork(
+                "profile",
+                _image_url_required(raw["profile_path"]),
+                raw["profile_path"],
+                None,
+                None,
+                None,
+                None,
+            ),
+        )
+
+    also_known_as = raw.get("also_known_as") or []
+
+    return NormalizedPerson(
+        external_id=str(raw["id"]),
+        name=raw.get("name", ""),
+        biography=raw.get("biography") or None,
+        birthday=_parse_date(raw.get("birthday")),
+        deathday=_parse_date(raw.get("deathday")),
+        birthplace=raw.get("place_of_birth") or None,
+        gender=_gender_label(raw.get("gender")),
+        known_for_department=raw.get("known_for_department"),
+        popularity=raw.get("popularity"),
+        aliases=list(also_known_as),
+        artwork=artwork,
+        external_ids=external_ids,
+    )