Files
Aurora-Core/app/repositories/polymorphic.py
T
NixReaperandClaude Sonnet 5 fef914db36
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Add artwork self-hosting: local-disk download worker + storage
Section 10: every image API responses reference has pointed straight
at TMDb's CDN since day one -- Aurora never actually downloaded or
served artwork itself, despite storing dimensions/checksum/local_path
columns for exactly that. Closes the gap:

- is_primary was declared on the Artwork model but never actually set
  anywhere -- every row defaulted to False. Now the normalizer marks
  the one image each provider designates as *the* image for an entity
  (TMDb's poster_path/backdrop_path/profile_path) as primary,
  distinct from the dozens of alternates also ingested, including
  correctly marking an existing row primary (not inserting a
  duplicate) when that image already appears in the alternates list.
- app/services/artwork/: an ArtworkStorage interface with a
  LocalDiskStorage implementation (checksum-named files, single-host
  deployment via a Docker volume shared between api and worker -- see
  docs/architecture.md for the S3/R2 upgrade path), and a download
  pipeline with content-type validation, a size cap, and path-traversal
  protection.
- After movie/series/person ingestion succeeds, tasks.py cascades one
  download_artwork_task per primary artwork id via arq's ctx['redis']
  (same fan-out pattern as the season cascade) -- fire-and-forget, not
  tracked in ingestion_jobs, since a failed image cache is a
  known-safe fallback to the provider URL, not a user-facing failure.
- app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is
  preferred over original_url once set (already-existing fallback
  logic in app/services/common.py:artwork_url, now actually exercised).

Also fixes a real interaction bug this surfaced: every ingestion
pipeline deleted and recreated all of an entity's Artwork rows on each
re-sync, which would have silently wiped local_path/checksum on every
re-ingestion, forcing a full re-download every time and defeating the
point of caching locally at all. app/repositories/polymorphic.py's new
sync_artwork upserts by provider_asset_id (TMDb's stable per-image
path) instead, preserving downloaded copies across re-sync, and
replaces three near-identical delete+recreate helpers that used to
live separately in movie_ingestion.py, series_ingestion.py, and
person_ingestion.py.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-09-05 03:26:28 -05:00

209 lines
7.3 KiB
Python

"""Shared data access for the polymorphic satellite tables — credits,
artwork, videos, external IDs — that attach to more than one entity type via
an (entity_type, entity_id) pair rather than a typed foreign key.
"""
from __future__ import annotations
import uuid
from collections.abc import Sequence
from typing import Protocol
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.models.artwork import Artwork
from app.models.credit import Credit
from app.models.enums import ArtworkType
from app.models.external_id import ExternalID
from app.models.provider import Provider
from app.models.video import Video
_VALID_ARTWORK_TYPES = {t.value for t in ArtworkType}
class _NormalizedArtworkLike(Protocol):
"""Structural type for app.services.metadata.normalizer.NormalizedArtwork
— a repository shouldn't import from the services layer, so this
describes the shape `sync_artwork` needs instead of the concrete class.
"""
artwork_type: str
url: str
provider_asset_id: str | None
width: int | None
height: int | None
aspect_ratio: float | None
vote_average: float | None
is_primary: bool
async def get_external_ids(
session: AsyncSession, entity_type: str, entity_id: uuid.UUID
) -> dict[str, str]:
stmt = (
select(Provider.slug, ExternalID.external_id)
.join(Provider, Provider.id == ExternalID.provider_id)
.where(ExternalID.entity_type == entity_type, ExternalID.entity_id == entity_id)
)
result = await session.execute(stmt)
return {slug: external_id for slug, external_id in result.all()}
async def get_external_ids_bulk(
session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID]
) -> dict[uuid.UUID, dict[str, str]]:
if not entity_ids:
return {}
stmt = (
select(ExternalID.entity_id, Provider.slug, ExternalID.external_id)
.join(Provider, Provider.id == ExternalID.provider_id)
.where(ExternalID.entity_type == entity_type, ExternalID.entity_id.in_(entity_ids))
)
result = await session.execute(stmt)
out: dict[uuid.UUID, dict[str, str]] = {eid: {} for eid in entity_ids}
for entity_id, slug, external_id in result.all():
out.setdefault(entity_id, {})[slug] = external_id
return out
async def get_primary_artwork_ids(
session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID]
) -> list[uuid.UUID]:
"""IDs of artwork rows marked `is_primary` for any of `entity_ids` — the
small subset of a title's images (TMDb's own selected poster/backdrop/
profile) that get auto-downloaded for local hosting, out of the dozens
of alternates ingestion also stores. See app/services/artwork/."""
if not entity_ids:
return []
stmt = select(Artwork.id).where(
Artwork.entity_type == entity_type,
Artwork.entity_id.in_(entity_ids),
Artwork.is_primary.is_(True),
)
result = await session.execute(stmt)
return list(result.scalars().all())
async def sync_artwork(
session: AsyncSession,
*,
entity_type: str,
entity_id: uuid.UUID,
provider_id: uuid.UUID,
artwork_list: Sequence[_NormalizedArtworkLike],
) -> None:
"""Upsert an entity's artwork rows keyed by `provider_asset_id` (TMDb's
stable per-image file path), instead of deleting and recreating them on
every re-sync.
This matters specifically because of `app/services/artwork/` (section
10's self-hosting): a delete-then-recreate replace would generate a
fresh Artwork.id with `local_path=None` every time a title gets
re-ingested, silently throwing away every previously-downloaded local
copy and forcing a full re-download on each sync — defeating the point
of caching them locally at all. Rows for images no longer present in
`artwork_list` (dropped from the provider's alternates) are still
deleted, same as before.
"""
existing = (
(
await session.execute(
select(Artwork).where(
Artwork.entity_type == entity_type, Artwork.entity_id == entity_id
)
)
)
.scalars()
.all()
)
existing_by_asset_id = {
row.provider_asset_id: row for row in existing if row.provider_asset_id is not None
}
seen_asset_ids: set[str] = set()
for art in artwork_list:
art_type = art.artwork_type if art.artwork_type in _VALID_ARTWORK_TYPES else "poster"
matched = (
existing_by_asset_id.get(art.provider_asset_id) if art.provider_asset_id else None
)
if matched is not None:
seen_asset_ids.add(art.provider_asset_id) # type: ignore[arg-type]
matched.artwork_type = ArtworkType(art_type)
matched.width = art.width
matched.height = art.height
matched.aspect_ratio = art.aspect_ratio
matched.vote_average = art.vote_average
matched.is_primary = art.is_primary
matched.original_url = art.url
# local_path/checksum deliberately untouched — see docstring.
else:
session.add(
Artwork(
entity_type=entity_type,
entity_id=entity_id,
artwork_type=ArtworkType(art_type),
width=art.width,
height=art.height,
aspect_ratio=art.aspect_ratio,
provider_id=provider_id,
provider_asset_id=art.provider_asset_id,
original_url=art.url,
vote_average=art.vote_average,
is_primary=art.is_primary,
)
)
if art.provider_asset_id:
seen_asset_ids.add(art.provider_asset_id)
for row in existing:
if row.provider_asset_id not in seen_asset_ids:
await session.delete(row)
await session.flush()
async def get_artwork(
session: AsyncSession,
entity_type: str,
entity_id: uuid.UUID,
artwork_type: ArtworkType | None = None,
) -> list[Artwork]:
stmt = (
select(Artwork)
.options(selectinload(Artwork.provider))
.where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
.order_by(Artwork.is_primary.desc(), Artwork.vote_average.desc().nulls_last())
)
if artwork_type is not None:
stmt = stmt.where(Artwork.artwork_type == artwork_type)
result = await session.execute(stmt)
return list(result.scalars().all())
async def get_videos(session: AsyncSession, entity_type: str, entity_id: uuid.UUID) -> list[Video]:
stmt = (
select(Video)
.options(selectinload(Video.provider))
.where(Video.entity_type == entity_type, Video.entity_id == entity_id)
.order_by(Video.is_official.desc(), Video.published_at.desc().nulls_last())
)
result = await session.execute(stmt)
return list(result.scalars().all())
async def get_credits(
session: AsyncSession, entity_type: str, entity_id: uuid.UUID
) -> list[Credit]:
stmt = (
select(Credit)
.options(selectinload(Credit.person))
.where(Credit.entity_type == entity_type, Credit.entity_id == entity_id)
.order_by(Credit.billing_order.asc().nulls_last())
)
result = await session.execute(stmt)
return list(result.scalars().all())