Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Section 10: every image API responses reference has pointed straight at TMDb's CDN since day one -- Aurora never actually downloaded or served artwork itself, despite storing dimensions/checksum/local_path columns for exactly that. Closes the gap: - is_primary was declared on the Artwork model but never actually set anywhere -- every row defaulted to False. Now the normalizer marks the one image each provider designates as *the* image for an entity (TMDb's poster_path/backdrop_path/profile_path) as primary, distinct from the dozens of alternates also ingested, including correctly marking an existing row primary (not inserting a duplicate) when that image already appears in the alternates list. - app/services/artwork/: an ArtworkStorage interface with a LocalDiskStorage implementation (checksum-named files, single-host deployment via a Docker volume shared between api and worker -- see docs/architecture.md for the S3/R2 upgrade path), and a download pipeline with content-type validation, a size cap, and path-traversal protection. - After movie/series/person ingestion succeeds, tasks.py cascades one download_artwork_task per primary artwork id via arq's ctx['redis'] (same fan-out pattern as the season cascade) -- fire-and-forget, not tracked in ingestion_jobs, since a failed image cache is a known-safe fallback to the provider URL, not a user-facing failure. - app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is preferred over original_url once set (already-existing fallback logic in app/services/common.py:artwork_url, now actually exercised). Also fixes a real interaction bug this surfaced: every ingestion pipeline deleted and recreated all of an entity's Artwork rows on each re-sync, which would have silently wiped local_path/checksum on every re-ingestion, forcing a full re-download every time and defeating the point of caching locally at all. app/repositories/polymorphic.py's new sync_artwork upserts by provider_asset_id (TMDb's stable per-image path) instead, preserving downloaded copies across re-sync, and replaces three near-identical delete+recreate helpers that used to live separately in movie_ingestion.py, series_ingestion.py, and person_ingestion.py. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cd4dd98adb
commit
fef914db36
@@ -6,6 +6,8 @@ an (entity_type, entity_id) pair rather than a typed foreign key.
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -18,6 +20,24 @@ from app.models.external_id import ExternalID
|
||||
from app.models.provider import Provider
|
||||
from app.models.video import Video
|
||||
|
||||
_VALID_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
|
||||
|
||||
class _NormalizedArtworkLike(Protocol):
|
||||
"""Structural type for app.services.metadata.normalizer.NormalizedArtwork
|
||||
— a repository shouldn't import from the services layer, so this
|
||||
describes the shape `sync_artwork` needs instead of the concrete class.
|
||||
"""
|
||||
|
||||
artwork_type: str
|
||||
url: str
|
||||
provider_asset_id: str | None
|
||||
width: int | None
|
||||
height: int | None
|
||||
aspect_ratio: float | None
|
||||
vote_average: float | None
|
||||
is_primary: bool
|
||||
|
||||
|
||||
async def get_external_ids(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID
|
||||
@@ -48,6 +68,104 @@ async def get_external_ids_bulk(
|
||||
return out
|
||||
|
||||
|
||||
async def get_primary_artwork_ids(
|
||||
session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID]
|
||||
) -> list[uuid.UUID]:
|
||||
"""IDs of artwork rows marked `is_primary` for any of `entity_ids` — the
|
||||
small subset of a title's images (TMDb's own selected poster/backdrop/
|
||||
profile) that get auto-downloaded for local hosting, out of the dozens
|
||||
of alternates ingestion also stores. See app/services/artwork/."""
|
||||
|
||||
if not entity_ids:
|
||||
return []
|
||||
stmt = select(Artwork.id).where(
|
||||
Artwork.entity_type == entity_type,
|
||||
Artwork.entity_id.in_(entity_ids),
|
||||
Artwork.is_primary.is_(True),
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def sync_artwork(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
provider_id: uuid.UUID,
|
||||
artwork_list: Sequence[_NormalizedArtworkLike],
|
||||
) -> None:
|
||||
"""Upsert an entity's artwork rows keyed by `provider_asset_id` (TMDb's
|
||||
stable per-image file path), instead of deleting and recreating them on
|
||||
every re-sync.
|
||||
|
||||
This matters specifically because of `app/services/artwork/` (section
|
||||
10's self-hosting): a delete-then-recreate replace would generate a
|
||||
fresh Artwork.id with `local_path=None` every time a title gets
|
||||
re-ingested, silently throwing away every previously-downloaded local
|
||||
copy and forcing a full re-download on each sync — defeating the point
|
||||
of caching them locally at all. Rows for images no longer present in
|
||||
`artwork_list` (dropped from the provider's alternates) are still
|
||||
deleted, same as before.
|
||||
"""
|
||||
|
||||
existing = (
|
||||
(
|
||||
await session.execute(
|
||||
select(Artwork).where(
|
||||
Artwork.entity_type == entity_type, Artwork.entity_id == entity_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
existing_by_asset_id = {
|
||||
row.provider_asset_id: row for row in existing if row.provider_asset_id is not None
|
||||
}
|
||||
|
||||
seen_asset_ids: set[str] = set()
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _VALID_ARTWORK_TYPES else "poster"
|
||||
matched = (
|
||||
existing_by_asset_id.get(art.provider_asset_id) if art.provider_asset_id else None
|
||||
)
|
||||
if matched is not None:
|
||||
seen_asset_ids.add(art.provider_asset_id) # type: ignore[arg-type]
|
||||
matched.artwork_type = ArtworkType(art_type)
|
||||
matched.width = art.width
|
||||
matched.height = art.height
|
||||
matched.aspect_ratio = art.aspect_ratio
|
||||
matched.vote_average = art.vote_average
|
||||
matched.is_primary = art.is_primary
|
||||
matched.original_url = art.url
|
||||
# local_path/checksum deliberately untouched — see docstring.
|
||||
else:
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
is_primary=art.is_primary,
|
||||
)
|
||||
)
|
||||
if art.provider_asset_id:
|
||||
seen_asset_ids.add(art.provider_asset_id)
|
||||
|
||||
for row in existing:
|
||||
if row.provider_asset_id not in seen_asset_ids:
|
||||
await session.delete(row)
|
||||
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def get_artwork(
|
||||
session: AsyncSession,
|
||||
entity_type: str,
|
||||
|
||||
Reference in New Issue
Block a user