Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Section 10: every image API responses reference has pointed straight at TMDb's CDN since day one -- Aurora never actually downloaded or served artwork itself, despite storing dimensions/checksum/local_path columns for exactly that. Closes the gap: - is_primary was declared on the Artwork model but never actually set anywhere -- every row defaulted to False. Now the normalizer marks the one image each provider designates as *the* image for an entity (TMDb's poster_path/backdrop_path/profile_path) as primary, distinct from the dozens of alternates also ingested, including correctly marking an existing row primary (not inserting a duplicate) when that image already appears in the alternates list. - app/services/artwork/: an ArtworkStorage interface with a LocalDiskStorage implementation (checksum-named files, single-host deployment via a Docker volume shared between api and worker -- see docs/architecture.md for the S3/R2 upgrade path), and a download pipeline with content-type validation, a size cap, and path-traversal protection. - After movie/series/person ingestion succeeds, tasks.py cascades one download_artwork_task per primary artwork id via arq's ctx['redis'] (same fan-out pattern as the season cascade) -- fire-and-forget, not tracked in ingestion_jobs, since a failed image cache is a known-safe fallback to the provider URL, not a user-facing failure. - app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is preferred over original_url once set (already-existing fallback logic in app/services/common.py:artwork_url, now actually exercised). Also fixes a real interaction bug this surfaced: every ingestion pipeline deleted and recreated all of an entity's Artwork rows on each re-sync, which would have silently wiped local_path/checksum on every re-ingestion, forcing a full re-download every time and defeating the point of caching locally at all. app/repositories/polymorphic.py's new sync_artwork upserts by provider_asset_id (TMDb's stable per-image path) instead, preserving downloaded copies across re-sync, and replaces three near-identical delete+recreate helpers that used to live separately in movie_ingestion.py, series_ingestion.py, and person_ingestion.py. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cd4dd98adb
commit
fef914db36
@@ -20,17 +20,16 @@ from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.models.company import SeriesCompany
|
||||
from app.models.credit import Credit
|
||||
from app.models.enums import ArtworkType, CompanyType, MatchConfidence, SeriesStatus, VideoType
|
||||
from app.models.enums import CompanyType, MatchConfidence, SeriesStatus, VideoType
|
||||
from app.models.genre import SeriesGenre
|
||||
from app.models.keyword import SeriesKeyword
|
||||
from app.models.tv import Series
|
||||
from app.models.video import Video
|
||||
from app.providers.base import MetadataProvider, ProviderNotFoundError
|
||||
from app.repositories import polymorphic, tv_repository
|
||||
from app.repositories import reference_repository as refs
|
||||
from app.repositories import tv_repository
|
||||
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
|
||||
from app.services.metadata.matcher import match_entity
|
||||
from app.services.metadata.normalizer import (
|
||||
@@ -41,7 +40,6 @@ from app.services.metadata.normalizer import (
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
_VIDEO_TYPES = {t.value for t in VideoType}
|
||||
|
||||
_LEADING_ARTICLES = ("the ", "a ", "an ")
|
||||
@@ -140,30 +138,6 @@ async def _replace_series_credits(
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_artwork(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Artwork).where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
|
||||
)
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_videos(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, videos
|
||||
) -> None:
|
||||
@@ -252,7 +226,13 @@ async def ingest_series(
|
||||
await _replace_series_credits(
|
||||
session, series.id, tmdb_provider.id, normalized.cast, normalized.crew, normalized.creators
|
||||
)
|
||||
await _replace_entity_artwork(session, "series", series.id, tmdb_provider.id, normalized.artwork)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="series",
|
||||
entity_id=series.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
await _replace_entity_videos(session, "series", series.id, tmdb_provider.id, normalized.videos)
|
||||
|
||||
for stub in normalized.seasons:
|
||||
@@ -262,12 +242,12 @@ async def ingest_series(
|
||||
season.air_date = stub.air_date
|
||||
season.episode_count = stub.episode_count
|
||||
await session.flush()
|
||||
await _replace_entity_artwork(
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
"season",
|
||||
season.id,
|
||||
tmdb_provider.id,
|
||||
[stub.poster] if stub.poster else [],
|
||||
entity_type="season",
|
||||
entity_id=season.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=[stub.poster] if stub.poster else [],
|
||||
)
|
||||
|
||||
await refs.link_external_id(
|
||||
@@ -373,7 +353,13 @@ async def ingest_season(
|
||||
season.episode_count = normalized.episode_count
|
||||
await session.flush()
|
||||
|
||||
await _replace_entity_artwork(session, "season", season.id, tmdb_provider.id, normalized.artwork)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="season",
|
||||
entity_id=season.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
|
||||
for ep in normalized.episodes:
|
||||
episode = await tv_repository.get_or_create_episode(
|
||||
@@ -390,7 +376,13 @@ async def ingest_season(
|
||||
await session.flush()
|
||||
|
||||
await _replace_episode_credits(session, episode.id, tmdb_provider.id, ep.guest_stars, ep.crew)
|
||||
await _replace_entity_artwork(session, "episode", episode.id, tmdb_provider.id, ep.stills)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="episode",
|
||||
entity_id=episode.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=ep.stills,
|
||||
)
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
entity_type="episode",
|
||||
|
||||
Reference in New Issue
Block a user