Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s

Section 10: every image API responses reference has pointed straight
at TMDb's CDN since day one -- Aurora never actually downloaded or
served artwork itself, despite storing dimensions/checksum/local_path
columns for exactly that. Closes the gap:

- is_primary was declared on the Artwork model but never actually set
  anywhere -- every row defaulted to False. Now the normalizer marks
  the one image each provider designates as *the* image for an entity
  (TMDb's poster_path/backdrop_path/profile_path) as primary,
  distinct from the dozens of alternates also ingested, including
  correctly marking an existing row primary (not inserting a
  duplicate) when that image already appears in the alternates list.
- app/services/artwork/: an ArtworkStorage interface with a
  LocalDiskStorage implementation (checksum-named files, single-host
  deployment via a Docker volume shared between api and worker -- see
  docs/architecture.md for the S3/R2 upgrade path), and a download
  pipeline with content-type validation, a size cap, and path-traversal
  protection.
- After movie/series/person ingestion succeeds, tasks.py cascades one
  download_artwork_task per primary artwork id via arq's ctx['redis']
  (same fan-out pattern as the season cascade) -- fire-and-forget, not
  tracked in ingestion_jobs, since a failed image cache is a
  known-safe fallback to the provider URL, not a user-facing failure.
- app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is
  preferred over original_url once set (already-existing fallback
  logic in app/services/common.py:artwork_url, now actually exercised).

Also fixes a real interaction bug this surfaced: every ingestion
pipeline deleted and recreated all of an entity's Artwork rows on each
re-sync, which would have silently wiped local_path/checksum on every
re-ingestion, forcing a full re-download every time and defeating the
point of caching locally at all. app/repositories/polymorphic.py's new
sync_artwork upserts by provider_asset_id (TMDb's stable per-image
path) instead, preserving downloaded copies across re-sync, and
replaces three near-identical delete+recreate helpers that used to
live separately in movie_ingestion.py, series_ingestion.py, and
person_ingestion.py.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
NixReaper
2026-09-05 03:26:28 -05:00
co-authored by Claude Sonnet 5
parent cd4dd98adb
commit fef914db36
24 changed files with 980 additions and 129 deletions
+14
View File
@@ -65,6 +65,20 @@ class Settings(BaseSettings):
RATE_LIMIT_ENABLED: bool = Field(default=True)
RATE_LIMIT_PER_MINUTE: int = Field(default=120)
# --- Artwork storage (section 10) ---
# Local-disk self-hosting, deliberately simple for 0.1: fine for a
# single-host deployment where the api and worker containers share one
# volume (see docker-compose.yml). A genuinely multi-host deployment
# would need real object storage (S3/R2/...) behind the same
# ArtworkStorage interface instead — see app/services/artwork/storage.py.
MEDIA_ROOT: str = Field(default="./media")
# Root-relative URL path artwork is served under (app/main.py mounts
# MEDIA_ROOT there). Kept relative rather than a full domain so it keeps
# working whether Aurora is reached via localhost, a Traefik host, or
# (once configured) amdbs.net.
MEDIA_URL_PREFIX: str = Field(default="/media")
ARTWORK_DOWNLOAD_MAX_BYTES: int = Field(default=15 * 1024 * 1024)
@field_validator("AURORA_ENV")
@classmethod
def _validate_env(cls, v: str) -> str:
+13
View File
@@ -10,11 +10,13 @@ import time
import uuid
from collections.abc import Awaitable, Callable
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest
from starlette.exceptions import HTTPException as StarletteHTTPException
@@ -78,6 +80,17 @@ app.add_middleware(
allow_headers=["*"],
)
# Serves what app/services/artwork/ downloads (section 10: self-hosted
# artwork). The directory must exist before StaticFiles will mount — the
# worker creates subdirectories as it writes files, but the root itself
# needs to be there even before the first download.
Path(settings.MEDIA_ROOT).mkdir(parents=True, exist_ok=True)
app.mount(
settings.MEDIA_URL_PREFIX,
StaticFiles(directory=settings.MEDIA_ROOT),
name="media",
)
@app.middleware("http")
async def request_context_middleware(
+118
View File
@@ -6,6 +6,8 @@ an (entity_type, entity_id) pair rather than a typed foreign key.
from __future__ import annotations
import uuid
from collections.abc import Sequence
from typing import Protocol
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -18,6 +20,24 @@ from app.models.external_id import ExternalID
from app.models.provider import Provider
from app.models.video import Video
_VALID_ARTWORK_TYPES = {t.value for t in ArtworkType}
class _NormalizedArtworkLike(Protocol):
"""Structural type for app.services.metadata.normalizer.NormalizedArtwork
— a repository shouldn't import from the services layer, so this
describes the shape `sync_artwork` needs instead of the concrete class.
"""
artwork_type: str
url: str
provider_asset_id: str | None
width: int | None
height: int | None
aspect_ratio: float | None
vote_average: float | None
is_primary: bool
async def get_external_ids(
session: AsyncSession, entity_type: str, entity_id: uuid.UUID
@@ -48,6 +68,104 @@ async def get_external_ids_bulk(
return out
async def get_primary_artwork_ids(
session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID]
) -> list[uuid.UUID]:
"""IDs of artwork rows marked `is_primary` for any of `entity_ids` — the
small subset of a title's images (TMDb's own selected poster/backdrop/
profile) that get auto-downloaded for local hosting, out of the dozens
of alternates ingestion also stores. See app/services/artwork/."""
if not entity_ids:
return []
stmt = select(Artwork.id).where(
Artwork.entity_type == entity_type,
Artwork.entity_id.in_(entity_ids),
Artwork.is_primary.is_(True),
)
result = await session.execute(stmt)
return list(result.scalars().all())
async def sync_artwork(
session: AsyncSession,
*,
entity_type: str,
entity_id: uuid.UUID,
provider_id: uuid.UUID,
artwork_list: Sequence[_NormalizedArtworkLike],
) -> None:
"""Upsert an entity's artwork rows keyed by `provider_asset_id` (TMDb's
stable per-image file path), instead of deleting and recreating them on
every re-sync.
This matters specifically because of `app/services/artwork/` (section
10's self-hosting): a delete-then-recreate replace would generate a
fresh Artwork.id with `local_path=None` every time a title gets
re-ingested, silently throwing away every previously-downloaded local
copy and forcing a full re-download on each sync — defeating the point
of caching them locally at all. Rows for images no longer present in
`artwork_list` (dropped from the provider's alternates) are still
deleted, same as before.
"""
existing = (
(
await session.execute(
select(Artwork).where(
Artwork.entity_type == entity_type, Artwork.entity_id == entity_id
)
)
)
.scalars()
.all()
)
existing_by_asset_id = {
row.provider_asset_id: row for row in existing if row.provider_asset_id is not None
}
seen_asset_ids: set[str] = set()
for art in artwork_list:
art_type = art.artwork_type if art.artwork_type in _VALID_ARTWORK_TYPES else "poster"
matched = (
existing_by_asset_id.get(art.provider_asset_id) if art.provider_asset_id else None
)
if matched is not None:
seen_asset_ids.add(art.provider_asset_id) # type: ignore[arg-type]
matched.artwork_type = ArtworkType(art_type)
matched.width = art.width
matched.height = art.height
matched.aspect_ratio = art.aspect_ratio
matched.vote_average = art.vote_average
matched.is_primary = art.is_primary
matched.original_url = art.url
# local_path/checksum deliberately untouched — see docstring.
else:
session.add(
Artwork(
entity_type=entity_type,
entity_id=entity_id,
artwork_type=ArtworkType(art_type),
width=art.width,
height=art.height,
aspect_ratio=art.aspect_ratio,
provider_id=provider_id,
provider_asset_id=art.provider_asset_id,
original_url=art.url,
vote_average=art.vote_average,
is_primary=art.is_primary,
)
)
if art.provider_asset_id:
seen_asset_ids.add(art.provider_asset_id)
for row in existing:
if row.provider_asset_id not in seen_asset_ids:
await session.delete(row)
await session.flush()
async def get_artwork(
session: AsyncSession,
entity_type: str,
View File
+110
View File
@@ -0,0 +1,110 @@
"""Downloads one Artwork row's original_url and stores it locally
(section 10; the "Artwork processing" background worker from section 1's
architecture diagram).
Deliberately narrow: only ever called for artwork already marked
`is_primary` (see `app/services/ingestion/runner.py:get_primary_artwork_ids`
and `app/workers/tasks.py`, which cascade a download job per primary image
after each ingestion succeeds) — TMDb returns dozens of backdrop/logo
alternates per title, and downloading every one of them automatically would
be a lot of unnecessary storage and bandwidth for images Aurora's own
clients are unlikely to ever show. Downloading on request for anything
beyond the primary image is a reasonable follow-up if that's ever needed.
"""
from __future__ import annotations
import hashlib
import uuid
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import get_settings
from app.core.logging import get_logger
from app.models.artwork import Artwork
from app.services.artwork.storage import ArtworkStorage
logger = get_logger(__name__)
_EXTENSION_BY_CONTENT_TYPE = {
"image/jpeg": "jpg",
"image/jpg": "jpg",
"image/png": "png",
"image/webp": "webp",
"image/gif": "gif",
"image/svg+xml": "svg",
}
class ArtworkDownloadError(Exception):
"""Raised for any download-side failure (bad content-type, too large,
network error, ...). Callers (the arq task) let this propagate so the
job gets retried like any other ingestion failure — see
app/services/ingestion/runner.py."""
async def download_artwork(
session: AsyncSession,
storage: ArtworkStorage,
artwork_id: uuid.UUID,
*,
client: httpx.AsyncClient | None = None,
force: bool = False,
) -> bool:
"""Fetch `artwork.original_url`, store it, and set `local_path` +
`checksum`. Returns False (no-op) if the row is gone or already has a
local copy (unless `force=True`); True if it downloaded something.
"""
artwork = await session.get(Artwork, artwork_id)
if artwork is None:
return False
if artwork.local_path and not force:
return False
settings = get_settings()
owns_client = client is None
http_client = client or httpx.AsyncClient(timeout=20.0, follow_redirects=True)
try:
content, content_type = await _fetch(http_client, artwork.original_url, settings.ARTWORK_DOWNLOAD_MAX_BYTES)
finally:
if owns_client:
await http_client.aclose()
checksum = hashlib.sha256(content).hexdigest()
extension = _EXTENSION_BY_CONTENT_TYPE.get(content_type, "jpg")
relative_path = f"{artwork.artwork_type.value}/{checksum[:2]}/{checksum}.{extension}"
local_url = await storage.save(relative_path, content)
artwork.local_path = local_url
artwork.checksum = checksum
await session.flush()
logger.info(
"artwork_downloaded",
artwork_id=str(artwork_id),
checksum=checksum,
bytes=len(content),
)
return True
async def _fetch(client: httpx.AsyncClient, url: str, max_bytes: int) -> tuple[bytes, str]:
try:
async with client.stream("GET", url) as response:
response.raise_for_status()
content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
if not content_type.startswith("image/"):
raise ArtworkDownloadError(f"expected an image, got content-type={content_type!r}")
body = bytearray()
async for chunk in response.aiter_bytes():
body.extend(chunk)
if len(body) > max_bytes:
raise ArtworkDownloadError(
f"artwork at {url} exceeds ARTWORK_DOWNLOAD_MAX_BYTES ({max_bytes} bytes)"
)
return bytes(body), content_type
except httpx.HTTPError as exc:
raise ArtworkDownloadError(f"failed to download {url}: {exc}") from exc
+66
View File
@@ -0,0 +1,66 @@
"""Artwork storage abstraction (section 10).
Downloaded artwork bytes need somewhere to live that both the worker
(writing) and the API (serving, via app/main.py's static mount) can reach.
`ArtworkStorage` is the seam: today there's only `LocalDiskStorage` — fine
for a single Docker host where the api and worker containers share one
volume (see docker-compose.yml), which is this project's actual deployment
shape (section 1: modular monolith, not a multi-host fleet). A deployment
that outgrows a single host would swap in an S3/R2-backed implementation
behind this same interface without touching the download pipeline that
calls it (app/services/artwork/download.py).
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from functools import lru_cache
from pathlib import Path
from app.core.config import get_settings
class ArtworkStorage(ABC):
@abstractmethod
async def save(self, relative_path: str, content: bytes) -> str:
"""Persist `content` at `relative_path` (already includes an
extension, e.g. "poster/ab/ab34...jpg"). Returns the root-relative
URL clients should use to fetch it."""
@abstractmethod
async def exists(self, relative_path: str) -> bool: ...
class LocalDiskStorage(ArtworkStorage):
def __init__(self, root: str, url_prefix: str) -> None:
self._root = Path(root)
self._url_prefix = url_prefix.rstrip("/")
def _full_path(self, relative_path: str) -> Path:
# relative_path is always built by us from a sha256 checksum, never
# from unsanitized external input — but guard anyway against ever
# writing outside MEDIA_ROOT.
root = self._root.resolve()
full = (self._root / relative_path).resolve()
if full != root and root not in full.parents:
raise ValueError(f"unsafe artwork path escapes MEDIA_ROOT: {relative_path!r}")
return full
async def save(self, relative_path: str, content: bytes) -> str:
full = self._full_path(relative_path)
full.parent.mkdir(parents=True, exist_ok=True)
# Blocking I/O, but artwork files are small (capped by
# ARTWORK_DOWNLOAD_MAX_BYTES) and this only ever runs inside the
# worker process, not a request handler — not worth wiring aiofiles
# for.
full.write_bytes(content)
return f"{self._url_prefix}/{relative_path}"
async def exists(self, relative_path: str) -> bool:
return self._full_path(relative_path).is_file()
@lru_cache
def get_artwork_storage() -> ArtworkStorage:
settings = get_settings()
return LocalDiskStorage(settings.MEDIA_ROOT, settings.MEDIA_URL_PREFIX)
+9 -30
View File
@@ -20,15 +20,15 @@ from sqlalchemy import delete
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.models.artwork import Artwork
from app.models.company import MovieCompany
from app.models.credit import Credit
from app.models.enums import ArtworkType, MatchConfidence, MovieStatus, VideoType
from app.models.enums import MatchConfidence, MovieStatus, VideoType
from app.models.genre import MovieGenre
from app.models.keyword import MovieKeyword
from app.models.movie import Movie, MovieRecommendation
from app.models.video import Video
from app.providers.base import MetadataProvider
from app.repositories import polymorphic
from app.repositories import reference_repository as refs
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
from app.services.metadata.matcher import match_entity
@@ -115,36 +115,9 @@ async def _replace_movie_credits(
)
_ARTWORK_TYPES = {t.value for t in ArtworkType}
_VIDEO_TYPES = {t.value for t in VideoType}
async def _replace_movie_artwork(
session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
) -> None:
await session.execute(
delete(Artwork).where(Artwork.entity_type == "movie", Artwork.entity_id == movie_id)
)
for art in artwork_list:
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
session.add(
Artwork(
entity_type="movie",
entity_id=movie_id,
artwork_type=ArtworkType(art_type),
language_code=None,
width=art.width,
height=art.height,
aspect_ratio=art.aspect_ratio,
provider_id=provider_id,
provider_asset_id=art.provider_asset_id,
original_url=art.url,
vote_average=art.vote_average,
is_primary=False,
)
)
async def _replace_movie_videos(
session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, videos
) -> None:
@@ -259,7 +232,13 @@ async def ingest_movie(
await _replace_movie_keywords(session, movie.id, normalized.keywords)
await _replace_movie_companies(session, movie.id, tmdb_provider.id, normalized.companies)
await _replace_movie_credits(session, movie.id, tmdb_provider.id, normalized.cast, normalized.crew)
await _replace_movie_artwork(session, movie.id, tmdb_provider.id, normalized.artwork)
await polymorphic.sync_artwork(
session,
entity_type="movie",
entity_id=movie.id,
provider_id=tmdb_provider.id,
artwork_list=normalized.artwork,
)
await _replace_movie_videos(session, movie.id, tmdb_provider.id, normalized.videos)
await _link_external_ids(session, movie.id, tmdb_provider.id, normalized)
+8 -20
View File
@@ -10,14 +10,13 @@ from __future__ import annotations
import uuid
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.models.artwork import Artwork
from app.models.enums import ArtworkType, MatchConfidence
from app.models.enums import MatchConfidence
from app.models.person import Person
from app.providers.base import MetadataProvider
from app.repositories import polymorphic
from app.repositories import reference_repository as refs
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
from app.services.metadata.matcher import match_entity
@@ -64,24 +63,13 @@ async def ingest_person(
await session.flush()
await session.execute(
delete(Artwork).where(Artwork.entity_type == "person", Artwork.entity_id == person.id)
await polymorphic.sync_artwork(
session,
entity_type="person",
entity_id=person.id,
provider_id=tmdb_provider.id,
artwork_list=normalized.artwork,
)
for art in normalized.artwork:
session.add(
Artwork(
entity_type="person",
entity_id=person.id,
artwork_type=ArtworkType.PROFILE,
width=art.width,
height=art.height,
aspect_ratio=art.aspect_ratio,
provider_id=tmdb_provider.id,
provider_asset_id=art.provider_asset_id,
original_url=art.url,
vote_average=art.vote_average,
)
)
await refs.link_external_id(
session,
+20
View File
@@ -19,6 +19,7 @@ from app.models.enums import IngestionJobStatus
from app.models.provider import IngestionJob
from app.providers.base import MetadataProvider
from app.providers.tmdb import TMDbProvider
from app.repositories.polymorphic import get_primary_artwork_ids
from app.repositories.provider_repository import (
create_ingestion_job,
get_ingestion_job,
@@ -165,3 +166,22 @@ async def get_series_season_numbers(series_id: uuid.UUID) -> list[int]:
async with session_scope() as session:
seasons = await get_seasons_for_series(session, series_id)
return [season.season_number for season in seasons]
async def get_series_season_ids(series_id: uuid.UUID) -> list[uuid.UUID]:
async with session_scope() as session:
seasons = await get_seasons_for_series(session, series_id)
return [season.id for season in seasons]
async def get_primary_artwork_for_download(
entity_type: str, entity_ids: list[uuid.UUID]
) -> list[uuid.UUID]:
"""Artwork ids worth auto-downloading for local hosting after an
ingestion succeeds — see app/services/artwork/download.py and
app/workers/tasks.py, which enqueue one download_artwork_task per id
this returns. A short-lived session of its own, same reasoning as
get_series_season_numbers above."""
async with session_scope() as session:
return await get_primary_artwork_ids(session, entity_type, entity_ids)
+28 -36
View File
@@ -20,17 +20,16 @@ from sqlalchemy import delete
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.models.artwork import Artwork
from app.models.company import SeriesCompany
from app.models.credit import Credit
from app.models.enums import ArtworkType, CompanyType, MatchConfidence, SeriesStatus, VideoType
from app.models.enums import CompanyType, MatchConfidence, SeriesStatus, VideoType
from app.models.genre import SeriesGenre
from app.models.keyword import SeriesKeyword
from app.models.tv import Series
from app.models.video import Video
from app.providers.base import MetadataProvider, ProviderNotFoundError
from app.repositories import polymorphic, tv_repository
from app.repositories import reference_repository as refs
from app.repositories import tv_repository
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
from app.services.metadata.matcher import match_entity
from app.services.metadata.normalizer import (
@@ -41,7 +40,6 @@ from app.services.metadata.normalizer import (
logger = get_logger(__name__)
_ARTWORK_TYPES = {t.value for t in ArtworkType}
_VIDEO_TYPES = {t.value for t in VideoType}
_LEADING_ARTICLES = ("the ", "a ", "an ")
@@ -140,30 +138,6 @@ async def _replace_series_credits(
)
async def _replace_entity_artwork(
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
) -> None:
await session.execute(
delete(Artwork).where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
)
for art in artwork_list:
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
session.add(
Artwork(
entity_type=entity_type,
entity_id=entity_id,
artwork_type=ArtworkType(art_type),
width=art.width,
height=art.height,
aspect_ratio=art.aspect_ratio,
provider_id=provider_id,
provider_asset_id=art.provider_asset_id,
original_url=art.url,
vote_average=art.vote_average,
)
)
async def _replace_entity_videos(
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, videos
) -> None:
@@ -252,7 +226,13 @@ async def ingest_series(
await _replace_series_credits(
session, series.id, tmdb_provider.id, normalized.cast, normalized.crew, normalized.creators
)
await _replace_entity_artwork(session, "series", series.id, tmdb_provider.id, normalized.artwork)
await polymorphic.sync_artwork(
session,
entity_type="series",
entity_id=series.id,
provider_id=tmdb_provider.id,
artwork_list=normalized.artwork,
)
await _replace_entity_videos(session, "series", series.id, tmdb_provider.id, normalized.videos)
for stub in normalized.seasons:
@@ -262,12 +242,12 @@ async def ingest_series(
season.air_date = stub.air_date
season.episode_count = stub.episode_count
await session.flush()
await _replace_entity_artwork(
await polymorphic.sync_artwork(
session,
"season",
season.id,
tmdb_provider.id,
[stub.poster] if stub.poster else [],
entity_type="season",
entity_id=season.id,
provider_id=tmdb_provider.id,
artwork_list=[stub.poster] if stub.poster else [],
)
await refs.link_external_id(
@@ -373,7 +353,13 @@ async def ingest_season(
season.episode_count = normalized.episode_count
await session.flush()
await _replace_entity_artwork(session, "season", season.id, tmdb_provider.id, normalized.artwork)
await polymorphic.sync_artwork(
session,
entity_type="season",
entity_id=season.id,
provider_id=tmdb_provider.id,
artwork_list=normalized.artwork,
)
for ep in normalized.episodes:
episode = await tv_repository.get_or_create_episode(
@@ -390,7 +376,13 @@ async def ingest_season(
await session.flush()
await _replace_episode_credits(session, episode.id, tmdb_provider.id, ep.guest_stars, ep.crew)
await _replace_entity_artwork(session, "episode", episode.id, tmdb_provider.id, ep.stills)
await polymorphic.sync_artwork(
session,
entity_type="episode",
entity_id=episode.id,
provider_id=tmdb_provider.id,
artwork_list=ep.stills,
)
await refs.link_external_id(
session,
entity_type="episode",
+32 -15
View File
@@ -66,6 +66,12 @@ class NormalizedArtwork:
aspect_ratio: float | None
language: str | None
vote_average: float | None = None
# True for the single image the provider designates as *the* image for
# this entity (TMDb's top-level poster_path/backdrop_path/profile_path),
# as opposed to the full alternates list — this is what gets
# auto-downloaded for local hosting (app/services/artwork/), so it's
# deliberately narrow rather than "the first one in the list".
is_primary: bool = False
@dataclass
@@ -182,6 +188,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
None,
None,
None,
is_primary=True,
)
)
if raw.get("backdrop_path"):
@@ -194,6 +201,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
None,
None,
None,
is_primary=True,
)
)
for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
@@ -309,19 +317,26 @@ def normalize_person(raw: dict) -> NormalizedPerson:
)
for img in images
]
if raw.get("profile_path") and not any(a.provider_asset_id == raw["profile_path"] for a in artwork):
artwork.insert(
0,
NormalizedArtwork(
"profile",
_image_url_required(raw["profile_path"]),
raw["profile_path"],
None,
None,
None,
None,
),
if raw.get("profile_path"):
existing = next(
(a for a in artwork if a.provider_asset_id == raw["profile_path"]), None
)
if existing is not None:
existing.is_primary = True
else:
artwork.insert(
0,
NormalizedArtwork(
"profile",
_image_url_required(raw["profile_path"]),
raw["profile_path"],
None,
None,
None,
None,
is_primary=True,
),
)
also_known_as = raw.get("also_known_as") or []
@@ -473,6 +488,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
None,
None,
None,
is_primary=True,
)
)
if raw.get("backdrop_path"):
@@ -485,6 +501,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
None,
None,
None,
is_primary=True,
)
)
for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
@@ -526,7 +543,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
poster=(
NormalizedArtwork(
"poster", _image_url_required(s["poster_path"]), s["poster_path"],
None, None, None, None,
None, None, None, None, is_primary=True,
)
if s.get("poster_path")
else None
@@ -621,7 +638,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
artwork.append(
NormalizedArtwork(
"poster", _image_url_required(raw["poster_path"]), raw["poster_path"],
None, None, None, None,
None, None, None, None, is_primary=True,
)
)
for img in images.get("posters", []):
@@ -656,7 +673,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
stills.append(
NormalizedArtwork(
"still", _image_url_required(ep["still_path"]), ep["still_path"],
None, None, None, None,
None, None, None, None, is_primary=True,
)
)
episodes.append(
+40 -4
View File
@@ -1,10 +1,15 @@
"""arq task functions. Thin wrappers around the ingestion runner so retry/
concurrency/observability behavior lives in arq's worker loop, not here.
Each task receives the `IngestionJob` row's id (created synchronously by the
enqueuing endpoint via `get_or_create_ingestion_job`, *not* generated here)
so status polling has a real, immediately-valid id to look up from the
moment the HTTP response comes back.
Each ingest_* task receives the `IngestionJob` row's id (created
synchronously by the enqueuing endpoint via `get_or_create_ingestion_job`,
*not* generated here) so status polling has a real, immediately-valid id to
look up from the moment the HTTP response comes back. `download_artwork_task`
is the exception — it's an internal side effect of ingestion (see
`_cascade_artwork_downloads`), not something a client triggers or polls, so
it isn't tracked in `ingestion_jobs` at all; arq's own per-function retries
are enough for it (section 10: artwork self-hosting is best-effort caching,
not a user-facing operation with a status to report).
"""
from __future__ import annotations
@@ -12,9 +17,14 @@ from __future__ import annotations
import uuid
from typing import Any
from app.core.database import session_scope
from app.core.logging import get_logger
from app.services.artwork.download import download_artwork
from app.services.artwork.storage import get_artwork_storage
from app.services.ingestion.runner import (
get_or_create_ingestion_job,
get_primary_artwork_for_download,
get_series_season_ids,
get_series_season_numbers,
run_movie_ingestion,
run_person_ingestion,
@@ -25,13 +35,29 @@ from app.services.ingestion.runner import (
logger = get_logger(__name__)
async def _cascade_artwork_downloads(
ctx: dict[str, Any], entity_type: str, entity_ids: list[uuid.UUID]
) -> None:
artwork_ids = await get_primary_artwork_for_download(entity_type, entity_ids)
if not artwork_ids:
return
redis = ctx["redis"]
for artwork_id in artwork_ids:
await redis.enqueue_job("download_artwork_task", str(artwork_id))
logger.info(
"artwork_downloads_cascaded", entity_type=entity_type, count=len(artwork_ids)
)
async def ingest_movie_task(ctx: dict[str, Any], job_id: str, external_id: str) -> str:
movie_id = await run_movie_ingestion(uuid.UUID(job_id), external_id)
await _cascade_artwork_downloads(ctx, "movie", [movie_id])
return str(movie_id)
async def ingest_person_task(ctx: dict[str, Any], job_id: str, external_id: str) -> str:
person_id = await run_person_ingestion(uuid.UUID(job_id), external_id)
await _cascade_artwork_downloads(ctx, "person", [person_id])
return str(person_id)
@@ -39,6 +65,11 @@ async def ingest_series_task(
ctx: dict[str, Any], job_id: str, external_id: str, include_seasons: bool = False
) -> str:
series_id = await run_series_ingestion(uuid.UUID(job_id), external_id)
await _cascade_artwork_downloads(ctx, "series", [series_id])
# Season posters come from the season *stubs* ingest_series already
# creates, independent of include_seasons (which only controls whether
# full per-season episode data gets fetched) -- so this always runs.
await _cascade_artwork_downloads(ctx, "season", await get_series_season_ids(series_id))
if include_seasons:
# Fan out one ingest_season job per season this series has, using
@@ -65,3 +96,8 @@ async def ingest_season_task(
) -> str:
season_id = await run_season_ingestion(uuid.UUID(job_id), series_external_id, season_number)
return str(season_id)
async def download_artwork_task(ctx: dict[str, Any], artwork_id: str) -> bool:
async with session_scope() as session:
return await download_artwork(session, get_artwork_storage(), uuid.UUID(artwork_id))
+8 -1
View File
@@ -14,6 +14,7 @@ from arq.connections import RedisSettings
from app.core.config import get_settings
from app.core.logging import configure_logging, get_logger
from app.workers.tasks import (
download_artwork_task,
ingest_movie_task,
ingest_person_task,
ingest_season_task,
@@ -33,7 +34,13 @@ async def shutdown(ctx: dict) -> None:
class WorkerSettings:
functions = [ingest_movie_task, ingest_person_task, ingest_series_task, ingest_season_task]
functions = [
ingest_movie_task,
ingest_person_task,
ingest_series_task,
ingest_season_task,
download_artwork_task,
]
on_startup = startup
on_shutdown = shutdown
redis_settings = RedisSettings.from_dsn(get_settings().REDIS_URL)