Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Section 10: every image API responses reference has pointed straight at TMDb's CDN since day one -- Aurora never actually downloaded or served artwork itself, despite storing dimensions/checksum/local_path columns for exactly that. Closes the gap: - is_primary was declared on the Artwork model but never actually set anywhere -- every row defaulted to False. Now the normalizer marks the one image each provider designates as *the* image for an entity (TMDb's poster_path/backdrop_path/profile_path) as primary, distinct from the dozens of alternates also ingested, including correctly marking an existing row primary (not inserting a duplicate) when that image already appears in the alternates list. - app/services/artwork/: an ArtworkStorage interface with a LocalDiskStorage implementation (checksum-named files, single-host deployment via a Docker volume shared between api and worker -- see docs/architecture.md for the S3/R2 upgrade path), and a download pipeline with content-type validation, a size cap, and path-traversal protection. - After movie/series/person ingestion succeeds, tasks.py cascades one download_artwork_task per primary artwork id via arq's ctx['redis'] (same fan-out pattern as the season cascade) -- fire-and-forget, not tracked in ingestion_jobs, since a failed image cache is a known-safe fallback to the provider URL, not a user-facing failure. - app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is preferred over original_url once set (already-existing fallback logic in app/services/common.py:artwork_url, now actually exercised). Also fixes a real interaction bug this surfaced: every ingestion pipeline deleted and recreated all of an entity's Artwork rows on each re-sync, which would have silently wiped local_path/checksum on every re-ingestion, forcing a full re-download every time and defeating the point of caching locally at all. app/repositories/polymorphic.py's new sync_artwork upserts by provider_asset_id (TMDb's stable per-image path) instead, preserving downloaded copies across re-sync, and replaces three near-identical delete+recreate helpers that used to live separately in movie_ingestion.py, series_ingestion.py, and person_ingestion.py. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cd4dd98adb
commit
fef914db36
@@ -65,6 +65,20 @@ class Settings(BaseSettings):
|
||||
RATE_LIMIT_ENABLED: bool = Field(default=True)
|
||||
RATE_LIMIT_PER_MINUTE: int = Field(default=120)
|
||||
|
||||
# --- Artwork storage (section 10) ---
|
||||
# Local-disk self-hosting, deliberately simple for 0.1: fine for a
|
||||
# single-host deployment where the api and worker containers share one
|
||||
# volume (see docker-compose.yml). A genuinely multi-host deployment
|
||||
# would need real object storage (S3/R2/...) behind the same
|
||||
# ArtworkStorage interface instead — see app/services/artwork/storage.py.
|
||||
MEDIA_ROOT: str = Field(default="./media")
|
||||
# Root-relative URL path artwork is served under (app/main.py mounts
|
||||
# MEDIA_ROOT there). Kept relative rather than a full domain so it keeps
|
||||
# working whether Aurora is reached via localhost, a Traefik host, or
|
||||
# (once configured) amdbs.net.
|
||||
MEDIA_URL_PREFIX: str = Field(default="/media")
|
||||
ARTWORK_DOWNLOAD_MAX_BYTES: int = Field(default=15 * 1024 * 1024)
|
||||
|
||||
@field_validator("AURORA_ENV")
|
||||
@classmethod
|
||||
def _validate_env(cls, v: str) -> str:
|
||||
|
||||
+13
@@ -10,11 +10,13 @@ import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
@@ -78,6 +80,17 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Serves what app/services/artwork/ downloads (section 10: self-hosted
|
||||
# artwork). The directory must exist before StaticFiles will mount — the
|
||||
# worker creates subdirectories as it writes files, but the root itself
|
||||
# needs to be there even before the first download.
|
||||
Path(settings.MEDIA_ROOT).mkdir(parents=True, exist_ok=True)
|
||||
app.mount(
|
||||
settings.MEDIA_URL_PREFIX,
|
||||
StaticFiles(directory=settings.MEDIA_ROOT),
|
||||
name="media",
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context_middleware(
|
||||
|
||||
@@ -6,6 +6,8 @@ an (entity_type, entity_id) pair rather than a typed foreign key.
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -18,6 +20,24 @@ from app.models.external_id import ExternalID
|
||||
from app.models.provider import Provider
|
||||
from app.models.video import Video
|
||||
|
||||
_VALID_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
|
||||
|
||||
class _NormalizedArtworkLike(Protocol):
|
||||
"""Structural type for app.services.metadata.normalizer.NormalizedArtwork
|
||||
— a repository shouldn't import from the services layer, so this
|
||||
describes the shape `sync_artwork` needs instead of the concrete class.
|
||||
"""
|
||||
|
||||
artwork_type: str
|
||||
url: str
|
||||
provider_asset_id: str | None
|
||||
width: int | None
|
||||
height: int | None
|
||||
aspect_ratio: float | None
|
||||
vote_average: float | None
|
||||
is_primary: bool
|
||||
|
||||
|
||||
async def get_external_ids(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID
|
||||
@@ -48,6 +68,104 @@ async def get_external_ids_bulk(
|
||||
return out
|
||||
|
||||
|
||||
async def get_primary_artwork_ids(
|
||||
session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID]
|
||||
) -> list[uuid.UUID]:
|
||||
"""IDs of artwork rows marked `is_primary` for any of `entity_ids` — the
|
||||
small subset of a title's images (TMDb's own selected poster/backdrop/
|
||||
profile) that get auto-downloaded for local hosting, out of the dozens
|
||||
of alternates ingestion also stores. See app/services/artwork/."""
|
||||
|
||||
if not entity_ids:
|
||||
return []
|
||||
stmt = select(Artwork.id).where(
|
||||
Artwork.entity_type == entity_type,
|
||||
Artwork.entity_id.in_(entity_ids),
|
||||
Artwork.is_primary.is_(True),
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def sync_artwork(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
provider_id: uuid.UUID,
|
||||
artwork_list: Sequence[_NormalizedArtworkLike],
|
||||
) -> None:
|
||||
"""Upsert an entity's artwork rows keyed by `provider_asset_id` (TMDb's
|
||||
stable per-image file path), instead of deleting and recreating them on
|
||||
every re-sync.
|
||||
|
||||
This matters specifically because of `app/services/artwork/` (section
|
||||
10's self-hosting): a delete-then-recreate replace would generate a
|
||||
fresh Artwork.id with `local_path=None` every time a title gets
|
||||
re-ingested, silently throwing away every previously-downloaded local
|
||||
copy and forcing a full re-download on each sync — defeating the point
|
||||
of caching them locally at all. Rows for images no longer present in
|
||||
`artwork_list` (dropped from the provider's alternates) are still
|
||||
deleted, same as before.
|
||||
"""
|
||||
|
||||
existing = (
|
||||
(
|
||||
await session.execute(
|
||||
select(Artwork).where(
|
||||
Artwork.entity_type == entity_type, Artwork.entity_id == entity_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
existing_by_asset_id = {
|
||||
row.provider_asset_id: row for row in existing if row.provider_asset_id is not None
|
||||
}
|
||||
|
||||
seen_asset_ids: set[str] = set()
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _VALID_ARTWORK_TYPES else "poster"
|
||||
matched = (
|
||||
existing_by_asset_id.get(art.provider_asset_id) if art.provider_asset_id else None
|
||||
)
|
||||
if matched is not None:
|
||||
seen_asset_ids.add(art.provider_asset_id) # type: ignore[arg-type]
|
||||
matched.artwork_type = ArtworkType(art_type)
|
||||
matched.width = art.width
|
||||
matched.height = art.height
|
||||
matched.aspect_ratio = art.aspect_ratio
|
||||
matched.vote_average = art.vote_average
|
||||
matched.is_primary = art.is_primary
|
||||
matched.original_url = art.url
|
||||
# local_path/checksum deliberately untouched — see docstring.
|
||||
else:
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
is_primary=art.is_primary,
|
||||
)
|
||||
)
|
||||
if art.provider_asset_id:
|
||||
seen_asset_ids.add(art.provider_asset_id)
|
||||
|
||||
for row in existing:
|
||||
if row.provider_asset_id not in seen_asset_ids:
|
||||
await session.delete(row)
|
||||
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def get_artwork(
|
||||
session: AsyncSession,
|
||||
entity_type: str,
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Downloads one Artwork row's original_url and stores it locally
|
||||
(section 10; the "Artwork processing" background worker from section 1's
|
||||
architecture diagram).
|
||||
|
||||
Deliberately narrow: only ever called for artwork already marked
|
||||
`is_primary` (see `app/services/ingestion/runner.py:get_primary_artwork_ids`
|
||||
and `app/workers/tasks.py`, which cascade a download job per primary image
|
||||
after each ingestion succeeds) — TMDb returns dozens of backdrop/logo
|
||||
alternates per title, and downloading every one of them automatically would
|
||||
be a lot of unnecessary storage and bandwidth for images Aurora's own
|
||||
clients are unlikely to ever show. Downloading on request for anything
|
||||
beyond the primary image is a reasonable follow-up if that's ever needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.services.artwork.storage import ArtworkStorage
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_EXTENSION_BY_CONTENT_TYPE = {
|
||||
"image/jpeg": "jpg",
|
||||
"image/jpg": "jpg",
|
||||
"image/png": "png",
|
||||
"image/webp": "webp",
|
||||
"image/gif": "gif",
|
||||
"image/svg+xml": "svg",
|
||||
}
|
||||
|
||||
|
||||
class ArtworkDownloadError(Exception):
|
||||
"""Raised for any download-side failure (bad content-type, too large,
|
||||
network error, ...). Callers (the arq task) let this propagate so the
|
||||
job gets retried like any other ingestion failure — see
|
||||
app/services/ingestion/runner.py."""
|
||||
|
||||
|
||||
async def download_artwork(
|
||||
session: AsyncSession,
|
||||
storage: ArtworkStorage,
|
||||
artwork_id: uuid.UUID,
|
||||
*,
|
||||
client: httpx.AsyncClient | None = None,
|
||||
force: bool = False,
|
||||
) -> bool:
|
||||
"""Fetch `artwork.original_url`, store it, and set `local_path` +
|
||||
`checksum`. Returns False (no-op) if the row is gone or already has a
|
||||
local copy (unless `force=True`); True if it downloaded something.
|
||||
"""
|
||||
|
||||
artwork = await session.get(Artwork, artwork_id)
|
||||
if artwork is None:
|
||||
return False
|
||||
if artwork.local_path and not force:
|
||||
return False
|
||||
|
||||
settings = get_settings()
|
||||
owns_client = client is None
|
||||
http_client = client or httpx.AsyncClient(timeout=20.0, follow_redirects=True)
|
||||
try:
|
||||
content, content_type = await _fetch(http_client, artwork.original_url, settings.ARTWORK_DOWNLOAD_MAX_BYTES)
|
||||
finally:
|
||||
if owns_client:
|
||||
await http_client.aclose()
|
||||
|
||||
checksum = hashlib.sha256(content).hexdigest()
|
||||
extension = _EXTENSION_BY_CONTENT_TYPE.get(content_type, "jpg")
|
||||
relative_path = f"{artwork.artwork_type.value}/{checksum[:2]}/{checksum}.{extension}"
|
||||
|
||||
local_url = await storage.save(relative_path, content)
|
||||
|
||||
artwork.local_path = local_url
|
||||
artwork.checksum = checksum
|
||||
await session.flush()
|
||||
logger.info(
|
||||
"artwork_downloaded",
|
||||
artwork_id=str(artwork_id),
|
||||
checksum=checksum,
|
||||
bytes=len(content),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _fetch(client: httpx.AsyncClient, url: str, max_bytes: int) -> tuple[bytes, str]:
|
||||
try:
|
||||
async with client.stream("GET", url) as response:
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if not content_type.startswith("image/"):
|
||||
raise ArtworkDownloadError(f"expected an image, got content-type={content_type!r}")
|
||||
|
||||
body = bytearray()
|
||||
async for chunk in response.aiter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > max_bytes:
|
||||
raise ArtworkDownloadError(
|
||||
f"artwork at {url} exceeds ARTWORK_DOWNLOAD_MAX_BYTES ({max_bytes} bytes)"
|
||||
)
|
||||
return bytes(body), content_type
|
||||
except httpx.HTTPError as exc:
|
||||
raise ArtworkDownloadError(f"failed to download {url}: {exc}") from exc
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Artwork storage abstraction (section 10).
|
||||
|
||||
Downloaded artwork bytes need somewhere to live that both the worker
|
||||
(writing) and the API (serving, via app/main.py's static mount) can reach.
|
||||
`ArtworkStorage` is the seam: today there's only `LocalDiskStorage` — fine
|
||||
for a single Docker host where the api and worker containers share one
|
||||
volume (see docker-compose.yml), which is this project's actual deployment
|
||||
shape (section 1: modular monolith, not a multi-host fleet). A deployment
|
||||
that outgrows a single host would swap in an S3/R2-backed implementation
|
||||
behind this same interface without touching the download pipeline that
|
||||
calls it (app/services/artwork/download.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
class ArtworkStorage(ABC):
|
||||
@abstractmethod
|
||||
async def save(self, relative_path: str, content: bytes) -> str:
|
||||
"""Persist `content` at `relative_path` (already includes an
|
||||
extension, e.g. "poster/ab/ab34...jpg"). Returns the root-relative
|
||||
URL clients should use to fetch it."""
|
||||
|
||||
@abstractmethod
|
||||
async def exists(self, relative_path: str) -> bool: ...
|
||||
|
||||
|
||||
class LocalDiskStorage(ArtworkStorage):
|
||||
def __init__(self, root: str, url_prefix: str) -> None:
|
||||
self._root = Path(root)
|
||||
self._url_prefix = url_prefix.rstrip("/")
|
||||
|
||||
def _full_path(self, relative_path: str) -> Path:
|
||||
# relative_path is always built by us from a sha256 checksum, never
|
||||
# from unsanitized external input — but guard anyway against ever
|
||||
# writing outside MEDIA_ROOT.
|
||||
root = self._root.resolve()
|
||||
full = (self._root / relative_path).resolve()
|
||||
if full != root and root not in full.parents:
|
||||
raise ValueError(f"unsafe artwork path escapes MEDIA_ROOT: {relative_path!r}")
|
||||
return full
|
||||
|
||||
async def save(self, relative_path: str, content: bytes) -> str:
|
||||
full = self._full_path(relative_path)
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Blocking I/O, but artwork files are small (capped by
|
||||
# ARTWORK_DOWNLOAD_MAX_BYTES) and this only ever runs inside the
|
||||
# worker process, not a request handler — not worth wiring aiofiles
|
||||
# for.
|
||||
full.write_bytes(content)
|
||||
return f"{self._url_prefix}/{relative_path}"
|
||||
|
||||
async def exists(self, relative_path: str) -> bool:
|
||||
return self._full_path(relative_path).is_file()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_artwork_storage() -> ArtworkStorage:
|
||||
settings = get_settings()
|
||||
return LocalDiskStorage(settings.MEDIA_ROOT, settings.MEDIA_URL_PREFIX)
|
||||
@@ -20,15 +20,15 @@ from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.models.company import MovieCompany
|
||||
from app.models.credit import Credit
|
||||
from app.models.enums import ArtworkType, MatchConfidence, MovieStatus, VideoType
|
||||
from app.models.enums import MatchConfidence, MovieStatus, VideoType
|
||||
from app.models.genre import MovieGenre
|
||||
from app.models.keyword import MovieKeyword
|
||||
from app.models.movie import Movie, MovieRecommendation
|
||||
from app.models.video import Video
|
||||
from app.providers.base import MetadataProvider
|
||||
from app.repositories import polymorphic
|
||||
from app.repositories import reference_repository as refs
|
||||
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
|
||||
from app.services.metadata.matcher import match_entity
|
||||
@@ -115,36 +115,9 @@ async def _replace_movie_credits(
|
||||
)
|
||||
|
||||
|
||||
_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
_VIDEO_TYPES = {t.value for t in VideoType}
|
||||
|
||||
|
||||
async def _replace_movie_artwork(
|
||||
session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Artwork).where(Artwork.entity_type == "movie", Artwork.entity_id == movie_id)
|
||||
)
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type="movie",
|
||||
entity_id=movie_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
language_code=None,
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
is_primary=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _replace_movie_videos(
|
||||
session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, videos
|
||||
) -> None:
|
||||
@@ -259,7 +232,13 @@ async def ingest_movie(
|
||||
await _replace_movie_keywords(session, movie.id, normalized.keywords)
|
||||
await _replace_movie_companies(session, movie.id, tmdb_provider.id, normalized.companies)
|
||||
await _replace_movie_credits(session, movie.id, tmdb_provider.id, normalized.cast, normalized.crew)
|
||||
await _replace_movie_artwork(session, movie.id, tmdb_provider.id, normalized.artwork)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="movie",
|
||||
entity_id=movie.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
await _replace_movie_videos(session, movie.id, tmdb_provider.id, normalized.videos)
|
||||
await _link_external_ids(session, movie.id, tmdb_provider.id, normalized)
|
||||
|
||||
|
||||
@@ -10,14 +10,13 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.models.enums import ArtworkType, MatchConfidence
|
||||
from app.models.enums import MatchConfidence
|
||||
from app.models.person import Person
|
||||
from app.providers.base import MetadataProvider
|
||||
from app.repositories import polymorphic
|
||||
from app.repositories import reference_repository as refs
|
||||
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
|
||||
from app.services.metadata.matcher import match_entity
|
||||
@@ -64,24 +63,13 @@ async def ingest_person(
|
||||
|
||||
await session.flush()
|
||||
|
||||
await session.execute(
|
||||
delete(Artwork).where(Artwork.entity_type == "person", Artwork.entity_id == person.id)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="person",
|
||||
entity_id=person.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
for art in normalized.artwork:
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type="person",
|
||||
entity_id=person.id,
|
||||
artwork_type=ArtworkType.PROFILE,
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=tmdb_provider.id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
)
|
||||
)
|
||||
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.models.enums import IngestionJobStatus
|
||||
from app.models.provider import IngestionJob
|
||||
from app.providers.base import MetadataProvider
|
||||
from app.providers.tmdb import TMDbProvider
|
||||
from app.repositories.polymorphic import get_primary_artwork_ids
|
||||
from app.repositories.provider_repository import (
|
||||
create_ingestion_job,
|
||||
get_ingestion_job,
|
||||
@@ -165,3 +166,22 @@ async def get_series_season_numbers(series_id: uuid.UUID) -> list[int]:
|
||||
async with session_scope() as session:
|
||||
seasons = await get_seasons_for_series(session, series_id)
|
||||
return [season.season_number for season in seasons]
|
||||
|
||||
|
||||
async def get_series_season_ids(series_id: uuid.UUID) -> list[uuid.UUID]:
|
||||
async with session_scope() as session:
|
||||
seasons = await get_seasons_for_series(session, series_id)
|
||||
return [season.id for season in seasons]
|
||||
|
||||
|
||||
async def get_primary_artwork_for_download(
|
||||
entity_type: str, entity_ids: list[uuid.UUID]
|
||||
) -> list[uuid.UUID]:
|
||||
"""Artwork ids worth auto-downloading for local hosting after an
|
||||
ingestion succeeds — see app/services/artwork/download.py and
|
||||
app/workers/tasks.py, which enqueue one download_artwork_task per id
|
||||
this returns. A short-lived session of its own, same reasoning as
|
||||
get_series_season_numbers above."""
|
||||
|
||||
async with session_scope() as session:
|
||||
return await get_primary_artwork_ids(session, entity_type, entity_ids)
|
||||
|
||||
@@ -20,17 +20,16 @@ from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.models.company import SeriesCompany
|
||||
from app.models.credit import Credit
|
||||
from app.models.enums import ArtworkType, CompanyType, MatchConfidence, SeriesStatus, VideoType
|
||||
from app.models.enums import CompanyType, MatchConfidence, SeriesStatus, VideoType
|
||||
from app.models.genre import SeriesGenre
|
||||
from app.models.keyword import SeriesKeyword
|
||||
from app.models.tv import Series
|
||||
from app.models.video import Video
|
||||
from app.providers.base import MetadataProvider, ProviderNotFoundError
|
||||
from app.repositories import polymorphic, tv_repository
|
||||
from app.repositories import reference_repository as refs
|
||||
from app.repositories import tv_repository
|
||||
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
|
||||
from app.services.metadata.matcher import match_entity
|
||||
from app.services.metadata.normalizer import (
|
||||
@@ -41,7 +40,6 @@ from app.services.metadata.normalizer import (
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
_VIDEO_TYPES = {t.value for t in VideoType}
|
||||
|
||||
_LEADING_ARTICLES = ("the ", "a ", "an ")
|
||||
@@ -140,30 +138,6 @@ async def _replace_series_credits(
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_artwork(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Artwork).where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
|
||||
)
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_videos(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, videos
|
||||
) -> None:
|
||||
@@ -252,7 +226,13 @@ async def ingest_series(
|
||||
await _replace_series_credits(
|
||||
session, series.id, tmdb_provider.id, normalized.cast, normalized.crew, normalized.creators
|
||||
)
|
||||
await _replace_entity_artwork(session, "series", series.id, tmdb_provider.id, normalized.artwork)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="series",
|
||||
entity_id=series.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
await _replace_entity_videos(session, "series", series.id, tmdb_provider.id, normalized.videos)
|
||||
|
||||
for stub in normalized.seasons:
|
||||
@@ -262,12 +242,12 @@ async def ingest_series(
|
||||
season.air_date = stub.air_date
|
||||
season.episode_count = stub.episode_count
|
||||
await session.flush()
|
||||
await _replace_entity_artwork(
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
"season",
|
||||
season.id,
|
||||
tmdb_provider.id,
|
||||
[stub.poster] if stub.poster else [],
|
||||
entity_type="season",
|
||||
entity_id=season.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=[stub.poster] if stub.poster else [],
|
||||
)
|
||||
|
||||
await refs.link_external_id(
|
||||
@@ -373,7 +353,13 @@ async def ingest_season(
|
||||
season.episode_count = normalized.episode_count
|
||||
await session.flush()
|
||||
|
||||
await _replace_entity_artwork(session, "season", season.id, tmdb_provider.id, normalized.artwork)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="season",
|
||||
entity_id=season.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=normalized.artwork,
|
||||
)
|
||||
|
||||
for ep in normalized.episodes:
|
||||
episode = await tv_repository.get_or_create_episode(
|
||||
@@ -390,7 +376,13 @@ async def ingest_season(
|
||||
await session.flush()
|
||||
|
||||
await _replace_episode_credits(session, episode.id, tmdb_provider.id, ep.guest_stars, ep.crew)
|
||||
await _replace_entity_artwork(session, "episode", episode.id, tmdb_provider.id, ep.stills)
|
||||
await polymorphic.sync_artwork(
|
||||
session,
|
||||
entity_type="episode",
|
||||
entity_id=episode.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
artwork_list=ep.stills,
|
||||
)
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
entity_type="episode",
|
||||
|
||||
@@ -66,6 +66,12 @@ class NormalizedArtwork:
|
||||
aspect_ratio: float | None
|
||||
language: str | None
|
||||
vote_average: float | None = None
|
||||
# True for the single image the provider designates as *the* image for
|
||||
# this entity (TMDb's top-level poster_path/backdrop_path/profile_path),
|
||||
# as opposed to the full alternates list — this is what gets
|
||||
# auto-downloaded for local hosting (app/services/artwork/), so it's
|
||||
# deliberately narrow rather than "the first one in the list".
|
||||
is_primary: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -182,6 +188,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
is_primary=True,
|
||||
)
|
||||
)
|
||||
if raw.get("backdrop_path"):
|
||||
@@ -194,6 +201,7 @@ def normalize_movie(raw: dict) -> NormalizedMovie:
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
is_primary=True,
|
||||
)
|
||||
)
|
||||
for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
|
||||
@@ -309,19 +317,26 @@ def normalize_person(raw: dict) -> NormalizedPerson:
|
||||
)
|
||||
for img in images
|
||||
]
|
||||
if raw.get("profile_path") and not any(a.provider_asset_id == raw["profile_path"] for a in artwork):
|
||||
artwork.insert(
|
||||
0,
|
||||
NormalizedArtwork(
|
||||
"profile",
|
||||
_image_url_required(raw["profile_path"]),
|
||||
raw["profile_path"],
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
if raw.get("profile_path"):
|
||||
existing = next(
|
||||
(a for a in artwork if a.provider_asset_id == raw["profile_path"]), None
|
||||
)
|
||||
if existing is not None:
|
||||
existing.is_primary = True
|
||||
else:
|
||||
artwork.insert(
|
||||
0,
|
||||
NormalizedArtwork(
|
||||
"profile",
|
||||
_image_url_required(raw["profile_path"]),
|
||||
raw["profile_path"],
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
is_primary=True,
|
||||
),
|
||||
)
|
||||
|
||||
also_known_as = raw.get("also_known_as") or []
|
||||
|
||||
@@ -473,6 +488,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
is_primary=True,
|
||||
)
|
||||
)
|
||||
if raw.get("backdrop_path"):
|
||||
@@ -485,6 +501,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
is_primary=True,
|
||||
)
|
||||
)
|
||||
for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
|
||||
@@ -526,7 +543,7 @@ def normalize_series(raw: dict) -> NormalizedSeries:
|
||||
poster=(
|
||||
NormalizedArtwork(
|
||||
"poster", _image_url_required(s["poster_path"]), s["poster_path"],
|
||||
None, None, None, None,
|
||||
None, None, None, None, is_primary=True,
|
||||
)
|
||||
if s.get("poster_path")
|
||||
else None
|
||||
@@ -621,7 +638,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
"poster", _image_url_required(raw["poster_path"]), raw["poster_path"],
|
||||
None, None, None, None,
|
||||
None, None, None, None, is_primary=True,
|
||||
)
|
||||
)
|
||||
for img in images.get("posters", []):
|
||||
@@ -656,7 +673,7 @@ def normalize_season(raw: dict) -> NormalizedSeason:
|
||||
stills.append(
|
||||
NormalizedArtwork(
|
||||
"still", _image_url_required(ep["still_path"]), ep["still_path"],
|
||||
None, None, None, None,
|
||||
None, None, None, None, is_primary=True,
|
||||
)
|
||||
)
|
||||
episodes.append(
|
||||
|
||||
+40
-4
@@ -1,10 +1,15 @@
|
||||
"""arq task functions. Thin wrappers around the ingestion runner so retry/
|
||||
concurrency/observability behavior lives in arq's worker loop, not here.
|
||||
|
||||
Each task receives the `IngestionJob` row's id (created synchronously by the
|
||||
enqueuing endpoint via `get_or_create_ingestion_job`, *not* generated here)
|
||||
so status polling has a real, immediately-valid id to look up from the
|
||||
moment the HTTP response comes back.
|
||||
Each ingest_* task receives the `IngestionJob` row's id (created
|
||||
synchronously by the enqueuing endpoint via `get_or_create_ingestion_job`,
|
||||
*not* generated here) so status polling has a real, immediately-valid id to
|
||||
look up from the moment the HTTP response comes back. `download_artwork_task`
|
||||
is the exception — it's an internal side effect of ingestion (see
|
||||
`_cascade_artwork_downloads`), not something a client triggers or polls, so
|
||||
it isn't tracked in `ingestion_jobs` at all; arq's own per-function retries
|
||||
are enough for it (section 10: artwork self-hosting is best-effort caching,
|
||||
not a user-facing operation with a status to report).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,9 +17,14 @@ from __future__ import annotations
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.core.database import session_scope
|
||||
from app.core.logging import get_logger
|
||||
from app.services.artwork.download import download_artwork
|
||||
from app.services.artwork.storage import get_artwork_storage
|
||||
from app.services.ingestion.runner import (
|
||||
get_or_create_ingestion_job,
|
||||
get_primary_artwork_for_download,
|
||||
get_series_season_ids,
|
||||
get_series_season_numbers,
|
||||
run_movie_ingestion,
|
||||
run_person_ingestion,
|
||||
@@ -25,13 +35,29 @@ from app.services.ingestion.runner import (
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def _cascade_artwork_downloads(
|
||||
ctx: dict[str, Any], entity_type: str, entity_ids: list[uuid.UUID]
|
||||
) -> None:
|
||||
artwork_ids = await get_primary_artwork_for_download(entity_type, entity_ids)
|
||||
if not artwork_ids:
|
||||
return
|
||||
redis = ctx["redis"]
|
||||
for artwork_id in artwork_ids:
|
||||
await redis.enqueue_job("download_artwork_task", str(artwork_id))
|
||||
logger.info(
|
||||
"artwork_downloads_cascaded", entity_type=entity_type, count=len(artwork_ids)
|
||||
)
|
||||
|
||||
|
||||
async def ingest_movie_task(ctx: dict[str, Any], job_id: str, external_id: str) -> str:
|
||||
movie_id = await run_movie_ingestion(uuid.UUID(job_id), external_id)
|
||||
await _cascade_artwork_downloads(ctx, "movie", [movie_id])
|
||||
return str(movie_id)
|
||||
|
||||
|
||||
async def ingest_person_task(ctx: dict[str, Any], job_id: str, external_id: str) -> str:
|
||||
person_id = await run_person_ingestion(uuid.UUID(job_id), external_id)
|
||||
await _cascade_artwork_downloads(ctx, "person", [person_id])
|
||||
return str(person_id)
|
||||
|
||||
|
||||
@@ -39,6 +65,11 @@ async def ingest_series_task(
|
||||
ctx: dict[str, Any], job_id: str, external_id: str, include_seasons: bool = False
|
||||
) -> str:
|
||||
series_id = await run_series_ingestion(uuid.UUID(job_id), external_id)
|
||||
await _cascade_artwork_downloads(ctx, "series", [series_id])
|
||||
# Season posters come from the season *stubs* ingest_series already
|
||||
# creates, independent of include_seasons (which only controls whether
|
||||
# full per-season episode data gets fetched) -- so this always runs.
|
||||
await _cascade_artwork_downloads(ctx, "season", await get_series_season_ids(series_id))
|
||||
|
||||
if include_seasons:
|
||||
# Fan out one ingest_season job per season this series has, using
|
||||
@@ -65,3 +96,8 @@ async def ingest_season_task(
|
||||
) -> str:
|
||||
season_id = await run_season_ingestion(uuid.UUID(job_id), series_external_id, season_number)
|
||||
return str(season_id)
|
||||
|
||||
|
||||
async def download_artwork_task(ctx: dict[str, Any], artwork_id: str) -> bool:
|
||||
async with session_scope() as session:
|
||||
return await download_artwork(session, get_artwork_storage(), uuid.UUID(artwork_id))
|
||||
|
||||
@@ -14,6 +14,7 @@ from arq.connections import RedisSettings
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import configure_logging, get_logger
|
||||
from app.workers.tasks import (
|
||||
download_artwork_task,
|
||||
ingest_movie_task,
|
||||
ingest_person_task,
|
||||
ingest_season_task,
|
||||
@@ -33,7 +34,13 @@ async def shutdown(ctx: dict) -> None:
|
||||
|
||||
|
||||
class WorkerSettings:
|
||||
functions = [ingest_movie_task, ingest_person_task, ingest_series_task, ingest_season_task]
|
||||
functions = [
|
||||
ingest_movie_task,
|
||||
ingest_person_task,
|
||||
ingest_series_task,
|
||||
ingest_season_task,
|
||||
download_artwork_task,
|
||||
]
|
||||
on_startup = startup
|
||||
on_shutdown = shutdown
|
||||
redis_settings = RedisSettings.from_dsn(get_settings().REDIS_URL)
|
||||
|
||||
Reference in New Issue
Block a user