Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s

Section 10: every image API responses reference has pointed straight
at TMDb's CDN since day one -- Aurora never actually downloaded or
served artwork itself, despite storing dimensions/checksum/local_path
columns for exactly that. Closes the gap:

- is_primary was declared on the Artwork model but never actually set
  anywhere -- every row defaulted to False. Now the normalizer marks
  the one image each provider designates as *the* image for an entity
  (TMDb's poster_path/backdrop_path/profile_path) as primary,
  distinct from the dozens of alternates also ingested, including
  correctly marking an existing row primary (not inserting a
  duplicate) when that image already appears in the alternates list.
- app/services/artwork/: an ArtworkStorage interface with a
  LocalDiskStorage implementation (checksum-named files, single-host
  deployment via a Docker volume shared between api and worker -- see
  docs/architecture.md for the S3/R2 upgrade path), and a download
  pipeline with content-type validation, a size cap, and path-traversal
  protection.
- After movie/series/person ingestion succeeds, tasks.py cascades one
  download_artwork_task per primary artwork id via arq's ctx['redis']
  (same fan-out pattern as the season cascade) -- fire-and-forget, not
  tracked in ingestion_jobs, since a failed image cache is a
  known-safe fallback to the provider URL, not a user-facing failure.
- app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is
  preferred over original_url once set (already-existing fallback
  logic in app/services/common.py:artwork_url, now actually exercised).

Also fixes a real interaction bug this surfaced: every ingestion
pipeline deleted and recreated all of an entity's Artwork rows on each
re-sync, which would have silently wiped local_path/checksum on every
re-ingestion, forcing a full re-download every time and defeating the
point of caching locally at all. app/repositories/polymorphic.py's new
sync_artwork upserts by provider_asset_id (TMDb's stable per-image
path) instead, preserving downloaded copies across re-sync, and
replaces three near-identical delete+recreate helpers that used to
live separately in movie_ingestion.py, series_ingestion.py, and
person_ingestion.py.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
NixReaper
2026-09-05 03:26:28 -05:00
co-authored by Claude Sonnet 5
parent cd4dd98adb
commit fef914db36
24 changed files with 980 additions and 129 deletions
+116
View File
@@ -0,0 +1,116 @@
"""Integration tests for the artwork download pipeline
(app/services/artwork/download.py). Uses httpx.MockTransport to simulate
TMDb's CDN so these run without network access, while still exercising a
real Artwork row against PostgreSQL."""
from __future__ import annotations
import httpx
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.artwork import Artwork
from app.models.enums import ArtworkType
from app.services.artwork.download import ArtworkDownloadError, download_artwork
from app.services.artwork.storage import LocalDiskStorage
from tests.integration.factories import make_provider
FAKE_JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"0" * 100 # not a *real* JPEG, just image-shaped bytes
def _client_for(handler) -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.MockTransport(handler))
async def _make_artwork(session: AsyncSession, *, local_path: str | None = None) -> Artwork:
provider = await make_provider(session)
artwork = Artwork(
entity_type="movie",
entity_id=provider.id, # any UUID; polymorphic tables have no real FK
artwork_type=ArtworkType.POSTER,
provider_id=provider.id,
original_url="https://image.tmdb.org/t/p/original/poster.jpg",
local_path=local_path,
)
session.add(artwork)
await session.flush()
return artwork
@pytest.mark.asyncio
async def test_download_artwork_saves_file_and_sets_local_path_and_checksum(
db_session: AsyncSession, tmp_path
):
artwork = await _make_artwork(db_session)
storage = LocalDiskStorage(str(tmp_path), "/media")
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=FAKE_JPEG_BYTES)
async with _client_for(handler) as client:
result = await download_artwork(db_session, storage, artwork.id, client=client)
assert result is True
await db_session.refresh(artwork)
assert artwork.local_path is not None
assert artwork.local_path.startswith("/media/poster/")
assert artwork.checksum is not None
assert await storage.exists(artwork.local_path.removeprefix("/media/"))
@pytest.mark.asyncio
async def test_download_artwork_skips_when_already_downloaded(db_session: AsyncSession, tmp_path):
artwork = await _make_artwork(db_session, local_path="/media/poster/ab/existing.jpg")
storage = LocalDiskStorage(str(tmp_path), "/media")
def handler(request: httpx.Request) -> httpx.Response:
raise AssertionError("should not make an HTTP request when already downloaded")
async with _client_for(handler) as client:
result = await download_artwork(db_session, storage, artwork.id, client=client)
assert result is False
@pytest.mark.asyncio
async def test_download_artwork_force_redownloads(db_session: AsyncSession, tmp_path):
artwork = await _make_artwork(db_session, local_path="/media/poster/ab/existing.jpg")
storage = LocalDiskStorage(str(tmp_path), "/media")
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "image/png"}, content=FAKE_JPEG_BYTES)
async with _client_for(handler) as client:
result = await download_artwork(db_session, storage, artwork.id, client=client, force=True)
assert result is True
@pytest.mark.asyncio
async def test_download_artwork_rejects_non_image_content_type(
db_session: AsyncSession, tmp_path
):
artwork = await _make_artwork(db_session)
storage = LocalDiskStorage(str(tmp_path), "/media")
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "text/html"}, content=b"<html></html>")
async with _client_for(handler) as client:
with pytest.raises(ArtworkDownloadError, match="expected an image"):
await download_artwork(db_session, storage, artwork.id, client=client)
@pytest.mark.asyncio
async def test_download_artwork_returns_false_for_missing_row(db_session: AsyncSession, tmp_path):
import uuid
storage = LocalDiskStorage(str(tmp_path), "/media")
def handler(request: httpx.Request) -> httpx.Response:
raise AssertionError("should not make an HTTP request for a nonexistent artwork row")
async with _client_for(handler) as client:
result = await download_artwork(db_session, storage, uuid.uuid4(), client=client)
assert result is False
+162
View File
@@ -0,0 +1,162 @@
"""Integration tests for app/repositories/polymorphic.py:sync_artwork.
The one guarantee this module exists for: re-ingesting a title must not
wipe out artwork that app/services/artwork/ already downloaded locally.
Before sync_artwork, every ingestion pipeline deleted and recreated all of
an entity's Artwork rows on each sync, which would silently reset
local_path/checksum to None every time -- defeating the entire point of
caching artwork locally.
"""
from __future__ import annotations
import uuid
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.artwork import Artwork
from app.repositories.polymorphic import sync_artwork
from app.services.metadata.normalizer import NormalizedArtwork
from tests.integration.factories import make_provider
def _artwork(asset_id: str, *, vote_average: float = 5.0, is_primary: bool = False) -> NormalizedArtwork:
return NormalizedArtwork(
artwork_type="poster",
url=f"https://image.tmdb.org/t/p/original/{asset_id}",
provider_asset_id=asset_id,
width=1000,
height=1500,
aspect_ratio=0.667,
language=None,
vote_average=vote_average,
is_primary=is_primary,
)
@pytest.mark.asyncio
async def test_sync_artwork_creates_rows_on_first_sync(db_session: AsyncSession):
provider = await make_provider(db_session)
entity_id = uuid.uuid4()
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg", is_primary=True), _artwork("b.jpg")],
)
await db_session.commit()
rows = (
(await db_session.execute(select(Artwork).where(Artwork.entity_id == entity_id)))
.scalars()
.all()
)
assert {r.provider_asset_id for r in rows} == {"a.jpg", "b.jpg"}
@pytest.mark.asyncio
async def test_sync_artwork_preserves_local_path_across_resync(db_session: AsyncSession):
provider = await make_provider(db_session)
entity_id = uuid.uuid4()
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg", vote_average=5.0)],
)
await db_session.flush()
row = (
await db_session.execute(select(Artwork).where(Artwork.entity_id == entity_id))
).scalar_one()
row.local_path = "/media/poster/ab/deadbeef.jpg"
row.checksum = "deadbeef"
await db_session.flush()
# Re-sync as if the movie were re-ingested: same image, but TMDb now
# reports a different vote_average.
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg", vote_average=9.0)],
)
await db_session.commit()
refreshed = (
await db_session.execute(select(Artwork).where(Artwork.entity_id == entity_id))
).scalar_one()
assert refreshed.id == row.id # same row, not delete+recreate
assert refreshed.local_path == "/media/poster/ab/deadbeef.jpg"
assert refreshed.checksum == "deadbeef"
assert refreshed.vote_average == 9.0 # non-download fields still update
@pytest.mark.asyncio
async def test_sync_artwork_removes_rows_no_longer_present(db_session: AsyncSession):
provider = await make_provider(db_session)
entity_id = uuid.uuid4()
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg"), _artwork("b.jpg")],
)
await db_session.flush()
# "b.jpg" dropped from the provider's alternates on re-sync.
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg")],
)
await db_session.commit()
rows = (
(await db_session.execute(select(Artwork).where(Artwork.entity_id == entity_id)))
.scalars()
.all()
)
assert {r.provider_asset_id for r in rows} == {"a.jpg"}
@pytest.mark.asyncio
async def test_sync_artwork_adds_new_rows_on_resync(db_session: AsyncSession):
provider = await make_provider(db_session)
entity_id = uuid.uuid4()
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg")],
)
await db_session.flush()
await sync_artwork(
db_session,
entity_type="movie",
entity_id=entity_id,
provider_id=provider.id,
artwork_list=[_artwork("a.jpg"), _artwork("c.jpg")],
)
await db_session.commit()
rows = (
(await db_session.execute(select(Artwork).where(Artwork.entity_id == entity_id)))
.scalars()
.all()
)
assert {r.provider_asset_id for r in rows} == {"a.jpg", "c.jpg"}