Add artwork self-hosting: local-disk download worker + storage
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Section 10: every image API responses reference has pointed straight at TMDb's CDN since day one -- Aurora never actually downloaded or served artwork itself, despite storing dimensions/checksum/local_path columns for exactly that. Closes the gap: - is_primary was declared on the Artwork model but never actually set anywhere -- every row defaulted to False. Now the normalizer marks the one image each provider designates as *the* image for an entity (TMDb's poster_path/backdrop_path/profile_path) as primary, distinct from the dozens of alternates also ingested, including correctly marking an existing row primary (not inserting a duplicate) when that image already appears in the alternates list. - app/services/artwork/: an ArtworkStorage interface with a LocalDiskStorage implementation (checksum-named files, single-host deployment via a Docker volume shared between api and worker -- see docs/architecture.md for the S3/R2 upgrade path), and a download pipeline with content-type validation, a size cap, and path-traversal protection. - After movie/series/person ingestion succeeds, tasks.py cascades one download_artwork_task per primary artwork id via arq's ctx['redis'] (same fan-out pattern as the season cascade) -- fire-and-forget, not tracked in ingestion_jobs, since a failed image cache is a known-safe fallback to the provider URL, not a user-facing failure. - app/main.py serves MEDIA_ROOT at MEDIA_URL_PREFIX; local_path is preferred over original_url once set (already-existing fallback logic in app/services/common.py:artwork_url, now actually exercised). Also fixes a real interaction bug this surfaced: every ingestion pipeline deleted and recreated all of an entity's Artwork rows on each re-sync, which would have silently wiped local_path/checksum on every re-ingestion, forcing a full re-download every time and defeating the point of caching locally at all. app/repositories/polymorphic.py's new sync_artwork upserts by provider_asset_id (TMDb's stable per-image path) instead, preserving downloaded copies across re-sync, and replaces three near-identical delete+recreate helpers that used to live separately in movie_ingestion.py, series_ingestion.py, and person_ingestion.py. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cd4dd98adb
commit
fef914db36
@@ -0,0 +1,58 @@
|
||||
"""Unit tests for the low-level HTTP fetch used by artwork downloads
|
||||
(app/services/artwork/download.py:_fetch). No database needed — this is
|
||||
pure request/response handling, tested against a mocked transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.services.artwork.download import ArtworkDownloadError, _fetch
|
||||
|
||||
FAKE_JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"0" * 100
|
||||
|
||||
|
||||
def _client_for(handler) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_returns_content_and_content_type():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=FAKE_JPEG_BYTES)
|
||||
|
||||
async with _client_for(handler) as client:
|
||||
content, content_type = await _fetch(client, "https://example.test/poster.jpg", 1_000_000)
|
||||
|
||||
assert content == FAKE_JPEG_BYTES
|
||||
assert content_type == "image/jpeg"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_rejects_non_image_content_type():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/html"}, content=b"<html></html>")
|
||||
|
||||
async with _client_for(handler) as client:
|
||||
with pytest.raises(ArtworkDownloadError, match="expected an image"):
|
||||
await _fetch(client, "https://example.test/oops.html", 1_000_000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_rejects_response_over_max_bytes():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=FAKE_JPEG_BYTES)
|
||||
|
||||
async with _client_for(handler) as client:
|
||||
with pytest.raises(ArtworkDownloadError, match="exceeds ARTWORK_DOWNLOAD_MAX_BYTES"):
|
||||
await _fetch(client, "https://example.test/huge.jpg", max_bytes=10)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_wraps_http_errors():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(404)
|
||||
|
||||
async with _client_for(handler) as client:
|
||||
with pytest.raises(ArtworkDownloadError, match="failed to download"):
|
||||
await _fetch(client, "https://example.test/missing.jpg", 1_000_000)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Unit tests for LocalDiskStorage (app/services/artwork/storage.py)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.artwork.storage import LocalDiskStorage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path) -> LocalDiskStorage:
|
||||
return LocalDiskStorage(str(tmp_path), "/media")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_writes_file_and_returns_public_url(storage: LocalDiskStorage):
|
||||
url = await storage.save("poster/ab/abcdef.jpg", b"hello world")
|
||||
assert url == "/media/poster/ab/abcdef.jpg"
|
||||
assert await storage.exists("poster/ab/abcdef.jpg")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_creates_intermediate_directories(storage: LocalDiskStorage):
|
||||
await storage.save("still/de/deadbeef.png", b"data")
|
||||
assert await storage.exists("still/de/deadbeef.png")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exists_is_false_for_missing_file(storage: LocalDiskStorage):
|
||||
assert not await storage.exists("poster/00/never-written.jpg")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_rejects_path_traversal(storage: LocalDiskStorage):
|
||||
with pytest.raises(ValueError, match="escapes MEDIA_ROOT"):
|
||||
await storage.save("../../etc/passwd", b"malicious")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_url_prefix_trailing_slash_is_normalized(tmp_path):
|
||||
storage = LocalDiskStorage(str(tmp_path), "/media/")
|
||||
url = await storage.save("poster/ab/x.jpg", b"data")
|
||||
assert url == "/media/poster/ab/x.jpg"
|
||||
@@ -145,6 +145,21 @@ def test_normalize_movie_maps_artwork_and_videos():
|
||||
assert movie.videos[0].is_official is True
|
||||
|
||||
|
||||
def test_normalize_movie_marks_exactly_one_primary_poster_and_backdrop():
|
||||
# is_primary picks out TMDb's top-level poster_path/backdrop_path (the
|
||||
# image auto-downloaded for local hosting) from the full alternates
|
||||
# list -- not just "the first one".
|
||||
movie = normalize_movie(FAKE_MOVIE)
|
||||
primary_posters = [a for a in movie.artwork if a.artwork_type == "poster" and a.is_primary]
|
||||
primary_backdrops = [
|
||||
a for a in movie.artwork if a.artwork_type == "backdrop" and a.is_primary
|
||||
]
|
||||
assert len(primary_posters) == 1
|
||||
assert primary_posters[0].provider_asset_id == "/poster.jpg"
|
||||
assert len(primary_backdrops) == 1
|
||||
assert primary_backdrops[0].provider_asset_id == "/backdrop.jpg"
|
||||
|
||||
|
||||
def test_normalize_movie_external_ids_never_include_tmdb_itself():
|
||||
# TMDb's own id is handled separately by the ingestion pipeline via
|
||||
# `normalized.external_id`, not folded into `external_ids` (section 3:
|
||||
@@ -187,6 +202,9 @@ def test_normalize_person_profile_artwork_deduplicated():
|
||||
# not be inserted twice.
|
||||
paths = [a.provider_asset_id for a in person.artwork]
|
||||
assert paths.count("/keanu.jpg") == 1
|
||||
# ...and since it wasn't inserted as a second row, the existing one must
|
||||
# be the one marked primary instead -- not silently left unmarked.
|
||||
assert next(a for a in person.artwork if a.provider_asset_id == "/keanu.jpg").is_primary
|
||||
|
||||
|
||||
FAKE_SERIES = {
|
||||
|
||||
Reference in New Issue
Block a user