Add full-catalog bulk import from TMDb's daily ID exports

app/services/bulk_import.py fetches and parses TMDb's daily ID export
files (published once a day specifically for full-catalog sync, unlike
/discover which caps at 10,000 results per query) -- today's file with
a fallback to yesterday's, since TMDb publishes at a fixed time and
"today's" may not exist yet.

app/scripts/bulk_import.py is a standalone ops script (not an arq task,
since enqueueing millions of jobs would blow past arq's job_timeout)
that fans every id from the export into the existing per-entity
ingestion pipeline -- get_or_create_ingestion_job + the same arq task
each admin endpoint already uses, so a single id's behavior is
identical whether it came from the admin API or this script. Bounded
concurrency (50) on the enqueue phase; idempotency-keyed jobs make
interrupting and re-running the script safe.

Real TMDb catalog size (checked live): ~1.24M movies, 231K series,
4.9M people, 9.8K collections -- ~6.4M entities. At default worker
concurrency this is realistically weeks of continuous processing, not
hours, documented as such in README.md.

Also added 429 (rate-limited) handling to TMDbProvider before running
this at volume: previously a 429 fell into the generic >=400 branch and
failed the request outright with zero retry, since only httpx.TransportError
triggered the existing retry decorator. Now a 429 raises
ProviderRateLimitedError carrying the response's own Retry-After hint,
and tenacity's wait strategy honors that hint directly instead of
guessing via exponential backoff.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

tests/unit/test_bulk_import.py

@@ -0,0 +1,85 @@
+"""Unit tests for app/services/bulk_import.py: parsing TMDb's daily ID
+export format and the today-then-yesterday fallback, using
+httpx.MockTransport so these run without real network access."""
+
+from __future__ import annotations
+
+import gzip
+import json
+
+import httpx
+import pytest
+
+from app.services.bulk_import import DailyExportUnavailableError, fetch_daily_export_ids
+
+
+def _gzip_ndjson(rows: list[dict]) -> bytes:
+    body = "\n".join(json.dumps(row) for row in rows).encode("utf-8")
+    return gzip.compress(body)
+
+
+def _client_for(handler) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+@pytest.mark.asyncio
+async def test_fetch_daily_export_ids_parses_today_file():
+    payload = _gzip_ndjson(
+        [
+            {"adult": False, "id": 603, "original_title": "The Matrix", "popularity": 45.5},
+            {"adult": False, "id": 604, "original_title": "The Matrix Reloaded", "popularity": 18.1},
+        ]
+    )
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, content=payload)
+
+    async with _client_for(handler) as client:
+        ids = await fetch_daily_export_ids("movie", client=client)
+
+    assert ids == ["603", "604"]
+
+
+@pytest.mark.asyncio
+async def test_fetch_daily_export_ids_falls_back_to_yesterday_when_todays_missing():
+    payload = _gzip_ndjson([{"id": 1396, "name": "Breaking Bad"}])
+    calls: list[str] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(str(request.url))
+        if len(calls) == 1:
+            return httpx.Response(404)
+        return httpx.Response(200, content=payload)
+
+    async with _client_for(handler) as client:
+        ids = await fetch_daily_export_ids("series", client=client)
+
+    assert ids == ["1396"]
+    assert len(calls) == 2
+    assert "tv_series_ids" in calls[0]
+
+
+@pytest.mark.asyncio
+async def test_fetch_daily_export_ids_raises_when_neither_day_available():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(404)
+
+    async with _client_for(handler) as client:
+        with pytest.raises(DailyExportUnavailableError):
+            await fetch_daily_export_ids("person", client=client)
+
+
+@pytest.mark.asyncio
+async def test_fetch_daily_export_ids_skips_blank_lines():
+    body = b"\n".join(
+        [json.dumps({"id": 1}).encode(), b"", json.dumps({"id": 2}).encode(), b""]
+    )
+    payload = gzip.compress(body)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, content=payload)
+
+    async with _client_for(handler) as client:
+        ids = await fetch_daily_export_ids("collection", client=client)
+
+    assert ids == ["1", "2"]

tests/unit/test_tmdb_provider.py

@@ -0,0 +1,87 @@
+"""Unit tests for TMDbProvider's 429 (rate-limited) handling -- added
+before running a full-catalog bulk import (app/scripts/bulk_import.py),
+where sustained high request volume makes 429s an expected, recoverable
+condition rather than a rare edge case. Uses httpx.MockTransport, and
+Retry-After: 0 in the mocked responses so retries don't actually slow
+these tests down."""
+
+from __future__ import annotations
+
+from unittest.mock import Mock
+
+import httpx
+import pytest
+
+from app.providers.base import ProviderRateLimitedError
+from app.providers.tmdb import TMDbProvider, _tmdb_wait
+
+
+def _client_for(handler) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="https://api.themoviedb.org/3")
+
+
+def _provider(handler) -> TMDbProvider:
+    return TMDbProvider(api_key="test-key", client=_client_for(handler))
+
+
+@pytest.mark.asyncio
+async def test_get_retries_after_a_single_429_and_succeeds():
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        if len(calls) == 1:
+            return httpx.Response(429, headers={"Retry-After": "0"})
+        return httpx.Response(200, json={"id": 603, "title": "The Matrix"})
+
+    provider = _provider(handler)
+    data = await provider.get_movie("603")
+
+    assert data == {"id": 603, "title": "The Matrix"}
+    assert len(calls) == 2
+
+
+@pytest.mark.asyncio
+async def test_get_reraises_provider_rate_limited_error_after_exhausting_retries():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(429, headers={"Retry-After": "0"})
+
+    provider = _provider(handler)
+
+    with pytest.raises(ProviderRateLimitedError):
+        await provider.get_movie("603")
+
+
+@pytest.mark.asyncio
+async def test_get_defaults_retry_after_when_header_is_not_a_number():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(429, headers={"Retry-After": "Wed, 21 Oct 2026 07:28:00 GMT"})
+
+    provider = _provider(handler)
+
+    with pytest.raises(ProviderRateLimitedError) as exc_info:
+        await provider.get_movie("603")
+
+    assert exc_info.value.retry_after == 1.0
+
+
+def _fake_retry_state(exception: Exception | None) -> Mock:
+    state = Mock()
+    if exception is None:
+        state.outcome = None
+    else:
+        state.outcome.exception.return_value = exception
+    return state
+
+
+def test_tmdb_wait_uses_retry_after_for_rate_limit_errors():
+    state = _fake_retry_state(ProviderRateLimitedError("rate limited", retry_after=12.5))
+    assert _tmdb_wait(state) == 12.5
+
+
+def test_tmdb_wait_falls_back_to_exponential_backoff_for_other_errors():
+    state = _fake_retry_state(httpx.TransportError("boom"))
+    state.attempt_number = 1
+    waited = _tmdb_wait(state)
+    assert isinstance(waited, float)
+    assert waited >= 0