Add full-catalog bulk import from TMDb's daily ID exports

app/services/bulk_import.py fetches and parses TMDb's daily ID export
files (published once a day specifically for full-catalog sync, unlike
/discover which caps at 10,000 results per query) -- today's file with
a fallback to yesterday's, since TMDb publishes at a fixed time and
"today's" may not exist yet.

app/scripts/bulk_import.py is a standalone ops script (not an arq task,
since enqueueing millions of jobs would blow past arq's job_timeout)
that fans every id from the export into the existing per-entity
ingestion pipeline -- get_or_create_ingestion_job + the same arq task
each admin endpoint already uses, so a single id's behavior is
identical whether it came from the admin API or this script. Bounded
concurrency (50) on the enqueue phase; idempotency-keyed jobs make
interrupting and re-running the script safe.

Real TMDb catalog size (checked live): ~1.24M movies, 231K series,
4.9M people, 9.8K collections -- ~6.4M entities. At default worker
concurrency this is realistically weeks of continuous processing, not
hours, documented as such in README.md.

Also added 429 (rate-limited) handling to TMDbProvider before running
this at volume: previously a 429 fell into the generic >=400 branch and
failed the request outright with zero retry, since only httpx.TransportError
triggered the existing retry decorator. Now a 429 raises
ProviderRateLimitedError carrying the response's own Retry-After hint,
and tenacity's wait strategy honors that hint directly instead of
guessing via exponential backoff.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/providers/base.py

@@ -27,6 +27,17 @@ class ProviderNotFoundError(ProviderError):
     """Raised when the provider has no record for the given external ID."""
 
 
+class ProviderRateLimitedError(ProviderError):
+    """Raised when the provider responds 429. Carries how long the provider
+    itself asked callers to wait (`Retry-After`, seconds) so a caller can
+    back off by the right amount rather than guessing — see
+    `app/providers/tmdb.py:TMDbProvider._get` for where this is retried."""
+
+    def __init__(self, message: str, *, retry_after: float) -> None:
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
 class MetadataProvider(ABC):
     slug: str
     name: str

app/providers/tmdb.py

@@ -10,7 +10,13 @@ from __future__ import annotations
 from typing import Any
 
 import httpx
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+from tenacity import (
+    RetryCallState,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from app.core.config import get_settings
 from app.providers.base import (
@@ -18,12 +24,26 @@ from app.providers.base import (
     ProviderError,
     ProviderNotConfiguredError,
     ProviderNotFoundError,
+    ProviderRateLimitedError,
 )
 
 _MOVIE_APPEND = "credits,images,videos,keywords,alternative_titles,external_ids,release_dates,recommendations"
 _SERIES_APPEND = "credits,images,videos,keywords,alternative_titles,external_ids,content_ratings"
 _PERSON_APPEND = "external_ids,images,combined_credits"
 
+_DEFAULT_EXPONENTIAL_WAIT = wait_exponential(multiplier=0.5, max=5)
+
+
+def _tmdb_wait(retry_state: RetryCallState) -> float:
+    """A 429's own `Retry-After` (carried on ProviderRateLimitedError) takes
+    priority over guessing via exponential backoff -- TMDb told us exactly
+    how long to wait, so use that instead of a generic schedule."""
+
+    exc = retry_state.outcome.exception() if retry_state.outcome else None
+    if isinstance(exc, ProviderRateLimitedError):
+        return exc.retry_after
+    return _DEFAULT_EXPONENTIAL_WAIT(retry_state)
+
 
 class TMDbProvider(MetadataProvider):
     slug = "tmdb"
@@ -45,9 +65,9 @@ class TMDbProvider(MetadataProvider):
 
     @retry(
         reraise=True,
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=0.5, max=5),
-        retry=retry_if_exception_type(httpx.TransportError),
+        stop=stop_after_attempt(5),
+        wait=_tmdb_wait,
+        retry=retry_if_exception_type((httpx.TransportError, ProviderRateLimitedError)),
     )
     async def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
         self._require_api_key()
@@ -61,6 +81,17 @@ class TMDbProvider(MetadataProvider):
             raise ProviderNotFoundError(f"TMDb has no record at {path}")
         if response.status_code == 401:
             raise ProviderNotConfiguredError("TMDb rejected the configured TMDB_API_KEY.")
+        if response.status_code == 429:
+            try:
+                # Retry-After is defined (RFC 9110) as either a delay in
+                # seconds or an HTTP-date; TMDb sends seconds, but fall back
+                # rather than crash if that ever changes.
+                retry_after = float(response.headers.get("Retry-After", "1"))
+            except ValueError:
+                retry_after = 1.0
+            raise ProviderRateLimitedError(
+                f"TMDb rate-limited {path}", retry_after=retry_after
+            )
         if response.status_code >= 400:
             raise ProviderError(f"TMDb error {response.status_code} at {path}: {response.text[:300]}")
         return response.json()