NixReaperandClaude Sonnet 5
a52ac676ff
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
app/services/bulk_import.py fetches and parses TMDb's daily ID export files (published once a day specifically for full-catalog sync, unlike /discover which caps at 10,000 results per query) -- today's file with a fallback to yesterday's, since TMDb publishes at a fixed time and "today's" may not exist yet. app/scripts/bulk_import.py is a standalone ops script (not an arq task, since enqueueing millions of jobs would blow past arq's job_timeout) that fans every id from the export into the existing per-entity ingestion pipeline -- get_or_create_ingestion_job + the same arq task each admin endpoint already uses, so a single id's behavior is identical whether it came from the admin API or this script. Bounded concurrency (50) on the enqueue phase; idempotency-keyed jobs make interrupting and re-running the script safe. Real TMDb catalog size (checked live): ~1.24M movies, 231K series, 4.9M people, 9.8K collections -- ~6.4M entities. At default worker concurrency this is realistically weeks of continuous processing, not hours, documented as such in README.md. Also added 429 (rate-limited) handling to TMDbProvider before running this at volume: previously a 429 fell into the generic >=400 branch and failed the request outright with zero retry, since only httpx.TransportError triggered the existing retry decorator. Now a 429 raises ProviderRateLimitedError carrying the response's own Retry-After hint, and tenacity's wait strategy honors that hint directly instead of guessing via exponential backoff. Co-Authored-By: Claude Sonnet 5 <[email protected]>
88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
"""Unit tests for TMDbProvider's 429 (rate-limited) handling -- added
|
|
before running a full-catalog bulk import (app/scripts/bulk_import.py),
|
|
where sustained high request volume makes 429s an expected, recoverable
|
|
condition rather than a rare edge case. Uses httpx.MockTransport, and
|
|
Retry-After: 0 in the mocked responses so retries don't actually slow
|
|
these tests down."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import Mock
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.providers.base import ProviderRateLimitedError
|
|
from app.providers.tmdb import TMDbProvider, _tmdb_wait
|
|
|
|
|
|
def _client_for(handler) -> httpx.AsyncClient:
|
|
return httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="https://api.themoviedb.org/3")
|
|
|
|
|
|
def _provider(handler) -> TMDbProvider:
|
|
return TMDbProvider(api_key="test-key", client=_client_for(handler))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_retries_after_a_single_429_and_succeeds():
|
|
calls: list[int] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
calls.append(1)
|
|
if len(calls) == 1:
|
|
return httpx.Response(429, headers={"Retry-After": "0"})
|
|
return httpx.Response(200, json={"id": 603, "title": "The Matrix"})
|
|
|
|
provider = _provider(handler)
|
|
data = await provider.get_movie("603")
|
|
|
|
assert data == {"id": 603, "title": "The Matrix"}
|
|
assert len(calls) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_reraises_provider_rate_limited_error_after_exhausting_retries():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(429, headers={"Retry-After": "0"})
|
|
|
|
provider = _provider(handler)
|
|
|
|
with pytest.raises(ProviderRateLimitedError):
|
|
await provider.get_movie("603")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_defaults_retry_after_when_header_is_not_a_number():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(429, headers={"Retry-After": "Wed, 21 Oct 2026 07:28:00 GMT"})
|
|
|
|
provider = _provider(handler)
|
|
|
|
with pytest.raises(ProviderRateLimitedError) as exc_info:
|
|
await provider.get_movie("603")
|
|
|
|
assert exc_info.value.retry_after == 1.0
|
|
|
|
|
|
def _fake_retry_state(exception: Exception | None) -> Mock:
|
|
state = Mock()
|
|
if exception is None:
|
|
state.outcome = None
|
|
else:
|
|
state.outcome.exception.return_value = exception
|
|
return state
|
|
|
|
|
|
def test_tmdb_wait_uses_retry_after_for_rate_limit_errors():
|
|
state = _fake_retry_state(ProviderRateLimitedError("rate limited", retry_after=12.5))
|
|
assert _tmdb_wait(state) == 12.5
|
|
|
|
|
|
def test_tmdb_wait_falls_back_to_exponential_backoff_for_other_errors():
|
|
state = _fake_retry_state(httpx.TransportError("boom"))
|
|
state.attempt_number = 1
|
|
waited = _tmdb_wait(state)
|
|
assert isinstance(waited, float)
|
|
assert waited >= 0
|