"""TMDb daily ID export fetching, for full-catalog bulk sync. TMDb publishes a full list of every movie/TV/person/collection id that exists in their database once a day, specifically to support this exact use case -- syncing an entire local mirror is what these files are for, as opposed to paginating `/discover`, which caps at 10,000 results per query regardless of how many actually match. Format: https://developer.themoviedb.org/docs/daily-id-exports -- a gzipped file, one JSON object per line, e.g. for movies: `{"adult":false,"id":3924,"original_title":"Blondie","popularity":1.61,"video":false}`. Only `id` is used here; ingestion re-fetches full details per entity through the normal provider pipeline (app/services/ingestion/), so nothing else in the export line matters. """ from __future__ import annotations import gzip import io import json from datetime import date, timedelta import httpx from app.core.logging import get_logger logger = get_logger(__name__) _EXPORT_URL = "http://files.tmdb.org/p/exports/{kind}_ids_{date}.json.gz" # Aurora's entity_type -> the TMDb export file's own name for that kind. _EXPORT_KIND: dict[str, str] = { "movie": "movie", "series": "tv_series", "person": "person", "collection": "collection", } class DailyExportUnavailableError(Exception): pass async def fetch_daily_export_ids(entity_type: str, *, client: httpx.AsyncClient) -> list[str]: """Fetch and parse today's (or, failing that, yesterday's) TMDb daily export for `entity_type`, returning every id as a string. TMDb publishes a new file once a day at a fixed time (early UTC morning); "today's" file may not exist yet depending on when this runs, so this tries today first and falls back to yesterday rather than failing.""" kind = _EXPORT_KIND[entity_type] last_status: int | None = None for day_offset in (0, 1): export_date = date.today() - timedelta(days=day_offset) url = _EXPORT_URL.format(kind=kind, date=export_date.strftime("%m_%d_%Y")) response = await client.get(url, follow_redirects=True) if response.status_code == 200: ids = _parse_export(response.content) logger.info( "tmdb_daily_export_fetched", entity_type=entity_type, url=url, count=len(ids) ) return ids last_status = response.status_code logger.info("tmdb_daily_export_not_found", url=url, status=response.status_code) raise DailyExportUnavailableError( f"no TMDb daily export found for {entity_type} (tried today and yesterday, " f"last status {last_status})" ) def _parse_export(raw_gzip: bytes) -> list[str]: ids: list[str] = [] with gzip.GzipFile(fileobj=io.BytesIO(raw_gzip)) as f: for line in f: line = line.strip() if not line: continue ids.append(str(json.loads(line)["id"])) return ids