Add series/season/episode ingestion pipeline
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s

Movie and person ingestion existed, but the series/season/episode API
endpoints had no way to actually populate them from TMDb -- the
biggest remaining gap in Core 0.1's television scope.

- normalize_series/normalize_season in the metadata normalizer,
  including two TMDb quirks that would otherwise silently produce
  empty data: /tv nests keywords and alternative_titles under
  "results" (not "keywords"/"titles" like /movie), and series-level
  creators come from a separate created_by array, not the credits
  cast/crew lists.
- ingest_series upserts the Series row plus season *stubs* (number,
  title, air date, poster -- everything TMDb inlines on /tv/{id});
  ingest_season fetches and upserts one season's full episode list,
  and requires the series to already exist (TMDb has no single
  endpoint for a whole series' episodes, so this has to be a separate
  per-season call).
- Wired through the same job-lifecycle path as movie/person ingestion
  (runner.py, arq tasks, admin endpoints), with the shared boilerplate
  between them factored into runner._run_ingestion and
  admin.py:_enqueue rather than copy-pasted a third and fourth time.
- Unit tests for both normalizers (runnable without a DB) and
  integration tests for the full ingest_series -> ingest_season flow,
  including the series-must-exist-first error and idempotency on
  re-ingestion.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
NixReaper
2026-09-04 19:43:56 -05:00
co-authored by Claude Sonnet 5
parent 669034896a
commit 3f6332559a
11 changed files with 1288 additions and 49 deletions
+53 -29
View File
@@ -1,19 +1,21 @@
"""Orchestrates one ingestion job end-to-end: create/track the `IngestionJob`
row, run the provider fetch + normalize + upsert pipeline, and record
success/failure. Used by both the admin trigger endpoint (section 17) and
the arq worker task (section 15) so the two entry points share identical
the arq worker task (section 15) so every entry point shares identical
job-lifecycle and idempotency behavior.
"""
from __future__ import annotations
import uuid
from collections.abc import Awaitable, Callable
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import session_scope
from app.core.logging import get_logger
from app.models.provider import IngestionJob
from app.providers.base import MetadataProvider
from app.providers.tmdb import TMDbProvider
from app.repositories.provider_repository import (
create_ingestion_job,
@@ -25,6 +27,7 @@ from app.repositories.provider_repository import (
)
from app.services.ingestion.movie_ingestion import ingest_movie
from app.services.ingestion.person_ingestion import ingest_person
from app.services.ingestion.series_ingestion import ingest_season, ingest_series
logger = get_logger(__name__)
@@ -64,9 +67,14 @@ async def get_or_create_ingestion_job(job_type: str, entity_type: str, external_
return job.id
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
"""Run the pipeline for an `IngestionJob` row that already exists (see
`get_or_create_ingestion_job`) — this never creates the row itself."""
async def _run_ingestion(
job_id: uuid.UUID,
label: str,
work: Callable[[AsyncSession, MetadataProvider], Awaitable[uuid.UUID]],
) -> uuid.UUID:
"""Shared job-lifecycle wrapper: mark running, run `work` in its own
session/transaction, mark succeeded/failed. `work` does the actual
fetch+normalize+upsert and returns the resulting Aurora entity id."""
async with session_scope() as session:
job = await _require_job(session, job_id)
@@ -75,13 +83,13 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
provider = TMDbProvider()
try:
async with session_scope() as session:
movie_id = await ingest_movie(session, provider, external_id, ingestion_job_id=job_id)
result_id = await work(session, provider)
async with session_scope() as session:
job = await _require_job(session, job_id)
await mark_job_succeeded(session, job, movie_id)
return movie_id
await mark_job_succeeded(session, job, result_id)
return result_id
except Exception as exc:
logger.error("movie_ingestion_failed", external_id=external_id, error=str(exc))
logger.error(f"{label}_failed", job_id=str(job_id), error=str(exc))
async with session_scope() as session:
job = await _require_job(session, job_id)
await mark_job_failed(session, job, str(exc))
@@ -90,27 +98,43 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
await provider.aclose()
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
return await _run_ingestion(
job_id,
"movie_ingestion",
lambda session, provider: ingest_movie(
session, provider, external_id, ingestion_job_id=job_id
),
)
async def run_person_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
"""Run the pipeline for an `IngestionJob` row that already exists (see
`get_or_create_ingestion_job`) — this never creates the row itself."""
return await _run_ingestion(
job_id,
"person_ingestion",
lambda session, provider: ingest_person(
session, provider, external_id, ingestion_job_id=job_id
),
)
async with session_scope() as session:
job = await _require_job(session, job_id)
await mark_job_running(session, job)
provider = TMDbProvider()
try:
async with session_scope() as session:
person_id = await ingest_person(session, provider, external_id, ingestion_job_id=job_id)
async with session_scope() as session:
job = await _require_job(session, job_id)
await mark_job_succeeded(session, job, person_id)
return person_id
except Exception as exc:
logger.error("person_ingestion_failed", external_id=external_id, error=str(exc))
async with session_scope() as session:
job = await _require_job(session, job_id)
await mark_job_failed(session, job, str(exc))
raise
finally:
await provider.aclose()
async def run_series_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
return await _run_ingestion(
job_id,
"series_ingestion",
lambda session, provider: ingest_series(
session, provider, external_id, ingestion_job_id=job_id
),
)
async def run_season_ingestion(
job_id: uuid.UUID, series_external_id: str, season_number: int
) -> uuid.UUID:
return await _run_ingestion(
job_id,
"season_ingestion",
lambda session, provider: ingest_season(
session, provider, series_external_id, season_number, ingestion_job_id=job_id
),
)