Add series/season/episode ingestion pipeline
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Movie and person ingestion existed, but the series/season/episode API
endpoints had no way to actually populate them from TMDb -- the
biggest remaining gap in Core 0.1's television scope.
- normalize_series/normalize_season in the metadata normalizer,
including two TMDb quirks that would otherwise silently produce
empty data: /tv nests keywords and alternative_titles under
"results" (not "keywords"/"titles" like /movie), and series-level
creators come from a separate created_by array, not the credits
cast/crew lists.
- ingest_series upserts the Series row plus season *stubs* (number,
title, air date, poster -- everything TMDb inlines on /tv/{id});
ingest_season fetches and upserts one season's full episode list,
and requires the series to already exist (TMDb has no single
endpoint for a whole series' episodes, so this has to be a separate
per-season call).
- Wired through the same job-lifecycle path as movie/person ingestion
(runner.py, arq tasks, admin endpoints), with the shared boilerplate
between them factored into runner._run_ingestion and
admin.py:_enqueue rather than copy-pasted a third and fourth time.
- Unit tests for both normalizers (runnable without a DB) and
integration tests for the full ingest_series -> ingest_season flow,
including the series-must-exist-first error and idempotency on
re-ingestion.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
669034896a
commit
3f6332559a
@@ -1,19 +1,21 @@
|
||||
"""Orchestrates one ingestion job end-to-end: create/track the `IngestionJob`
|
||||
row, run the provider fetch + normalize + upsert pipeline, and record
|
||||
success/failure. Used by both the admin trigger endpoint (section 17) and
|
||||
the arq worker task (section 15) so the two entry points share identical
|
||||
the arq worker task (section 15) so every entry point shares identical
|
||||
job-lifecycle and idempotency behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import session_scope
|
||||
from app.core.logging import get_logger
|
||||
from app.models.provider import IngestionJob
|
||||
from app.providers.base import MetadataProvider
|
||||
from app.providers.tmdb import TMDbProvider
|
||||
from app.repositories.provider_repository import (
|
||||
create_ingestion_job,
|
||||
@@ -25,6 +27,7 @@ from app.repositories.provider_repository import (
|
||||
)
|
||||
from app.services.ingestion.movie_ingestion import ingest_movie
|
||||
from app.services.ingestion.person_ingestion import ingest_person
|
||||
from app.services.ingestion.series_ingestion import ingest_season, ingest_series
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -64,9 +67,14 @@ async def get_or_create_ingestion_job(job_type: str, entity_type: str, external_
|
||||
return job.id
|
||||
|
||||
|
||||
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
"""Run the pipeline for an `IngestionJob` row that already exists (see
|
||||
`get_or_create_ingestion_job`) — this never creates the row itself."""
|
||||
async def _run_ingestion(
|
||||
job_id: uuid.UUID,
|
||||
label: str,
|
||||
work: Callable[[AsyncSession, MetadataProvider], Awaitable[uuid.UUID]],
|
||||
) -> uuid.UUID:
|
||||
"""Shared job-lifecycle wrapper: mark running, run `work` in its own
|
||||
session/transaction, mark succeeded/failed. `work` does the actual
|
||||
fetch+normalize+upsert and returns the resulting Aurora entity id."""
|
||||
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
@@ -75,13 +83,13 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
provider = TMDbProvider()
|
||||
try:
|
||||
async with session_scope() as session:
|
||||
movie_id = await ingest_movie(session, provider, external_id, ingestion_job_id=job_id)
|
||||
result_id = await work(session, provider)
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_succeeded(session, job, movie_id)
|
||||
return movie_id
|
||||
await mark_job_succeeded(session, job, result_id)
|
||||
return result_id
|
||||
except Exception as exc:
|
||||
logger.error("movie_ingestion_failed", external_id=external_id, error=str(exc))
|
||||
logger.error(f"{label}_failed", job_id=str(job_id), error=str(exc))
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_failed(session, job, str(exc))
|
||||
@@ -90,27 +98,43 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
await provider.aclose()
|
||||
|
||||
|
||||
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"movie_ingestion",
|
||||
lambda session, provider: ingest_movie(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def run_person_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
"""Run the pipeline for an `IngestionJob` row that already exists (see
|
||||
`get_or_create_ingestion_job`) — this never creates the row itself."""
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"person_ingestion",
|
||||
lambda session, provider: ingest_person(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_running(session, job)
|
||||
|
||||
provider = TMDbProvider()
|
||||
try:
|
||||
async with session_scope() as session:
|
||||
person_id = await ingest_person(session, provider, external_id, ingestion_job_id=job_id)
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_succeeded(session, job, person_id)
|
||||
return person_id
|
||||
except Exception as exc:
|
||||
logger.error("person_ingestion_failed", external_id=external_id, error=str(exc))
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_failed(session, job, str(exc))
|
||||
raise
|
||||
finally:
|
||||
await provider.aclose()
|
||||
async def run_series_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"series_ingestion",
|
||||
lambda session, provider: ingest_series(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def run_season_ingestion(
|
||||
job_id: uuid.UUID, series_external_id: str, season_number: int
|
||||
) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"season_ingestion",
|
||||
lambda session, provider: ingest_season(
|
||||
session, provider, series_external_id, season_number, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user