Add series/season/episode ingestion pipeline
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Movie and person ingestion existed, but the series/season/episode API
endpoints had no way to actually populate them from TMDb -- the
biggest remaining gap in Core 0.1's television scope.
- normalize_series/normalize_season in the metadata normalizer,
including two TMDb quirks that would otherwise silently produce
empty data: /tv nests keywords and alternative_titles under
"results" (not "keywords"/"titles" like /movie), and series-level
creators come from a separate created_by array, not the credits
cast/crew lists.
- ingest_series upserts the Series row plus season *stubs* (number,
title, air date, poster -- everything TMDb inlines on /tv/{id});
ingest_season fetches and upserts one season's full episode list,
and requires the series to already exist (TMDb has no single
endpoint for a whole series' episodes, so this has to be a separate
per-season call).
- Wired through the same job-lifecycle path as movie/person ingestion
(runner.py, arq tasks, admin endpoints), with the shared boilerplate
between them factored into runner._run_ingestion and
admin.py:_enqueue rather than copy-pasted a third and fourth time.
- Unit tests for both normalizers (runnable without a DB) and
integration tests for the full ingest_series -> ingest_season flow,
including the series-must-exist-first error and idempotency on
re-ingestion.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
669034896a
commit
3f6332559a
+75
-16
@@ -34,6 +34,15 @@ class IngestPersonRequest(BaseModel):
|
||||
tmdb_id: str
|
||||
|
||||
|
||||
class IngestSeriesRequest(BaseModel):
|
||||
tmdb_id: str
|
||||
|
||||
|
||||
class IngestSeasonRequest(BaseModel):
|
||||
tmdb_id: str
|
||||
season_number: int
|
||||
|
||||
|
||||
class IngestionEnqueuedOut(BaseModel):
|
||||
job_id: str
|
||||
arq_job_id: str
|
||||
@@ -51,16 +60,21 @@ class IngestionJobOut(BaseModel):
|
||||
result_aurora_id: str | None
|
||||
|
||||
|
||||
@router.post("/ingest/movie", response_model=IngestionEnqueuedOut)
|
||||
async def enqueue_movie_ingestion(
|
||||
body: IngestMovieRequest, queue: ArqRedis = Depends(get_queue)
|
||||
async def _enqueue(
|
||||
queue: ArqRedis,
|
||||
*,
|
||||
job_type: str,
|
||||
entity_type: str,
|
||||
external_id: str,
|
||||
task_name: str,
|
||||
task_args: tuple = (),
|
||||
) -> IngestionEnqueuedOut:
|
||||
# Created synchronously (and idempotently — same tmdb_id reuses the same
|
||||
# row) so the response's job_id is immediately valid against
|
||||
# GET /ingest/jobs/{job_id}. arq's own job id is a separate, unrelated
|
||||
# identifier used only for queue bookkeeping.
|
||||
job_id = await get_or_create_ingestion_job("movie_sync", "movie", body.tmdb_id)
|
||||
arq_job = await queue.enqueue_job("ingest_movie_task", str(job_id), body.tmdb_id)
|
||||
# The IngestionJob row is created synchronously (and idempotently — the
|
||||
# same external_id reuses the same row) so the response's job_id is
|
||||
# immediately valid against GET /ingest/jobs/{job_id}. arq's own job id
|
||||
# is a separate, unrelated identifier used only for queue bookkeeping.
|
||||
job_id = await get_or_create_ingestion_job(job_type, entity_type, external_id)
|
||||
arq_job = await queue.enqueue_job(task_name, str(job_id), *task_args)
|
||||
if arq_job is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE, "Could not enqueue ingestion job (queue busy)."
|
||||
@@ -68,17 +82,62 @@ async def enqueue_movie_ingestion(
|
||||
return IngestionEnqueuedOut(job_id=str(job_id), arq_job_id=arq_job.job_id)
|
||||
|
||||
|
||||
@router.post("/ingest/movie", response_model=IngestionEnqueuedOut)
|
||||
async def enqueue_movie_ingestion(
|
||||
body: IngestMovieRequest, queue: ArqRedis = Depends(get_queue)
|
||||
) -> IngestionEnqueuedOut:
|
||||
return await _enqueue(
|
||||
queue,
|
||||
job_type="movie_sync",
|
||||
entity_type="movie",
|
||||
external_id=body.tmdb_id,
|
||||
task_name="ingest_movie_task",
|
||||
task_args=(body.tmdb_id,),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ingest/person", response_model=IngestionEnqueuedOut)
|
||||
async def enqueue_person_ingestion(
|
||||
body: IngestPersonRequest, queue: ArqRedis = Depends(get_queue)
|
||||
) -> IngestionEnqueuedOut:
|
||||
job_id = await get_or_create_ingestion_job("person_sync", "person", body.tmdb_id)
|
||||
arq_job = await queue.enqueue_job("ingest_person_task", str(job_id), body.tmdb_id)
|
||||
if arq_job is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE, "Could not enqueue ingestion job (queue busy)."
|
||||
)
|
||||
return IngestionEnqueuedOut(job_id=str(job_id), arq_job_id=arq_job.job_id)
|
||||
return await _enqueue(
|
||||
queue,
|
||||
job_type="person_sync",
|
||||
entity_type="person",
|
||||
external_id=body.tmdb_id,
|
||||
task_name="ingest_person_task",
|
||||
task_args=(body.tmdb_id,),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ingest/series", response_model=IngestionEnqueuedOut)
|
||||
async def enqueue_series_ingestion(
|
||||
body: IngestSeriesRequest, queue: ArqRedis = Depends(get_queue)
|
||||
) -> IngestionEnqueuedOut:
|
||||
return await _enqueue(
|
||||
queue,
|
||||
job_type="series_sync",
|
||||
entity_type="series",
|
||||
external_id=body.tmdb_id,
|
||||
task_name="ingest_series_task",
|
||||
task_args=(body.tmdb_id,),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ingest/season", response_model=IngestionEnqueuedOut)
|
||||
async def enqueue_season_ingestion(
|
||||
body: IngestSeasonRequest, queue: ArqRedis = Depends(get_queue)
|
||||
) -> IngestionEnqueuedOut:
|
||||
# Season ingestion requires the series to already be ingested — see
|
||||
# app/services/ingestion/series_ingestion.py:ingest_season.
|
||||
return await _enqueue(
|
||||
queue,
|
||||
job_type="season_sync",
|
||||
entity_type="season",
|
||||
external_id=f"{body.tmdb_id}:{body.season_number}",
|
||||
task_name="ingest_season_task",
|
||||
task_args=(body.tmdb_id, body.season_number),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ingest/jobs/{job_id}", response_model=IngestionJobOut)
|
||||
|
||||
Reference in New Issue
Block a user