Add series/season/episode ingestion pipeline
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Movie and person ingestion existed, but the series/season/episode API
endpoints had no way to actually populate them from TMDb -- the
biggest remaining gap in Core 0.1's television scope.
- normalize_series/normalize_season in the metadata normalizer,
including two TMDb quirks that would otherwise silently produce
empty data: /tv nests keywords and alternative_titles under
"results" (not "keywords"/"titles" like /movie), and series-level
creators come from a separate created_by array, not the credits
cast/crew lists.
- ingest_series upserts the Series row plus season *stubs* (number,
title, air date, poster -- everything TMDb inlines on /tv/{id});
ingest_season fetches and upserts one season's full episode list,
and requires the series to already exist (TMDb has no single
endpoint for a whole series' episodes, so this has to be a separate
per-season call).
- Wired through the same job-lifecycle path as movie/person ingestion
(runner.py, arq tasks, admin endpoints), with the shared boilerplate
between them factored into runner._run_ingestion and
admin.py:_enqueue rather than copy-pasted a third and fourth time.
- Unit tests for both normalizers (runnable without a DB) and
integration tests for the full ingest_series -> ingest_season flow,
including the series-must-exist-first error and idempotency on
re-ingestion.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
NixReaper
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
669034896a
commit
3f6332559a
@@ -1,19 +1,21 @@
|
||||
"""Orchestrates one ingestion job end-to-end: create/track the `IngestionJob`
|
||||
row, run the provider fetch + normalize + upsert pipeline, and record
|
||||
success/failure. Used by both the admin trigger endpoint (section 17) and
|
||||
the arq worker task (section 15) so the two entry points share identical
|
||||
the arq worker task (section 15) so every entry point shares identical
|
||||
job-lifecycle and idempotency behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import session_scope
|
||||
from app.core.logging import get_logger
|
||||
from app.models.provider import IngestionJob
|
||||
from app.providers.base import MetadataProvider
|
||||
from app.providers.tmdb import TMDbProvider
|
||||
from app.repositories.provider_repository import (
|
||||
create_ingestion_job,
|
||||
@@ -25,6 +27,7 @@ from app.repositories.provider_repository import (
|
||||
)
|
||||
from app.services.ingestion.movie_ingestion import ingest_movie
|
||||
from app.services.ingestion.person_ingestion import ingest_person
|
||||
from app.services.ingestion.series_ingestion import ingest_season, ingest_series
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -64,9 +67,14 @@ async def get_or_create_ingestion_job(job_type: str, entity_type: str, external_
|
||||
return job.id
|
||||
|
||||
|
||||
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
"""Run the pipeline for an `IngestionJob` row that already exists (see
|
||||
`get_or_create_ingestion_job`) — this never creates the row itself."""
|
||||
async def _run_ingestion(
|
||||
job_id: uuid.UUID,
|
||||
label: str,
|
||||
work: Callable[[AsyncSession, MetadataProvider], Awaitable[uuid.UUID]],
|
||||
) -> uuid.UUID:
|
||||
"""Shared job-lifecycle wrapper: mark running, run `work` in its own
|
||||
session/transaction, mark succeeded/failed. `work` does the actual
|
||||
fetch+normalize+upsert and returns the resulting Aurora entity id."""
|
||||
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
@@ -75,13 +83,13 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
provider = TMDbProvider()
|
||||
try:
|
||||
async with session_scope() as session:
|
||||
movie_id = await ingest_movie(session, provider, external_id, ingestion_job_id=job_id)
|
||||
result_id = await work(session, provider)
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_succeeded(session, job, movie_id)
|
||||
return movie_id
|
||||
await mark_job_succeeded(session, job, result_id)
|
||||
return result_id
|
||||
except Exception as exc:
|
||||
logger.error("movie_ingestion_failed", external_id=external_id, error=str(exc))
|
||||
logger.error(f"{label}_failed", job_id=str(job_id), error=str(exc))
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_failed(session, job, str(exc))
|
||||
@@ -90,27 +98,43 @@ async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
await provider.aclose()
|
||||
|
||||
|
||||
async def run_movie_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"movie_ingestion",
|
||||
lambda session, provider: ingest_movie(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def run_person_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
"""Run the pipeline for an `IngestionJob` row that already exists (see
|
||||
`get_or_create_ingestion_job`) — this never creates the row itself."""
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"person_ingestion",
|
||||
lambda session, provider: ingest_person(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_running(session, job)
|
||||
|
||||
provider = TMDbProvider()
|
||||
try:
|
||||
async with session_scope() as session:
|
||||
person_id = await ingest_person(session, provider, external_id, ingestion_job_id=job_id)
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_succeeded(session, job, person_id)
|
||||
return person_id
|
||||
except Exception as exc:
|
||||
logger.error("person_ingestion_failed", external_id=external_id, error=str(exc))
|
||||
async with session_scope() as session:
|
||||
job = await _require_job(session, job_id)
|
||||
await mark_job_failed(session, job, str(exc))
|
||||
raise
|
||||
finally:
|
||||
await provider.aclose()
|
||||
async def run_series_ingestion(job_id: uuid.UUID, external_id: str) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"series_ingestion",
|
||||
lambda session, provider: ingest_series(
|
||||
session, provider, external_id, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def run_season_ingestion(
|
||||
job_id: uuid.UUID, series_external_id: str, season_number: int
|
||||
) -> uuid.UUID:
|
||||
return await _run_ingestion(
|
||||
job_id,
|
||||
"season_ingestion",
|
||||
lambda session, provider: ingest_season(
|
||||
session, provider, series_external_id, season_number, ingestion_job_id=job_id
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
"""Series/season/episode ingestion pipeline (section 15).
|
||||
|
||||
Mirrors `movie_ingestion.py`'s flow (fetch -> normalize -> match -> upsert
|
||||
-> provenance) but split into two entry points because TMDb requires a
|
||||
separate API call per season for episode data:
|
||||
|
||||
- `ingest_series` upserts the Series row and season *stubs* (number, title,
|
||||
air date, poster — everything TMDb includes inline on `/tv/{id}`), but
|
||||
not episodes.
|
||||
- `ingest_season` fetches and upserts one season's full episode list. It
|
||||
requires the series to have been ingested already (episodes attach to an
|
||||
Aurora series_id that has to already exist).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.models.artwork import Artwork
|
||||
from app.models.company import SeriesCompany
|
||||
from app.models.credit import Credit
|
||||
from app.models.enums import ArtworkType, CompanyType, MatchConfidence, SeriesStatus, VideoType
|
||||
from app.models.genre import SeriesGenre
|
||||
from app.models.keyword import SeriesKeyword
|
||||
from app.models.tv import Series
|
||||
from app.models.video import Video
|
||||
from app.providers.base import MetadataProvider, ProviderNotFoundError
|
||||
from app.repositories import reference_repository as refs
|
||||
from app.repositories import tv_repository
|
||||
from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record
|
||||
from app.services.metadata.matcher import match_entity
|
||||
from app.services.metadata.normalizer import (
|
||||
NormalizedSeries,
|
||||
normalize_season,
|
||||
normalize_series,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_ARTWORK_TYPES = {t.value for t in ArtworkType}
|
||||
_VIDEO_TYPES = {t.value for t in VideoType}
|
||||
|
||||
_LEADING_ARTICLES = ("the ", "a ", "an ")
|
||||
|
||||
|
||||
def _sort_title(title: str) -> str:
|
||||
lowered = title.lower()
|
||||
for article in _LEADING_ARTICLES:
|
||||
if lowered.startswith(article):
|
||||
return title[len(article) :].strip()
|
||||
return title
|
||||
|
||||
|
||||
async def _replace_series_genres(session: AsyncSession, series_id: uuid.UUID, names: list[str]) -> None:
|
||||
await session.execute(delete(SeriesGenre).where(SeriesGenre.series_id == series_id))
|
||||
for name in names:
|
||||
genre = await refs.get_or_create_genre(session, name)
|
||||
session.add(SeriesGenre(series_id=series_id, genre_id=genre.id))
|
||||
|
||||
|
||||
async def _replace_series_keywords(
|
||||
session: AsyncSession, series_id: uuid.UUID, names: list[str]
|
||||
) -> None:
|
||||
await session.execute(delete(SeriesKeyword).where(SeriesKeyword.series_id == series_id))
|
||||
for name in names:
|
||||
keyword = await refs.get_or_create_keyword(session, name)
|
||||
session.add(SeriesKeyword(series_id=series_id, keyword_id=keyword.id))
|
||||
|
||||
|
||||
async def _replace_series_companies(
|
||||
session: AsyncSession,
|
||||
series_id: uuid.UUID,
|
||||
provider_id: uuid.UUID,
|
||||
networks,
|
||||
companies,
|
||||
) -> None:
|
||||
await session.execute(delete(SeriesCompany).where(SeriesCompany.series_id == series_id))
|
||||
for ref, company_type, role in (
|
||||
*[(n, CompanyType.NETWORK, "network") for n in networks],
|
||||
*[(c, CompanyType.PRODUCTION_COMPANY, "production") for c in companies],
|
||||
):
|
||||
country_code = None
|
||||
if ref.country_code:
|
||||
country = await refs.get_or_create_country(session, ref.country_code, ref.country_code)
|
||||
country_code = country.code
|
||||
company = await refs.get_or_create_company(
|
||||
session,
|
||||
provider_id=provider_id,
|
||||
external_id=ref.external_id,
|
||||
name=ref.name,
|
||||
company_type=company_type,
|
||||
country_code=country_code,
|
||||
)
|
||||
session.add(SeriesCompany(series_id=series_id, company_id=company.id, role=role))
|
||||
|
||||
|
||||
async def _replace_series_credits(
|
||||
session: AsyncSession,
|
||||
series_id: uuid.UUID,
|
||||
provider_id: uuid.UUID,
|
||||
cast,
|
||||
crew,
|
||||
creators,
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Credit).where(Credit.entity_type == "series", Credit.entity_id == series_id)
|
||||
)
|
||||
for order, ref in enumerate(cast):
|
||||
person = await refs.get_or_create_person(
|
||||
session, provider_id=provider_id, external_id=ref.external_id, name=ref.name, gender=ref.gender
|
||||
)
|
||||
session.add(
|
||||
Credit(
|
||||
person_id=person.id,
|
||||
entity_type="series",
|
||||
entity_id=series_id,
|
||||
department="Acting",
|
||||
character=ref.character,
|
||||
billing_order=ref.billing_order if ref.billing_order is not None else order,
|
||||
provider_id=provider_id,
|
||||
)
|
||||
)
|
||||
for ref in (*crew, *creators):
|
||||
person = await refs.get_or_create_person(
|
||||
session, provider_id=provider_id, external_id=ref.external_id, name=ref.name, gender=ref.gender
|
||||
)
|
||||
session.add(
|
||||
Credit(
|
||||
person_id=person.id,
|
||||
entity_type="series",
|
||||
entity_id=series_id,
|
||||
department=ref.department or "Crew",
|
||||
job=ref.job,
|
||||
provider_id=provider_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_artwork(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, artwork_list
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Artwork).where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id)
|
||||
)
|
||||
for art in artwork_list:
|
||||
art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster"
|
||||
session.add(
|
||||
Artwork(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
artwork_type=ArtworkType(art_type),
|
||||
width=art.width,
|
||||
height=art.height,
|
||||
aspect_ratio=art.aspect_ratio,
|
||||
provider_id=provider_id,
|
||||
provider_asset_id=art.provider_asset_id,
|
||||
original_url=art.url,
|
||||
vote_average=art.vote_average,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _replace_entity_videos(
|
||||
session: AsyncSession, entity_type: str, entity_id: uuid.UUID, provider_id: uuid.UUID, videos
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Video).where(Video.entity_type == entity_type, Video.entity_id == entity_id)
|
||||
)
|
||||
for v in videos:
|
||||
video_type = v.video_type if v.video_type in _VIDEO_TYPES else "other"
|
||||
session.add(
|
||||
Video(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
video_type=VideoType(video_type),
|
||||
provider_id=provider_id,
|
||||
provider_video_id=v.provider_video_id,
|
||||
site=v.site,
|
||||
url=v.url,
|
||||
is_official=v.is_official,
|
||||
published_at=v.published_at,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def ingest_series(
|
||||
session: AsyncSession,
|
||||
provider: MetadataProvider,
|
||||
external_id: str,
|
||||
*,
|
||||
ingestion_job_id: uuid.UUID | None = None,
|
||||
) -> uuid.UUID:
|
||||
"""Fetch, normalize, and upsert a series plus its season *stubs*
|
||||
(without episodes — see `ingest_season`). Returns the Aurora series
|
||||
UUID (new or existing)."""
|
||||
|
||||
tmdb_provider = await get_or_create_provider(session, provider.slug, provider.name)
|
||||
|
||||
aurora_id_existing, confidence = await match_entity(
|
||||
session, provider_id=tmdb_provider.id, entity_type="series", external_id=external_id
|
||||
)
|
||||
|
||||
raw = await provider.get_series(external_id)
|
||||
normalized: NormalizedSeries = normalize_series(raw)
|
||||
|
||||
existing_series: Series | None = None
|
||||
if aurora_id_existing is not None:
|
||||
existing_series = await session.get(Series, aurora_id_existing)
|
||||
if existing_series is None:
|
||||
aurora_id_existing = None
|
||||
|
||||
series: Series
|
||||
if existing_series is not None:
|
||||
series = existing_series
|
||||
else:
|
||||
series = Series(title=normalized.title, sort_title=_sort_title(normalized.title))
|
||||
session.add(series)
|
||||
await session.flush()
|
||||
|
||||
series.title = normalized.title
|
||||
series.original_title = normalized.original_title
|
||||
series.sort_title = _sort_title(normalized.title)
|
||||
series.overview = normalized.overview
|
||||
series.tagline = normalized.tagline
|
||||
series.status = SeriesStatus(normalized.status)
|
||||
series.first_air_date = normalized.first_air_date
|
||||
series.last_air_date = normalized.last_air_date
|
||||
series.number_of_seasons = normalized.number_of_seasons
|
||||
series.number_of_episodes = normalized.number_of_episodes
|
||||
series.popularity = normalized.popularity
|
||||
series.vote_average = normalized.vote_average
|
||||
series.vote_count = normalized.vote_count
|
||||
series.source_provider_id = tmdb_provider.id
|
||||
|
||||
if normalized.original_language:
|
||||
language = await refs.get_or_create_language(
|
||||
session, normalized.original_language, normalized.original_language.upper()
|
||||
)
|
||||
series.original_language_code = language.code
|
||||
|
||||
await session.flush()
|
||||
|
||||
await _replace_series_genres(session, series.id, normalized.genres)
|
||||
await _replace_series_keywords(session, series.id, normalized.keywords)
|
||||
await _replace_series_companies(
|
||||
session, series.id, tmdb_provider.id, normalized.networks, normalized.companies
|
||||
)
|
||||
await _replace_series_credits(
|
||||
session, series.id, tmdb_provider.id, normalized.cast, normalized.crew, normalized.creators
|
||||
)
|
||||
await _replace_entity_artwork(session, "series", series.id, tmdb_provider.id, normalized.artwork)
|
||||
await _replace_entity_videos(session, "series", series.id, tmdb_provider.id, normalized.videos)
|
||||
|
||||
for stub in normalized.seasons:
|
||||
season = await tv_repository.get_or_create_season(session, series.id, stub.season_number)
|
||||
season.title = stub.title
|
||||
season.overview = stub.overview
|
||||
season.air_date = stub.air_date
|
||||
season.episode_count = stub.episode_count
|
||||
await session.flush()
|
||||
await _replace_entity_artwork(
|
||||
session,
|
||||
"season",
|
||||
season.id,
|
||||
tmdb_provider.id,
|
||||
[stub.poster] if stub.poster else [],
|
||||
)
|
||||
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
entity_type="series",
|
||||
entity_id=series.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
external_id=external_id,
|
||||
)
|
||||
for slug, ext_id in normalized.external_ids.items():
|
||||
ext_provider = await get_or_create_provider(session, slug, slug.upper())
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
entity_type="series",
|
||||
entity_id=series.id,
|
||||
provider_id=ext_provider.id,
|
||||
external_id=ext_id,
|
||||
)
|
||||
|
||||
await upsert_provider_record(
|
||||
session,
|
||||
provider_id=tmdb_provider.id,
|
||||
entity_type="series",
|
||||
external_id=external_id,
|
||||
raw_data=raw,
|
||||
aurora_entity_id=series.id,
|
||||
match_confidence=confidence if aurora_id_existing else MatchConfidence.NEW,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
)
|
||||
|
||||
logger.info("series_ingested", series_id=str(series.id), external_id=external_id)
|
||||
return series.id
|
||||
|
||||
|
||||
async def _replace_episode_credits(
|
||||
session: AsyncSession, episode_id: uuid.UUID, provider_id: uuid.UUID, guest_stars, crew
|
||||
) -> None:
|
||||
await session.execute(
|
||||
delete(Credit).where(Credit.entity_type == "episode", Credit.entity_id == episode_id)
|
||||
)
|
||||
for ref in guest_stars:
|
||||
person = await refs.get_or_create_person(
|
||||
session, provider_id=provider_id, external_id=ref.external_id, name=ref.name, gender=ref.gender
|
||||
)
|
||||
session.add(
|
||||
Credit(
|
||||
person_id=person.id,
|
||||
entity_type="episode",
|
||||
entity_id=episode_id,
|
||||
department="Acting",
|
||||
character=ref.character,
|
||||
billing_order=ref.billing_order,
|
||||
is_guest=True,
|
||||
provider_id=provider_id,
|
||||
)
|
||||
)
|
||||
for ref in crew:
|
||||
person = await refs.get_or_create_person(
|
||||
session, provider_id=provider_id, external_id=ref.external_id, name=ref.name, gender=ref.gender
|
||||
)
|
||||
session.add(
|
||||
Credit(
|
||||
person_id=person.id,
|
||||
entity_type="episode",
|
||||
entity_id=episode_id,
|
||||
department=ref.department or "Crew",
|
||||
job=ref.job,
|
||||
provider_id=provider_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def ingest_season(
|
||||
session: AsyncSession,
|
||||
provider: MetadataProvider,
|
||||
series_external_id: str,
|
||||
season_number: int,
|
||||
*,
|
||||
ingestion_job_id: uuid.UUID | None = None,
|
||||
) -> uuid.UUID:
|
||||
"""Fetch, normalize, and upsert one season's full episode list. The
|
||||
series must already be ingested (`ingest_series`) — episodes attach to
|
||||
an Aurora series_id that has to exist first."""
|
||||
|
||||
tmdb_provider = await get_or_create_provider(session, provider.slug, provider.name)
|
||||
|
||||
series_id = await refs.find_entity_by_external_id(
|
||||
session, tmdb_provider.id, "series", series_external_id
|
||||
)
|
||||
if series_id is None:
|
||||
raise ProviderNotFoundError(
|
||||
f"Cannot ingest season {season_number}: series tmdb:{series_external_id} has not "
|
||||
"been ingested yet. Call POST /admin/ingest/series first."
|
||||
)
|
||||
|
||||
raw = await provider.get_season(series_external_id, season_number)
|
||||
normalized = normalize_season(raw)
|
||||
|
||||
season = await tv_repository.get_or_create_season(session, series_id, season_number)
|
||||
season.title = normalized.title
|
||||
season.overview = normalized.overview
|
||||
season.air_date = normalized.air_date
|
||||
season.episode_count = normalized.episode_count
|
||||
await session.flush()
|
||||
|
||||
await _replace_entity_artwork(session, "season", season.id, tmdb_provider.id, normalized.artwork)
|
||||
|
||||
for ep in normalized.episodes:
|
||||
episode = await tv_repository.get_or_create_episode(
|
||||
session,
|
||||
series_id=series_id,
|
||||
season_id=season.id,
|
||||
season_number=season_number,
|
||||
episode_number=ep.episode_number,
|
||||
)
|
||||
episode.title = ep.title
|
||||
episode.overview = ep.overview
|
||||
episode.air_date = ep.air_date
|
||||
episode.runtime_minutes = ep.runtime_minutes
|
||||
await session.flush()
|
||||
|
||||
await _replace_episode_credits(session, episode.id, tmdb_provider.id, ep.guest_stars, ep.crew)
|
||||
await _replace_entity_artwork(session, "episode", episode.id, tmdb_provider.id, ep.stills)
|
||||
await refs.link_external_id(
|
||||
session,
|
||||
entity_type="episode",
|
||||
entity_id=episode.id,
|
||||
provider_id=tmdb_provider.id,
|
||||
external_id=ep.external_id,
|
||||
)
|
||||
|
||||
await upsert_provider_record(
|
||||
session,
|
||||
provider_id=tmdb_provider.id,
|
||||
entity_type="season",
|
||||
external_id=f"{series_external_id}:{season_number}",
|
||||
raw_data=raw,
|
||||
aurora_entity_id=season.id,
|
||||
match_confidence=MatchConfidence.NEW,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"season_ingested",
|
||||
season_id=str(season.id),
|
||||
series_external_id=series_external_id,
|
||||
season_number=season_number,
|
||||
)
|
||||
return season.id
|
||||
@@ -339,3 +339,351 @@ def normalize_person(raw: dict) -> NormalizedPerson:
|
||||
artwork=artwork,
|
||||
external_ids=external_ids,
|
||||
)
|
||||
|
||||
|
||||
_SERIES_STATUS_MAP = {
|
||||
"Planned": "planned",
|
||||
"Pilot": "pilot",
|
||||
"In Production": "in_production",
|
||||
"Returning Series": "returning_series",
|
||||
"Ended": "ended",
|
||||
"Canceled": "canceled",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedSeasonStub:
|
||||
"""A season summary as it appears embedded in a series payload — enough
|
||||
to create/update the Season row, but without episodes (those require a
|
||||
separate per-season fetch — see `normalize_season`)."""
|
||||
|
||||
season_number: int
|
||||
title: str | None
|
||||
overview: str | None
|
||||
air_date: date | None
|
||||
episode_count: int | None
|
||||
poster: NormalizedArtwork | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedSeries:
|
||||
external_id: str
|
||||
title: str
|
||||
original_title: str | None
|
||||
overview: str | None
|
||||
tagline: str | None
|
||||
status: str
|
||||
original_language: str | None
|
||||
first_air_date: date | None
|
||||
last_air_date: date | None
|
||||
number_of_seasons: int | None
|
||||
number_of_episodes: int | None
|
||||
popularity: float | None
|
||||
vote_average: float | None
|
||||
vote_count: int | None
|
||||
genres: list[str] = field(default_factory=list)
|
||||
keywords: list[str] = field(default_factory=list)
|
||||
alternate_titles: list[str] = field(default_factory=list)
|
||||
networks: list[NormalizedCompanyRef] = field(default_factory=list)
|
||||
companies: list[NormalizedCompanyRef] = field(default_factory=list)
|
||||
creators: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
cast: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
crew: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
artwork: list[NormalizedArtwork] = field(default_factory=list)
|
||||
videos: list[NormalizedVideo] = field(default_factory=list)
|
||||
external_ids: dict[str, str] = field(default_factory=dict)
|
||||
seasons: list[NormalizedSeasonStub] = field(default_factory=list)
|
||||
|
||||
|
||||
def normalize_series(raw: dict) -> NormalizedSeries:
|
||||
credits_ = raw.get("credits") or {}
|
||||
images = raw.get("images") or {}
|
||||
videos = (raw.get("videos") or {}).get("results") or []
|
||||
# Unlike /movie, TMDb's /tv keywords and alternative_titles nest under
|
||||
# "results", not "keywords"/"titles".
|
||||
keywords = (raw.get("keywords") or {}).get("results") or []
|
||||
alt_titles = (raw.get("alternative_titles") or {}).get("results") or []
|
||||
ext_ids = raw.get("external_ids") or {}
|
||||
|
||||
external_ids: dict[str, str] = {}
|
||||
if ext_ids.get("imdb_id"):
|
||||
external_ids["imdb"] = ext_ids["imdb_id"]
|
||||
if ext_ids.get("tvdb_id"):
|
||||
external_ids["tvdb"] = str(ext_ids["tvdb_id"])
|
||||
if ext_ids.get("wikidata_id"):
|
||||
external_ids["wikidata"] = ext_ids["wikidata_id"]
|
||||
|
||||
cast = [
|
||||
NormalizedPersonRef(
|
||||
external_id=str(c["id"]),
|
||||
name=c["name"],
|
||||
profile_path=_image_url(c.get("profile_path")),
|
||||
department="Acting",
|
||||
character=c.get("character"),
|
||||
billing_order=c.get("order"),
|
||||
gender=_gender_label(c.get("gender")),
|
||||
)
|
||||
for c in credits_.get("cast", [])
|
||||
]
|
||||
crew = [
|
||||
NormalizedPersonRef(
|
||||
external_id=str(c["id"]),
|
||||
name=c["name"],
|
||||
profile_path=_image_url(c.get("profile_path")),
|
||||
department=c.get("department") or "Crew",
|
||||
job=c.get("job"),
|
||||
gender=_gender_label(c.get("gender")),
|
||||
)
|
||||
for c in credits_.get("crew", [])
|
||||
]
|
||||
creators = [
|
||||
NormalizedPersonRef(
|
||||
external_id=str(c["id"]),
|
||||
name=c["name"],
|
||||
profile_path=_image_url(c.get("profile_path")),
|
||||
department="Creator",
|
||||
gender=_gender_label(c.get("gender")),
|
||||
)
|
||||
for c in raw.get("created_by", [])
|
||||
]
|
||||
|
||||
networks = [
|
||||
NormalizedCompanyRef(
|
||||
external_id=str(n["id"]), name=n["name"], country_code=n.get("origin_country") or None
|
||||
)
|
||||
for n in raw.get("networks", [])
|
||||
]
|
||||
companies = [
|
||||
NormalizedCompanyRef(
|
||||
external_id=str(pc["id"]),
|
||||
name=pc["name"],
|
||||
country_code=pc.get("origin_country") or None,
|
||||
)
|
||||
for pc in raw.get("production_companies", [])
|
||||
]
|
||||
|
||||
artwork: list[NormalizedArtwork] = []
|
||||
if raw.get("poster_path"):
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
"poster",
|
||||
_image_url_required(raw["poster_path"]),
|
||||
raw["poster_path"],
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
)
|
||||
if raw.get("backdrop_path"):
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
"backdrop",
|
||||
_image_url_required(raw["backdrop_path"]),
|
||||
raw["backdrop_path"],
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
)
|
||||
for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")):
|
||||
for img in images.get(tmdb_key, []):
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
kind,
|
||||
_image_url_required(img["file_path"]),
|
||||
img["file_path"],
|
||||
img.get("width"),
|
||||
img.get("height"),
|
||||
img.get("aspect_ratio"),
|
||||
img.get("iso_639_1"),
|
||||
img.get("vote_average"),
|
||||
)
|
||||
)
|
||||
|
||||
normalized_videos = [
|
||||
NormalizedVideo(
|
||||
video_type=(v.get("type") or "other").lower().replace(" ", "_"),
|
||||
provider_video_id=v["id"],
|
||||
site=v.get("site"),
|
||||
url=_video_url(v),
|
||||
language=v.get("iso_639_1"),
|
||||
country=v.get("iso_3166_1"),
|
||||
is_official=bool(v.get("official")),
|
||||
published_at=_parse_date(v.get("published_at", "")[:10] if v.get("published_at") else None),
|
||||
)
|
||||
for v in videos
|
||||
]
|
||||
|
||||
seasons = [
|
||||
NormalizedSeasonStub(
|
||||
season_number=s["season_number"],
|
||||
title=s.get("name"),
|
||||
overview=s.get("overview") or None,
|
||||
air_date=_parse_date(s.get("air_date")),
|
||||
episode_count=s.get("episode_count"),
|
||||
poster=(
|
||||
NormalizedArtwork(
|
||||
"poster", _image_url_required(s["poster_path"]), s["poster_path"],
|
||||
None, None, None, None,
|
||||
)
|
||||
if s.get("poster_path")
|
||||
else None
|
||||
),
|
||||
)
|
||||
for s in raw.get("seasons", [])
|
||||
# TMDb includes a "season 0" for specials; keep it (it's a real,
|
||||
# commonly-referenced season) but drop entries missing a number.
|
||||
if s.get("season_number") is not None
|
||||
]
|
||||
|
||||
return NormalizedSeries(
|
||||
external_id=str(raw["id"]),
|
||||
title=raw.get("name") or raw.get("original_name") or "",
|
||||
original_title=raw.get("original_name"),
|
||||
overview=raw.get("overview") or None,
|
||||
tagline=raw.get("tagline") or None,
|
||||
status=_SERIES_STATUS_MAP.get(raw.get("status", "Returning Series"), "returning_series"),
|
||||
original_language=raw.get("original_language"),
|
||||
first_air_date=_parse_date(raw.get("first_air_date")),
|
||||
last_air_date=_parse_date(raw.get("last_air_date")),
|
||||
number_of_seasons=raw.get("number_of_seasons"),
|
||||
number_of_episodes=raw.get("number_of_episodes"),
|
||||
popularity=raw.get("popularity"),
|
||||
vote_average=raw.get("vote_average"),
|
||||
vote_count=raw.get("vote_count"),
|
||||
genres=[g["name"] for g in raw.get("genres", [])],
|
||||
keywords=[k["name"] for k in keywords],
|
||||
alternate_titles=[t["title"] for t in alt_titles if t.get("title")],
|
||||
networks=networks,
|
||||
companies=companies,
|
||||
creators=creators,
|
||||
cast=cast,
|
||||
crew=crew,
|
||||
artwork=artwork,
|
||||
videos=normalized_videos,
|
||||
external_ids=external_ids,
|
||||
seasons=seasons,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedEpisode:
|
||||
external_id: str
|
||||
episode_number: int
|
||||
title: str | None
|
||||
overview: str | None
|
||||
air_date: date | None
|
||||
runtime_minutes: int | None
|
||||
cast: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
guest_stars: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
crew: list[NormalizedPersonRef] = field(default_factory=list)
|
||||
stills: list[NormalizedArtwork] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedSeason:
|
||||
season_number: int
|
||||
title: str | None
|
||||
overview: str | None
|
||||
air_date: date | None
|
||||
episode_count: int | None
|
||||
artwork: list[NormalizedArtwork] = field(default_factory=list)
|
||||
episodes: list[NormalizedEpisode] = field(default_factory=list)
|
||||
external_ids: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _episode_person_refs(entries: list[dict]) -> list[NormalizedPersonRef]:
|
||||
return [
|
||||
NormalizedPersonRef(
|
||||
external_id=str(e["id"]),
|
||||
name=e["name"],
|
||||
profile_path=_image_url(e.get("profile_path")),
|
||||
department="Acting",
|
||||
character=e.get("character"),
|
||||
billing_order=e.get("order"),
|
||||
gender=_gender_label(e.get("gender")),
|
||||
)
|
||||
for e in entries
|
||||
]
|
||||
|
||||
|
||||
def normalize_season(raw: dict) -> NormalizedSeason:
|
||||
images = raw.get("images") or {}
|
||||
ext_ids = raw.get("external_ids") or {}
|
||||
external_ids: dict[str, str] = {}
|
||||
if ext_ids.get("tvdb_id"):
|
||||
external_ids["tvdb"] = str(ext_ids["tvdb_id"])
|
||||
|
||||
artwork: list[NormalizedArtwork] = []
|
||||
if raw.get("poster_path"):
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
"poster", _image_url_required(raw["poster_path"]), raw["poster_path"],
|
||||
None, None, None, None,
|
||||
)
|
||||
)
|
||||
for img in images.get("posters", []):
|
||||
artwork.append(
|
||||
NormalizedArtwork(
|
||||
"poster",
|
||||
_image_url_required(img["file_path"]),
|
||||
img["file_path"],
|
||||
img.get("width"),
|
||||
img.get("height"),
|
||||
img.get("aspect_ratio"),
|
||||
img.get("iso_639_1"),
|
||||
img.get("vote_average"),
|
||||
)
|
||||
)
|
||||
|
||||
episodes = []
|
||||
for ep in raw.get("episodes", []):
|
||||
crew = [
|
||||
NormalizedPersonRef(
|
||||
external_id=str(c["id"]),
|
||||
name=c["name"],
|
||||
profile_path=_image_url(c.get("profile_path")),
|
||||
department=c.get("department") or "Crew",
|
||||
job=c.get("job"),
|
||||
gender=_gender_label(c.get("gender")),
|
||||
)
|
||||
for c in ep.get("crew", [])
|
||||
]
|
||||
stills: list[NormalizedArtwork] = []
|
||||
if ep.get("still_path"):
|
||||
stills.append(
|
||||
NormalizedArtwork(
|
||||
"still", _image_url_required(ep["still_path"]), ep["still_path"],
|
||||
None, None, None, None,
|
||||
)
|
||||
)
|
||||
episodes.append(
|
||||
NormalizedEpisode(
|
||||
external_id=str(ep["id"]),
|
||||
episode_number=ep["episode_number"],
|
||||
title=ep.get("name"),
|
||||
overview=ep.get("overview") or None,
|
||||
air_date=_parse_date(ep.get("air_date")),
|
||||
runtime_minutes=ep.get("runtime"),
|
||||
# TMDb's per-episode payload has no separate "main cast"
|
||||
# list distinct from guest_stars/crew — the series' regular
|
||||
# cast is assumed, not repeated per episode.
|
||||
cast=[],
|
||||
guest_stars=_episode_person_refs(ep.get("guest_stars", [])),
|
||||
crew=crew,
|
||||
stills=stills,
|
||||
)
|
||||
)
|
||||
|
||||
return NormalizedSeason(
|
||||
season_number=raw["season_number"],
|
||||
title=raw.get("name"),
|
||||
overview=raw.get("overview") or None,
|
||||
air_date=_parse_date(raw.get("air_date")),
|
||||
episode_count=len(episodes) or None,
|
||||
artwork=artwork,
|
||||
episodes=episodes,
|
||||
external_ids=external_ids,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user