Add series/season/episode ingestion pipeline
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Movie and person ingestion existed, but the series/season/episode API
endpoints had no way to actually populate them from TMDb -- the
biggest remaining gap in Core 0.1's television scope.
- normalize_series/normalize_season in the metadata normalizer,
including two TMDb quirks that would otherwise silently produce
empty data: /tv nests keywords and alternative_titles under
"results" (not "keywords"/"titles" like /movie), and series-level
creators come from a separate created_by array, not the credits
cast/crew lists.
- ingest_series upserts the Series row plus season *stubs* (number,
title, air date, poster -- everything TMDb inlines on /tv/{id});
ingest_season fetches and upserts one season's full episode list,
and requires the series to already exist (TMDb has no single
endpoint for a whole series' episodes, so this has to be a separate
per-season call).
- Wired through the same job-lifecycle path as movie/person ingestion
(runner.py, arq tasks, admin endpoints), with the shared boilerplate
between them factored into runner._run_ingestion and
admin.py:_enqueue rather than copy-pasted a third and fourth time.
- Unit tests for both normalizers (runnable without a DB) and
integration tests for the full ingest_series -> ingest_season flow,
including the series-must-exist-first error and idempotency on
re-ingestion.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
669034896a
commit
3f6332559a
@@ -6,7 +6,12 @@ from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from app.services.metadata.normalizer import normalize_movie, normalize_person
|
||||
from app.services.metadata.normalizer import (
|
||||
normalize_movie,
|
||||
normalize_person,
|
||||
normalize_season,
|
||||
normalize_series,
|
||||
)
|
||||
|
||||
FAKE_MOVIE = {
|
||||
"id": 603,
|
||||
@@ -182,3 +187,177 @@ def test_normalize_person_profile_artwork_deduplicated():
|
||||
# not be inserted twice.
|
||||
paths = [a.provider_asset_id for a in person.artwork]
|
||||
assert paths.count("/keanu.jpg") == 1
|
||||
|
||||
|
||||
FAKE_SERIES = {
|
||||
"id": 1396,
|
||||
"name": "Ozymandias",
|
||||
"original_name": "Ozymandias",
|
||||
"overview": "A chemistry teacher turns to crime.",
|
||||
"tagline": "Change the equation.",
|
||||
"status": "Ended",
|
||||
"original_language": "en",
|
||||
"first_air_date": "2008-01-20",
|
||||
"last_air_date": "2013-09-29",
|
||||
"number_of_seasons": 5,
|
||||
"number_of_episodes": 62,
|
||||
"popularity": 120.5,
|
||||
"vote_average": 8.9,
|
||||
"vote_count": 15000,
|
||||
"genres": [{"id": 18, "name": "Drama"}, {"id": 80, "name": "Crime"}],
|
||||
"created_by": [{"id": 66633, "name": "Vince Gilligan", "gender": 2}],
|
||||
"networks": [{"id": 174, "name": "AMC", "origin_country": "US"}],
|
||||
"production_companies": [
|
||||
{"id": 2605, "name": "Gran Via Productions", "origin_country": "US"}
|
||||
],
|
||||
"poster_path": "/poster.jpg",
|
||||
"backdrop_path": "/backdrop.jpg",
|
||||
"credits": {
|
||||
"cast": [
|
||||
{
|
||||
"id": 17419,
|
||||
"name": "Bryan Cranston",
|
||||
"character": "Walter White",
|
||||
"order": 0,
|
||||
"gender": 2,
|
||||
}
|
||||
],
|
||||
"crew": [],
|
||||
},
|
||||
"images": {"posters": [], "backdrops": [], "logos": []},
|
||||
"videos": {"results": []},
|
||||
"keywords": {"results": [{"id": 1, "name": "drug dealer"}]},
|
||||
"alternative_titles": {"results": [{"title": "Breaking Bad", "iso_3166_1": "US"}]},
|
||||
"external_ids": {"imdb_id": "tt0903747", "tvdb_id": 81189},
|
||||
"seasons": [
|
||||
{
|
||||
"season_number": 1,
|
||||
"name": "Season 1",
|
||||
"overview": "The first season.",
|
||||
"air_date": "2008-01-20",
|
||||
"episode_count": 7,
|
||||
"poster_path": "/season1.jpg",
|
||||
},
|
||||
{
|
||||
# TMDb includes a "specials" season with number 0 — must not be
|
||||
# dropped.
|
||||
"season_number": 0,
|
||||
"name": "Specials",
|
||||
"overview": None,
|
||||
"air_date": None,
|
||||
"episode_count": 3,
|
||||
"poster_path": None,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
FAKE_SEASON = {
|
||||
"season_number": 1,
|
||||
"name": "Season 1",
|
||||
"overview": "The first season.",
|
||||
"air_date": "2008-01-20",
|
||||
"poster_path": "/season1.jpg",
|
||||
"images": {"posters": []},
|
||||
"external_ids": {"tvdb_id": 12345},
|
||||
"episodes": [
|
||||
{
|
||||
"id": 62085,
|
||||
"episode_number": 1,
|
||||
"name": "Pilot",
|
||||
"overview": "Walter White begins his transformation.",
|
||||
"air_date": "2008-01-20",
|
||||
"runtime": 58,
|
||||
"still_path": "/still1.jpg",
|
||||
"crew": [
|
||||
{"id": 66633, "name": "Vince Gilligan", "department": "Directing", "job": "Director", "gender": 2}
|
||||
],
|
||||
"guest_stars": [
|
||||
{"id": 99999, "name": "Some Guest", "character": "Random Guy", "order": 5, "gender": 2}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": 62086,
|
||||
"episode_number": 2,
|
||||
"name": "Cat's in the Bag...",
|
||||
"overview": None,
|
||||
"air_date": "2008-01-27",
|
||||
"runtime": 48,
|
||||
"still_path": None,
|
||||
"crew": [],
|
||||
"guest_stars": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_series_maps_core_fields():
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
assert series.external_id == "1396"
|
||||
assert series.title == "Ozymandias"
|
||||
assert series.status == "ended"
|
||||
assert series.first_air_date == date(2008, 1, 20)
|
||||
assert series.number_of_seasons == 5
|
||||
|
||||
|
||||
def test_normalize_series_maps_networks_vs_production_companies():
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
assert [n.name for n in series.networks] == ["AMC"]
|
||||
assert [c.name for c in series.companies] == ["Gran Via Productions"]
|
||||
|
||||
|
||||
def test_normalize_series_maps_creators_separately_from_crew():
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
assert len(series.creators) == 1
|
||||
assert series.creators[0].name == "Vince Gilligan"
|
||||
assert series.creators[0].department == "Creator"
|
||||
assert all(c.department != "Creator" for c in series.crew)
|
||||
|
||||
|
||||
def test_normalize_series_keywords_and_alt_titles_use_tv_shaped_keys():
|
||||
# TMDb nests these under "results" for /tv, unlike "keywords"/"titles"
|
||||
# for /movie — a normalizer bug here would silently return empty lists.
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
assert series.keywords == ["drug dealer"]
|
||||
assert series.alternate_titles == ["Breaking Bad"]
|
||||
|
||||
|
||||
def test_normalize_series_keeps_season_zero_specials():
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
season_numbers = sorted(s.season_number for s in series.seasons)
|
||||
assert season_numbers == [0, 1]
|
||||
|
||||
|
||||
def test_normalize_series_external_ids_include_tvdb():
|
||||
series = normalize_series(FAKE_SERIES)
|
||||
assert series.external_ids["imdb"] == "tt0903747"
|
||||
assert series.external_ids["tvdb"] == "81189"
|
||||
|
||||
|
||||
def test_normalize_season_maps_episodes():
|
||||
season = normalize_season(FAKE_SEASON)
|
||||
assert season.season_number == 1
|
||||
assert len(season.episodes) == 2
|
||||
assert season.episodes[0].external_id == "62085"
|
||||
assert season.episodes[0].title == "Pilot"
|
||||
assert season.episodes[0].runtime_minutes == 58
|
||||
|
||||
|
||||
def test_normalize_season_separates_guest_stars_from_main_cast():
|
||||
season = normalize_season(FAKE_SEASON)
|
||||
episode = season.episodes[0]
|
||||
assert episode.cast == []
|
||||
assert len(episode.guest_stars) == 1
|
||||
assert episode.guest_stars[0].character == "Random Guy"
|
||||
|
||||
|
||||
def test_normalize_season_maps_episode_crew_and_stills():
|
||||
season = normalize_season(FAKE_SEASON)
|
||||
episode = season.episodes[0]
|
||||
assert episode.crew[0].job == "Director"
|
||||
assert len(episode.stills) == 1
|
||||
assert episode.stills[0].artwork_type == "still"
|
||||
|
||||
|
||||
def test_normalize_season_handles_episode_without_still():
|
||||
season = normalize_season(FAKE_SEASON)
|
||||
assert season.episodes[1].stills == []
|
||||
|
||||
Reference in New Issue
Block a user