From c20df1c503bcb8b54ea6e898b12d451ad51ba180 Mon Sep 17 00:00:00 2001 From: NixReaper Date: Fri, 4 Sep 2026 17:34:03 -0500 Subject: [PATCH] Initial Aurora Core 0.1 implementation Production-grade media metadata backend per docs/aurora-core-spec.md: Aurora-owned UUIDv7 identity system, normalized movie/TV/people/credits schema, provider abstraction with a TMDb adapter, an idempotent async ingestion pipeline (normalize -> match -> upsert -> provenance), a Postgres FTS/trigram search layer, a stateless FastAPI v1 API with Redis caching/rate-limiting, arq workers, dev/prod Docker Compose topologies, unit + integration + load tests, CI, and architecture/ database/API/ERD documentation. Co-Authored-By: Claude Sonnet 5 --- .env.example | 44 + .github/workflows/ci.yml | 140 ++ .gitignore | 23 + README.md | 183 +++ alembic.ini | 41 + app/__init__.py | 0 app/api/__init__.py | 0 app/api/deps.py | 37 + app/api/v1/__init__.py | 0 app/api/v1/admin.py | 93 ++ app/api/v1/collections.py | 26 + app/api/v1/companies.py | 26 + app/api/v1/episodes.py | 26 + app/api/v1/health.py | 36 + app/api/v1/movies.py | 64 + app/api/v1/people.py | 44 + app/api/v1/search.py | 25 + app/api/v1/seasons.py | 22 + app/api/v1/series.py | 44 + app/core/__init__.py | 0 app/core/config.py | 103 ++ app/core/database.py | 88 ++ app/core/ids.py | 135 ++ app/core/logging.py | 62 + app/core/queue.py | 35 + app/core/redis.py | 50 + app/core/security.py | 76 ++ app/main.py | 164 +++ app/models/__init__.py | 75 ++ app/models/artwork.py | 52 + app/models/base.py | 80 ++ app/models/collection.py | 59 + app/models/company.py | 62 + app/models/credit.py | 50 + app/models/enums.py | 84 ++ app/models/external_id.py | 45 + app/models/genre.py | 42 + app/models/keyword.py | 42 + app/models/language_country.py | 46 + app/models/movie.py | 127 ++ app/models/person.py | 32 + app/models/provider.py | 108 ++ app/models/tv.py | 118 ++ app/models/video.py | 50 + app/providers/__init__.py | 0 app/providers/base.py | 58 + app/providers/tmdb.py | 89 ++ app/repositories/__init__.py | 0 app/repositories/collection_repository.py | 39 + app/repositories/company_repository.py | 15 + app/repositories/movie_repository.py | 61 + app/repositories/person_repository.py | 26 + app/repositories/polymorphic.py | 90 ++ app/repositories/provider_repository.py | 145 +++ app/repositories/reference_repository.py | 270 ++++ app/repositories/tv_repository.py | 79 ++ app/schemas/__init__.py | 0 app/schemas/collection.py | 24 + app/schemas/common.py | 119 ++ app/schemas/company.py | 24 + app/schemas/health.py | 20 + app/schemas/movie.py | 80 ++ app/schemas/person.py | 55 + app/schemas/search.py | 20 + app/schemas/tv.py | 123 ++ app/services/__init__.py | 0 app/services/cache.py | 61 + app/services/collection_service.py | 75 ++ app/services/common.py | 105 ++ app/services/company_service.py | 80 ++ app/services/ingestion/__init__.py | 0 app/services/ingestion/movie_ingestion.py | 314 +++++ app/services/ingestion/person_ingestion.py | 115 ++ app/services/ingestion/runner.py | 103 ++ app/services/metadata/__init__.py | 0 app/services/metadata/matcher.py | 37 + app/services/metadata/normalizer.py | 341 +++++ app/services/movie_service.py | 147 +++ app/services/person_service.py | 127 ++ app/services/search/__init__.py | 0 app/services/search/base.py | 40 + app/services/search/postgresql_provider.py | 121 ++ app/services/search/service.py | 49 + app/services/tv_service.py | 229 ++++ app/workers/__init__.py | 0 app/workers/tasks.py | 22 + app/workers/worker.py | 37 + docker-compose.prod.yml | 137 ++ docker-compose.yml | 92 ++ docker/api.Dockerfile | 37 + docker/worker.Dockerfile | 30 + docs/api.md | 143 ++ docs/architecture.md | 154 +++ docs/aurora-core-spec.md | 1359 ++++++++++++++++++++ docs/database.md | 165 +++ docs/erd.md | 201 +++ migrations/env.py | 59 + migrations/script.py.mako | 27 + migrations/versions/0001_initial_schema.py | 813 ++++++++++++ pyproject.toml | 75 ++ tests/__init__.py | 0 tests/conftest.py | 87 ++ tests/integration/__init__.py | 0 tests/integration/factories.py | 81 ++ tests/integration/test_health.py | 26 + tests/integration/test_movie_ingestion.py | 105 ++ tests/integration/test_movies_api.py | 91 ++ tests/load/locustfile.py | 103 ++ tests/unit/__init__.py | 0 tests/unit/test_ids.py | 73 ++ tests/unit/test_normalizer.py | 184 +++ 111 files changed, 10041 insertions(+) create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/deps.py create mode 100644 app/api/v1/__init__.py create mode 100644 app/api/v1/admin.py create mode 100644 app/api/v1/collections.py create mode 100644 app/api/v1/companies.py create mode 100644 app/api/v1/episodes.py create mode 100644 app/api/v1/health.py create mode 100644 app/api/v1/movies.py create mode 100644 app/api/v1/people.py create mode 100644 app/api/v1/search.py create mode 100644 app/api/v1/seasons.py create mode 100644 app/api/v1/series.py create mode 100644 app/core/__init__.py create mode 100644 app/core/config.py create mode 100644 app/core/database.py create mode 100644 app/core/ids.py create mode 100644 app/core/logging.py create mode 100644 app/core/queue.py create mode 100644 app/core/redis.py create mode 100644 app/core/security.py create mode 100644 app/main.py create mode 100644 app/models/__init__.py create mode 100644 app/models/artwork.py create mode 100644 app/models/base.py create mode 100644 app/models/collection.py create mode 100644 app/models/company.py create mode 100644 app/models/credit.py create mode 100644 app/models/enums.py create mode 100644 app/models/external_id.py create mode 100644 app/models/genre.py create mode 100644 app/models/keyword.py create mode 100644 app/models/language_country.py create mode 100644 app/models/movie.py create mode 100644 app/models/person.py create mode 100644 app/models/provider.py create mode 100644 app/models/tv.py create mode 100644 app/models/video.py create mode 100644 app/providers/__init__.py create mode 100644 app/providers/base.py create mode 100644 app/providers/tmdb.py create mode 100644 app/repositories/__init__.py create mode 100644 app/repositories/collection_repository.py create mode 100644 app/repositories/company_repository.py create mode 100644 app/repositories/movie_repository.py create mode 100644 app/repositories/person_repository.py create mode 100644 app/repositories/polymorphic.py create mode 100644 app/repositories/provider_repository.py create mode 100644 app/repositories/reference_repository.py create mode 100644 app/repositories/tv_repository.py create mode 100644 app/schemas/__init__.py create mode 100644 app/schemas/collection.py create mode 100644 app/schemas/common.py create mode 100644 app/schemas/company.py create mode 100644 app/schemas/health.py create mode 100644 app/schemas/movie.py create mode 100644 app/schemas/person.py create mode 100644 app/schemas/search.py create mode 100644 app/schemas/tv.py create mode 100644 app/services/__init__.py create mode 100644 app/services/cache.py create mode 100644 app/services/collection_service.py create mode 100644 app/services/common.py create mode 100644 app/services/company_service.py create mode 100644 app/services/ingestion/__init__.py create mode 100644 app/services/ingestion/movie_ingestion.py create mode 100644 app/services/ingestion/person_ingestion.py create mode 100644 app/services/ingestion/runner.py create mode 100644 app/services/metadata/__init__.py create mode 100644 app/services/metadata/matcher.py create mode 100644 app/services/metadata/normalizer.py create mode 100644 app/services/movie_service.py create mode 100644 app/services/person_service.py create mode 100644 app/services/search/__init__.py create mode 100644 app/services/search/base.py create mode 100644 app/services/search/postgresql_provider.py create mode 100644 app/services/search/service.py create mode 100644 app/services/tv_service.py create mode 100644 app/workers/__init__.py create mode 100644 app/workers/tasks.py create mode 100644 app/workers/worker.py create mode 100644 docker-compose.prod.yml create mode 100644 docker-compose.yml create mode 100644 docker/api.Dockerfile create mode 100644 docker/worker.Dockerfile create mode 100644 docs/api.md create mode 100644 docs/architecture.md create mode 100644 docs/aurora-core-spec.md create mode 100644 docs/database.md create mode 100644 docs/erd.md create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/0001_initial_schema.py create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/factories.py create mode 100644 tests/integration/test_health.py create mode 100644 tests/integration/test_movie_ingestion.py create mode 100644 tests/integration/test_movies_api.py create mode 100644 tests/load/locustfile.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_ids.py create mode 100644 tests/unit/test_normalizer.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a5f3443 --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# Copy to .env and fill in real values for local development. +# NEVER commit a real .env file. + +# --- Runtime --- +AURORA_ENV=development +AURORA_LOG_LEVEL=INFO +AURORA_API_DOCS_ENABLED=true + +# --- Database --- +# Async URL used by the application (asyncpg driver). +DATABASE_URL=postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core +# Sync URL used by Alembic migrations (psycopg2 driver). +DATABASE_URL_SYNC=postgresql+psycopg2://aurora:aurora@localhost:5432/aurora_core +DATABASE_POOL_SIZE=10 +DATABASE_MAX_OVERFLOW=20 + +# --- Redis --- +REDIS_URL=redis://localhost:6379/0 +# If false, the app runs without caching/rate-limiting when Redis is unreachable +# instead of failing readiness checks. +REDIS_REQUIRED=false + +# --- Security --- +# Generate with: python -c "import secrets; print(secrets.token_urlsafe(48))" +AURORA_SECRET_KEY=changeme-generate-a-real-secret +# Comma-separated API keys allowed to call /api/v1/admin/* endpoints. +AURORA_ADMIN_API_KEYS=changeme-admin-key + +# --- CORS --- +CORS_ORIGINS=http://localhost:3000,http://localhost:5173 + +# --- Providers --- +# https://www.themoviedb.org/settings/api +TMDB_API_KEY= + +# --- Workers --- +WORKER_CONCURRENCY=4 + +# --- Caching --- +CACHE_DEFAULT_TTL=300 + +# --- Rate limiting --- +RATE_LIMIT_ENABLED=true +RATE_LIMIT_PER_MINUTE=120 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4ead3b0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,140 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +env: + PYTHON_VERSION: "3.12" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - run: pip install -e ".[dev]" + - run: ruff check app migrations tests + + typecheck: + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - run: pip install -e ".[dev]" + - run: mypy app + + unit-tests: + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - run: pip install -e ".[dev]" + - run: pytest tests/unit -v + + # Also proves migrations apply cleanly to an empty database (section 27): + # this job's Postgres service starts empty, and the first step to touch + # it is `alembic upgrade head`. + integration-tests: + runs-on: ubuntu-latest + needs: unit-tests + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: aurora + POSTGRES_PASSWORD: aurora + POSTGRES_DB: aurora_core_test + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U aurora" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + ports: ["6379:6379"] + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core_test + DATABASE_URL_SYNC: postgresql+psycopg2://aurora:aurora@localhost:5432/aurora_core_test + TEST_DATABASE_URL: postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core_test + REDIS_URL: redis://localhost:6379/0 + TEST_REDIS_URL: redis://localhost:6379/15 + AURORA_SECRET_KEY: ci-secret + AURORA_ADMIN_API_KEYS: ci-admin-key + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - run: pip install -e ".[dev]" + - name: Run migrations against an empty database + run: alembic upgrade head + - name: Start the API and smoke-test it + run: | + uvicorn app.main:app --host 0.0.0.0 --port 8000 & + for i in $(seq 1 20); do + curl -fsS http://localhost:8000/health && break + sleep 1 + done + curl -fsS http://localhost:8000/ready + curl -fsS "http://localhost:8000/api/v1/search?q=test" + kill %1 + - name: Run integration tests + run: pytest tests/integration -v + + container-build: + runs-on: ubuntu-latest + needs: [typecheck, integration-tests] + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build API image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/api.Dockerfile + push: false + tags: aurora-core-api:ci + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build worker image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/worker.Dockerfile + push: false + tags: aurora-core-worker:ci + cache-from: type=gha + cache-to: type=gha,mode=max + + dependency-audit: + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - run: pip install -e "." + - run: pip install pip-audit + - run: pip-audit --strict diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d900e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +.venv/ +venv/ +.env +.env.* +!.env.example +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +htmlcov/ +.coverage +.coverage.* +dist/ +build/ +*.log +.DS_Store +Thumbs.db +.idea/ +.vscode/ +media_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a216558 --- /dev/null +++ b/README.md @@ -0,0 +1,183 @@ +# Aurora Core + +Aurora Core is Aurora Dragon Studio's production-grade media metadata, +discovery, and intelligence backend — a TMDb/IMDb-style canonical data layer +for movies, TV, people, and everything that connects them. It is **not** a +Plex application: Plex, Overseerr, Radarr, Sonarr, and future Aurora +software are consumers built around Core, never dependencies of it. + +See [`docs/aurora-core-spec.md`](docs/aurora-core-spec.md) for the full +build specification this repository implements, and +[`docs/architecture.md`](docs/architecture.md) for how it's put together. + +## Architecture at a glance + +A modular monolith (not microservices): a stateless FastAPI API behind +Traefik, PostgreSQL as the sole source of truth, Redis for disposable +caching/rate-limiting/queueing, and arq workers for async metadata +ingestion. Full diagram and reasoning in +[`docs/architecture.md`](docs/architecture.md). + +- **API**: FastAPI + Pydantic v2, stateless, horizontally scalable +- **Database**: PostgreSQL via SQLAlchemy 2.0 (async) + Alembic migrations +- **Aurora IDs**: UUIDv7 primary keys, never a provider's ID — see + [`docs/database.md`](docs/database.md) +- **Cache/queue**: Redis + arq +- **Search**: PostgreSQL full-text + trigram, behind a swappable + `SearchProvider` abstraction +- **Providers**: TMDb today (`app/providers/tmdb.py`), pluggable + +## Prerequisites + +- Docker + Docker Compose (recommended path) +- *or*, for running outside Docker: Python 3.12, a local PostgreSQL 17 with + `pg_trgm` available, and Redis 7 + +## Local setup (Docker — recommended) + +```bash +cp .env.example .env +# edit .env: set AURORA_SECRET_KEY, AURORA_ADMIN_API_KEYS, and (optionally) +# TMDB_API_KEY — see https://www.themoviedb.org/settings/api + +docker compose up --build +``` + +This starts PostgreSQL, Redis, runs `alembic upgrade head` (the `migrate` +service), then the API (`http://localhost:8000`) and a worker. API docs: +`http://localhost:8000/docs`. + +## Local setup (without Docker) + +```bash +python -m venv .venv +.venv/Scripts/activate # Windows; use `source .venv/bin/activate` on macOS/Linux +pip install -e ".[dev]" + +cp .env.example .env # point DATABASE_URL / DATABASE_URL_SYNC / REDIS_URL at your own instances + +alembic upgrade head +uvicorn app.main:app --reload +``` + +Run a worker in a second terminal: + +```bash +arq app.workers.worker.WorkerSettings +``` + +## Environment configuration + +All configuration is environment-based (`app/core/config.py`); see +[`.env.example`](.env.example) for every variable and its meaning. Never +commit a real `.env` — only `.env.example` (placeholders) belongs in Git. + +## Database migrations + +```bash +alembic upgrade head # apply all migrations +alembic revision --autogenerate -m "add xyz" # generate the next one after changing models +alembic downgrade -1 # roll back one revision +``` + +The application never creates tables at startup — see +[`docs/database.md`](docs/database.md#migrations). + +## Ingesting data + +Ingestion always runs asynchronously through a worker, never inline in a +request (see [`docs/architecture.md`](docs/architecture.md)). With +`TMDB_API_KEY` set and a worker running: + +```bash +curl -X POST http://localhost:8000/api/v1/admin/ingest/movie \ + -H "X-API-Key: $AURORA_ADMIN_API_KEYS" \ + -H "Content-Type: application/json" \ + -d '{"tmdb_id": "603"}' +# -> {"job_id": "...", "arq_job_id": "...", "status": "enqueued"} + +curl http://localhost:8000/api/v1/admin/ingest/jobs/ \ + -H "X-API-Key: $AURORA_ADMIN_API_KEYS" +# -> poll until "status": "succeeded" + +curl "http://localhost:8000/api/v1/search?q=the%20matrix" +``` + +## Tests + +```bash +pytest tests/unit -v # no external dependencies + +# Integration/API tests need a real Postgres + Redis: +docker compose up -d postgres redis +createdb -h localhost -U aurora aurora_core_test # or: docker exec -it aurora-core-postgres-1 createdb -U aurora aurora_core_test +export TEST_DATABASE_URL=postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core_test +export TEST_REDIS_URL=redis://localhost:6379/15 +pytest tests/integration -v +``` + +Lint and type-check: + +```bash +ruff check app migrations tests +mypy app +``` + +Load testing (targets: 500 concurrent users, search p95 < 500ms, other +endpoints p95 < 250ms, 5xx rate < 0.1% — see +[`docs/aurora-core-spec.md`](docs/aurora-core-spec.md) section 2): + +```bash +pip install -e ".[load]" +locust -f tests/load/locustfile.py --host http://localhost:8000 +# open http://localhost:8089, seed some data first (see tests/load/locustfile.py) +``` + +## API documentation + +- Interactive (OpenAPI/Swagger): `/docs`, controlled by + `AURORA_API_DOCS_ENABLED` (set `false` in production if you don't want it + publicly exposed) +- Reference: [`docs/api.md`](docs/api.md) +- Schema/ERD: [`docs/database.md`](docs/database.md), + [`docs/erd.md`](docs/erd.md) + +## Worker operation + +Workers (arq, `app/workers/worker.py`) scale independently of the API — +`docker-compose.prod.yml` runs 2 worker replicas and 3 API replicas as an +example. Concurrency per worker process is controlled by +`WORKER_CONCURRENCY`. Jobs are idempotent (keyed by +`tmdb::`), retryable (`max_tries` in +`app/workers/worker.py`), and observable via +`GET /api/v1/admin/ingest/jobs/{job_id}`. + +## Production deployment + +```bash +docker compose -f docker-compose.prod.yml up --build -d +``` + +Traefik-ready (labels on the `api` service), PgBouncer in front of +PostgreSQL (transaction pooling — `DATABASE_BEHIND_TRANSACTION_POOLER=true` +disables asyncpg's server-side prepared statements, which don't survive +transaction-mode pooling), horizontally-scaled API and worker replicas. See +[`docs/architecture.md`](docs/architecture.md). + +## Known limitations + +See [`docs/database.md#known-limitations`](docs/database.md#known-limitations) +and [`docs/api.md#known-limitations`](docs/api.md#known-limitations) for the +current, tracked list (no curated language/country seed data, no fuzzy +entity matching, no field-level provenance, no pagination beyond search, +person credits limited to movies). None of these block the Core 0.1 +milestone (section 34): search → movie/person detail works end-to-end. + +This repository was built in an environment without Docker or a +pre-existing PostgreSQL instance; the initial Alembic migration was +hand-written from the models (not autogenerated against a live database) +and validated with `alembic history`, but has not yet been executed against +a running PostgreSQL instance. **Run `docker compose up --build` and confirm +`alembic upgrade head` succeeds against a genuinely empty database before +treating the schema as final** — see +[`docs/database.md#migrations`](docs/database.md#migrations). diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..29f2691 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,41 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +version_path_separator = os +sqlalchemy.url = driver://user:pass@localhost/dbname + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 0000000..93e8024 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,37 @@ +"""Shared FastAPI dependencies and helpers for the v1 API.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from fastapi import HTTPException, status +from pydantic import BaseModel +from redis.asyncio import Redis + +from app.services.cache import cache_get_json, cache_set_json + + +def not_found(entity_type: str, aurora_id_str: str) -> HTTPException: + return HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": f"{entity_type}_not_found", "message": f"No such {entity_type}: {aurora_id_str}"}, + ) + + +async def cached_response[T: BaseModel]( + redis: Redis, + key: str, + model: type[T], + loader: Callable[[], Awaitable[T]], + ttl: int | None = None, +) -> T: + """Serve `model` from Redis if present, otherwise call `loader`, cache + the result, and return it. Fails open to `loader` on any cache error.""" + + cached = await cache_get_json(redis, key) + if cached is not None: + return model.model_validate(cached) + + result = await loader() + await cache_set_json(redis, key, result.model_dump(mode="json"), ttl=ttl) + return result diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/v1/admin.py b/app/api/v1/admin.py new file mode 100644 index 0000000..d285c38 --- /dev/null +++ b/app/api/v1/admin.py @@ -0,0 +1,93 @@ +"""Administrative/provider endpoints (section 17): trigger ingestion and +check job status. Separately protected by an admin API key — never mounted +on the public read path. +""" + +from __future__ import annotations + +import uuid + +from arq import ArqRedis +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import not_found +from app.core.database import get_db +from app.core.queue import get_queue +from app.core.security import require_admin_api_key +from app.repositories.provider_repository import get_ingestion_job + +router = APIRouter( + prefix="/admin", + tags=["admin"], + dependencies=[Depends(require_admin_api_key)], +) + + +class IngestMovieRequest(BaseModel): + tmdb_id: str + + +class IngestPersonRequest(BaseModel): + tmdb_id: str + + +class IngestionEnqueuedOut(BaseModel): + job_id: str + arq_job_id: str + status: str = "enqueued" + + +class IngestionJobOut(BaseModel): + id: str + job_type: str + entity_type: str | None + external_id: str | None + status: str + attempts: int + error: str | None + result_aurora_id: str | None + + +@router.post("/ingest/movie", response_model=IngestionEnqueuedOut) +async def enqueue_movie_ingestion( + body: IngestMovieRequest, queue: ArqRedis = Depends(get_queue) +) -> IngestionEnqueuedOut: + job = await queue.enqueue_job("ingest_movie_task", body.tmdb_id) + if job is None: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, "Could not enqueue ingestion job (queue busy)." + ) + return IngestionEnqueuedOut(job_id=job.job_id, arq_job_id=job.job_id) + + +@router.post("/ingest/person", response_model=IngestionEnqueuedOut) +async def enqueue_person_ingestion( + body: IngestPersonRequest, queue: ArqRedis = Depends(get_queue) +) -> IngestionEnqueuedOut: + job = await queue.enqueue_job("ingest_person_task", body.tmdb_id) + if job is None: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, "Could not enqueue ingestion job (queue busy)." + ) + return IngestionEnqueuedOut(job_id=job.job_id, arq_job_id=job.job_id) + + +@router.get("/ingest/jobs/{job_id}", response_model=IngestionJobOut) +async def get_ingestion_job_status( + job_id: uuid.UUID, session: AsyncSession = Depends(get_db) +) -> IngestionJobOut: + job = await get_ingestion_job(session, job_id) + if job is None: + raise not_found("ingestion_job", str(job_id)) + return IngestionJobOut( + id=str(job.id), + job_type=job.job_type, + entity_type=job.entity_type, + external_id=job.external_id, + status=job.status.value, + attempts=job.attempts, + error=job.error, + result_aurora_id=str(job.result_aurora_id) if job.result_aurora_id else None, + ) diff --git a/app/api/v1/collections.py b/app/api/v1/collections.py new file mode 100644 index 0000000..0c4f1aa --- /dev/null +++ b/app/api/v1/collections.py @@ -0,0 +1,26 @@ +"""Collection/franchise endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import not_found +from app.core.database import get_db +from app.core.security import enforce_rate_limit +from app.schemas.collection import CollectionDetailOut +from app.services import collection_service + +router = APIRouter( + prefix="/collections", tags=["collections"], dependencies=[Depends(enforce_rate_limit)] +) + + +@router.get("/{aurora_id}", response_model=CollectionDetailOut) +async def get_collection( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> CollectionDetailOut: + try: + return await collection_service.get_collection_detail(session, aurora_id) + except collection_service.CollectionNotFoundError as exc: + raise not_found("collection", aurora_id) from exc diff --git a/app/api/v1/companies.py b/app/api/v1/companies.py new file mode 100644 index 0000000..8134ede --- /dev/null +++ b/app/api/v1/companies.py @@ -0,0 +1,26 @@ +"""Company endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import not_found +from app.core.database import get_db +from app.core.security import enforce_rate_limit +from app.schemas.company import CompanyDetailOut +from app.services import company_service + +router = APIRouter( + prefix="/companies", tags=["companies"], dependencies=[Depends(enforce_rate_limit)] +) + + +@router.get("/{aurora_id}", response_model=CompanyDetailOut) +async def get_company( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> CompanyDetailOut: + try: + return await company_service.get_company_detail(session, aurora_id) + except company_service.CompanyNotFoundError as exc: + raise not_found("company", aurora_id) from exc diff --git a/app/api/v1/episodes.py b/app/api/v1/episodes.py new file mode 100644 index 0000000..19210a1 --- /dev/null +++ b/app/api/v1/episodes.py @@ -0,0 +1,26 @@ +"""Episode endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import not_found +from app.core.database import get_db +from app.core.security import enforce_rate_limit +from app.schemas.tv import EpisodeDetailOut +from app.services import tv_service + +router = APIRouter( + prefix="/episodes", tags=["episodes"], dependencies=[Depends(enforce_rate_limit)] +) + + +@router.get("/{aurora_id}", response_model=EpisodeDetailOut) +async def get_episode( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> EpisodeDetailOut: + try: + return await tv_service.get_episode_detail(session, aurora_id) + except tv_service.EpisodeNotFoundError as exc: + raise not_found("episode", aurora_id) from exc diff --git a/app/api/v1/health.py b/app/api/v1/health.py new file mode 100644 index 0000000..3b4c22d --- /dev/null +++ b/app/api/v1/health.py @@ -0,0 +1,36 @@ +"""Liveness and readiness endpoints (section 24).""" + +from __future__ import annotations + +from fastapi import APIRouter + +from app.core.config import get_settings +from app.core.database import database_is_ready +from app.core.redis import redis_is_ready +from app.schemas.health import DependencyStatus, HealthOut, ReadyOut + +router = APIRouter(tags=["health"]) + + +@router.get("/health", response_model=HealthOut) +async def health() -> HealthOut: + """Liveness probe: process is up and able to respond. No dependency + checks — that's /ready.""" + + return HealthOut(status="ok") + + +@router.get("/ready", response_model=ReadyOut) +async def ready() -> ReadyOut: + settings = get_settings() + + db_ok = await database_is_ready() + redis_ok = await redis_is_ready() + + dependencies = [ + DependencyStatus(name="postgresql", ok=db_ok, required=True), + DependencyStatus(name="redis", ok=redis_ok, required=settings.REDIS_REQUIRED), + ] + + is_ready = db_ok and (redis_ok or not settings.REDIS_REQUIRED) + return ReadyOut(status="ready" if is_ready else "not_ready", dependencies=dependencies) diff --git a/app/api/v1/movies.py b/app/api/v1/movies.py new file mode 100644 index 0000000..ef4cf34 --- /dev/null +++ b/app/api/v1/movies.py @@ -0,0 +1,64 @@ +"""Movie endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import cached_response, not_found +from app.core.database import get_db +from app.core.redis import get_redis +from app.core.security import enforce_rate_limit +from app.schemas.movie import MovieCreditsOut, MovieDetailOut, MovieImagesOut, MovieVideosOut +from app.services import movie_service +from app.services.cache import build_key + +router = APIRouter(prefix="/movies", tags=["movies"], dependencies=[Depends(enforce_rate_limit)]) + + +@router.get("/{aurora_id}", response_model=MovieDetailOut) +async def get_movie( + aurora_id: str, + session: AsyncSession = Depends(get_db), + redis: Redis = Depends(get_redis), +) -> MovieDetailOut: + try: + return await cached_response( + redis, + build_key("movie", aurora_id), + MovieDetailOut, + lambda: movie_service.get_movie_detail(session, aurora_id), + ) + except movie_service.MovieNotFoundError as exc: + raise not_found("movie", aurora_id) from exc + + +@router.get("/{aurora_id}/credits", response_model=MovieCreditsOut) +async def get_movie_credits( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> MovieCreditsOut: + try: + return await movie_service.get_movie_credits(session, aurora_id) + except movie_service.MovieNotFoundError as exc: + raise not_found("movie", aurora_id) from exc + + +@router.get("/{aurora_id}/images", response_model=MovieImagesOut) +async def get_movie_images( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> MovieImagesOut: + try: + return await movie_service.get_movie_images(session, aurora_id) + except movie_service.MovieNotFoundError as exc: + raise not_found("movie", aurora_id) from exc + + +@router.get("/{aurora_id}/videos", response_model=MovieVideosOut) +async def get_movie_videos( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> MovieVideosOut: + try: + return await movie_service.get_movie_videos(session, aurora_id) + except movie_service.MovieNotFoundError as exc: + raise not_found("movie", aurora_id) from exc diff --git a/app/api/v1/people.py b/app/api/v1/people.py new file mode 100644 index 0000000..2b81038 --- /dev/null +++ b/app/api/v1/people.py @@ -0,0 +1,44 @@ +"""Person endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import cached_response, not_found +from app.core.database import get_db +from app.core.redis import get_redis +from app.core.security import enforce_rate_limit +from app.schemas.person import PersonCreditsOut, PersonDetailOut +from app.services import person_service +from app.services.cache import build_key + +router = APIRouter(prefix="/people", tags=["people"], dependencies=[Depends(enforce_rate_limit)]) + + +@router.get("/{aurora_id}", response_model=PersonDetailOut) +async def get_person( + aurora_id: str, + session: AsyncSession = Depends(get_db), + redis: Redis = Depends(get_redis), +) -> PersonDetailOut: + try: + return await cached_response( + redis, + build_key("person", aurora_id), + PersonDetailOut, + lambda: person_service.get_person_detail(session, aurora_id), + ) + except person_service.PersonNotFoundError as exc: + raise not_found("person", aurora_id) from exc + + +@router.get("/{aurora_id}/credits", response_model=PersonCreditsOut) +async def get_person_credits( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> PersonCreditsOut: + try: + return await person_service.get_person_credits(session, aurora_id) + except person_service.PersonNotFoundError as exc: + raise not_found("person", aurora_id) from exc diff --git a/app/api/v1/search.py b/app/api/v1/search.py new file mode 100644 index 0000000..af204ee --- /dev/null +++ b/app/api/v1/search.py @@ -0,0 +1,25 @@ +"""Search endpoint (section 16, section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.core.security import enforce_rate_limit +from app.schemas.search import SearchResultsOut +from app.services.search.postgresql_provider import PostgreSQLSearchProvider +from app.services.search.service import SearchService + +router = APIRouter(prefix="/search", tags=["search"], dependencies=[Depends(enforce_rate_limit)]) + +_search_service = SearchService(PostgreSQLSearchProvider()) + + +@router.get("", response_model=SearchResultsOut) +async def search( + q: str = Query(..., min_length=1, max_length=300), + limit: int = Query(10, ge=1, le=50), + session: AsyncSession = Depends(get_db), +) -> SearchResultsOut: + return await _search_service.search(session, q, limit_per_type=limit) diff --git a/app/api/v1/seasons.py b/app/api/v1/seasons.py new file mode 100644 index 0000000..5d90212 --- /dev/null +++ b/app/api/v1/seasons.py @@ -0,0 +1,22 @@ +"""Season endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import not_found +from app.core.database import get_db +from app.core.security import enforce_rate_limit +from app.schemas.tv import SeasonDetailOut +from app.services import tv_service + +router = APIRouter(prefix="/seasons", tags=["seasons"], dependencies=[Depends(enforce_rate_limit)]) + + +@router.get("/{aurora_id}", response_model=SeasonDetailOut) +async def get_season(aurora_id: str, session: AsyncSession = Depends(get_db)) -> SeasonDetailOut: + try: + return await tv_service.get_season_detail(session, aurora_id) + except tv_service.SeasonNotFoundError as exc: + raise not_found("season", aurora_id) from exc diff --git a/app/api/v1/series.py b/app/api/v1/series.py new file mode 100644 index 0000000..603bc6d --- /dev/null +++ b/app/api/v1/series.py @@ -0,0 +1,44 @@ +"""Series endpoints (section 17).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import cached_response, not_found +from app.core.database import get_db +from app.core.redis import get_redis +from app.core.security import enforce_rate_limit +from app.schemas.tv import SeriesDetailOut, SeriesSeasonsOut +from app.services import tv_service +from app.services.cache import build_key + +router = APIRouter(prefix="/series", tags=["series"], dependencies=[Depends(enforce_rate_limit)]) + + +@router.get("/{aurora_id}", response_model=SeriesDetailOut) +async def get_series( + aurora_id: str, + session: AsyncSession = Depends(get_db), + redis: Redis = Depends(get_redis), +) -> SeriesDetailOut: + try: + return await cached_response( + redis, + build_key("series", aurora_id), + SeriesDetailOut, + lambda: tv_service.get_series_detail(session, aurora_id), + ) + except tv_service.SeriesNotFoundError as exc: + raise not_found("series", aurora_id) from exc + + +@router.get("/{aurora_id}/seasons", response_model=SeriesSeasonsOut) +async def get_series_seasons( + aurora_id: str, session: AsyncSession = Depends(get_db) +) -> SeriesSeasonsOut: + try: + return await tv_service.get_series_seasons(session, aurora_id) + except tv_service.SeriesNotFoundError as exc: + raise not_found("series", aurora_id) from exc diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 0000000..bd2cd2a --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,103 @@ +"""Environment-based application configuration. + +All runtime configuration flows through this module. Nothing else in the +codebase should read `os.environ` directly, so that configuration stays +auditable and testable in one place. +""" + +from __future__ import annotations + +from functools import lru_cache + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + # --- Runtime --- + AURORA_ENV: str = Field(default="development") + AURORA_LOG_LEVEL: str = Field(default="INFO") + AURORA_API_DOCS_ENABLED: bool = Field(default=True) + + # --- Database --- + DATABASE_URL: str = Field( + default="postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core" + ) + DATABASE_URL_SYNC: str = Field( + default="postgresql+psycopg2://aurora:aurora@localhost:5432/aurora_core" + ) + DATABASE_POOL_SIZE: int = Field(default=10) + DATABASE_MAX_OVERFLOW: int = Field(default=20) + # Set true when DATABASE_URL points at PgBouncer in transaction pooling + # mode (docker-compose.prod.yml): asyncpg's server-side prepared + # statements don't survive connections being handed to different clients + # between transactions, so they must be disabled. + DATABASE_BEHIND_TRANSACTION_POOLER: bool = Field(default=False) + + # --- Redis --- + REDIS_URL: str = Field(default="redis://localhost:6379/0") + REDIS_REQUIRED: bool = Field(default=False) + + # --- Security --- + AURORA_SECRET_KEY: str = Field(default="changeme-generate-a-real-secret") + AURORA_ADMIN_API_KEYS: str = Field(default="") + + # --- CORS --- + CORS_ORIGINS: str = Field(default="http://localhost:3000") + + # --- Providers --- + TMDB_API_KEY: str = Field(default="") + + # --- Workers --- + WORKER_CONCURRENCY: int = Field(default=4) + + # --- Caching --- + CACHE_DEFAULT_TTL: int = Field(default=300) + + # --- Rate limiting --- + RATE_LIMIT_ENABLED: bool = Field(default=True) + RATE_LIMIT_PER_MINUTE: int = Field(default=120) + + @field_validator("AURORA_ENV") + @classmethod + def _validate_env(cls, v: str) -> str: + allowed = {"development", "test", "staging", "production"} + if v not in allowed: + raise ValueError(f"AURORA_ENV must be one of {allowed}, got {v!r}") + return v + + @property + def cors_origins_list(self) -> list[str]: + return [origin.strip() for origin in self.CORS_ORIGINS.split(",") if origin.strip()] + + @property + def admin_api_keys_set(self) -> set[str]: + return {key.strip() for key in self.AURORA_ADMIN_API_KEYS.split(",") if key.strip()} + + @property + def is_production(self) -> bool: + return self.AURORA_ENV == "production" + + @property + def docs_url(self) -> str | None: + if not self.AURORA_API_DOCS_ENABLED: + return None + return "/docs" + + @property + def openapi_url(self) -> str | None: + if not self.AURORA_API_DOCS_ENABLED: + return None + return "/openapi.json" + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/app/core/database.py b/app/core/database.py new file mode 100644 index 0000000..251d8b1 --- /dev/null +++ b/app/core/database.py @@ -0,0 +1,88 @@ +"""Async SQLAlchemy engine/session management. + +The API is stateless: every request gets its own AsyncSession from a shared +connection pool, and nothing is held across requests. This module never +creates tables — schema changes always go through Alembic (see +`migrations/`). +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.core.config import get_settings + +settings = get_settings() + +_connect_args: dict = {} +if settings.DATABASE_BEHIND_TRANSACTION_POOLER: + _connect_args["statement_cache_size"] = 0 + +_engine: AsyncEngine = create_async_engine( + settings.DATABASE_URL, + pool_size=settings.DATABASE_POOL_SIZE, + max_overflow=settings.DATABASE_MAX_OVERFLOW, + pool_pre_ping=True, + pool_recycle=1800, + connect_args=_connect_args, + echo=False, +) + +async_session_factory = async_sessionmaker( + bind=_engine, + class_=AsyncSession, + expire_on_commit=False, + autoflush=False, +) + + +def get_engine() -> AsyncEngine: + return _engine + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + """FastAPI dependency yielding a request-scoped session.""" + + async with async_session_factory() as session: + yield session + + +@asynccontextmanager +async def session_scope() -> AsyncGenerator[AsyncSession, None]: + """Context manager for use outside request handlers (workers, scripts). + + Commits on success, rolls back on exception. + """ + + async with async_session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + +async def database_is_ready() -> bool: + """Cheap connectivity check used by the /ready endpoint.""" + + from sqlalchemy import text + + try: + async with _engine.connect() as conn: + await conn.execute(text("SELECT 1")) + return True + except Exception: + return False + + +async def dispose_engine() -> None: + await _engine.dispose() diff --git a/app/core/ids.py b/app/core/ids.py new file mode 100644 index 0000000..82c2ab0 --- /dev/null +++ b/app/core/ids.py @@ -0,0 +1,135 @@ +"""Aurora's own identity system. + +Aurora Core owns its entity identities. Provider identifiers (TMDb, IMDb, +TVDb, ...) are stored as external references, never as primary keys — see +`app.models.external_id`. + +Every Aurora entity has: + +- a UUIDv7 database primary key (time-ordered, efficient as a btree index, + generated with :func:`new_uuid7`) +- a public "Aurora ID" string derived from that same UUID, e.g. + ``aur_movie_1jz8f2n3k4m5p6q7r8s9t0v1w2``, produced by :func:`aurora_id` + +Because the public ID is a direct encoding of the UUIDv7 primary key, the two +never drift apart, and Aurora IDs sort lexicographically in creation order — +useful for cursors and debugging without leaking a sequential integer. +""" + +from __future__ import annotations + +import re + +from uuid6 import uuid7 + +# Crockford base32: excludes I, L, O, U to avoid visual ambiguity in IDs. +_CROCKFORD_ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz" +_CROCKFORD_INDEX = {ch: i for i, ch in enumerate(_CROCKFORD_ALPHABET)} + +# Aurora entity type -> ID prefix. Keep stable once shipped; these are +# effectively part of the public API contract. +ENTITY_PREFIXES: dict[str, str] = { + "movie": "movie", + "series": "series", + "season": "season", + "episode": "episode", + "person": "person", + "credit": "credit", + "company": "company", + "collection": "collection", + "genre": "genre", + "keyword": "keyword", + "artwork": "art", + "video": "vid", + "external_id": "ext", + "provider": "provider", + "provider_record": "prec", + "ingestion_job": "job", + "certification": "cert", + "language": "lang", + "country": "country", + "alternate_title": "altt", + "release": "rel", +} + + +def new_uuid7() -> str: + """Return a fresh UUIDv7 as its canonical hyphenated string form.""" + + return str(uuid7()) + + +def _encode_uuid(uuid_str: str) -> str: + """Encode a UUID's 128 bits as lowercase Crockford base32 (26 chars). + + Byte-order-preserving: since the UUID is time-ordered (v7) and this + encoding processes bytes most-significant-first, the resulting strings + sort lexicographically the same way the UUIDs (and thus creation time) + do. + """ + + raw = bytes.fromhex(uuid_str.replace("-", "")) + value = int.from_bytes(raw, "big") + chars: list[str] = [] + # 128 bits / 5 bits-per-symbol = 25.6 -> 26 symbols, top symbol uses 3 bits. + for shift in range(125, -5, -5): + chars.append(_CROCKFORD_ALPHABET[(value >> shift) & 0b11111]) + return "".join(chars) + + +def _decode_to_uuid(encoded: str) -> str: + if len(encoded) != 26: + raise ValueError(f"invalid Aurora ID segment length: {encoded!r}") + value = 0 + for ch in encoded: + try: + value = (value << 5) | _CROCKFORD_INDEX[ch] + except KeyError as exc: + raise ValueError(f"invalid Aurora ID character: {ch!r}") from exc + try: + raw = value.to_bytes(16, "big") + except OverflowError as exc: + raise ValueError(f"invalid Aurora ID segment: {encoded!r}") from exc + hex_str = raw.hex() + return f"{hex_str[0:8]}-{hex_str[8:12]}-{hex_str[12:16]}-{hex_str[16:20]}-{hex_str[20:32]}" + + +def aurora_id(entity_type: str, uuid_str: str) -> str: + """Build a public Aurora ID string from an entity type and UUIDv7 PK.""" + + prefix = ENTITY_PREFIXES.get(entity_type) + if prefix is None: + raise ValueError(f"unknown Aurora entity type: {entity_type!r}") + return f"aur_{prefix}_{_encode_uuid(uuid_str)}" + + +_AURORA_ID_RE = re.compile(r"^aur_(?P[a-z_]+)_(?P[0-9a-hjkmnp-tv-z]{26})$") + +_PREFIX_TO_ENTITY: dict[str, str] = {v: k for k, v in ENTITY_PREFIXES.items()} + + +def parse_aurora_id(aid: str, expected_type: str | None = None) -> str: + """Validate an Aurora ID and return the underlying UUID string. + + Raises ValueError if the ID is malformed or (when `expected_type` is + given) does not belong to that entity type. + """ + + match = _AURORA_ID_RE.match(aid) + if not match: + raise ValueError(f"malformed Aurora ID: {aid!r}") + prefix = match.group("prefix") + entity_type = _PREFIX_TO_ENTITY.get(prefix) + if entity_type is None: + raise ValueError(f"unknown Aurora ID prefix: {prefix!r}") + if expected_type is not None and entity_type != expected_type: + raise ValueError(f"expected Aurora ID of type {expected_type!r}, got {entity_type!r}") + return _decode_to_uuid(match.group("encoded")) + + +def is_valid_aurora_id(aid: str, expected_type: str | None = None) -> bool: + try: + parse_aurora_id(aid, expected_type=expected_type) + except ValueError: + return False + return True diff --git a/app/core/logging.py b/app/core/logging.py new file mode 100644 index 0000000..cec0c73 --- /dev/null +++ b/app/core/logging.py @@ -0,0 +1,62 @@ +"""Structured, JSON-compatible logging with per-request correlation IDs. + +Every request gets a request ID (from the `X-Request-ID` header if present, +otherwise generated) bound into the log context for the lifetime of that +request, via `RequestIDMiddleware` in `app.main`. +""" + +from __future__ import annotations + +import logging +import sys +from contextvars import ContextVar + +import structlog + +from app.core.config import get_settings + +request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None) + + +def _add_request_id(_logger: object, _method_name: str, event_dict: dict) -> dict: + request_id = request_id_ctx.get() + if request_id is not None: + event_dict["request_id"] = request_id + return event_dict + + +def configure_logging() -> None: + settings = get_settings() + level = getattr(logging, settings.AURORA_LOG_LEVEL.upper(), logging.INFO) + + logging.basicConfig( + format="%(message)s", + stream=sys.stdout, + level=level, + ) + + shared_processors: list = [ + structlog.contextvars.merge_contextvars, + _add_request_id, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.StackInfoRenderer(), + ] + + renderer: structlog.types.Processor + if settings.is_production or settings.AURORA_ENV != "development": + renderer = structlog.processors.JSONRenderer() + else: + renderer = structlog.dev.ConsoleRenderer() + + structlog.configure( + processors=[*shared_processors, structlog.processors.format_exc_info, renderer], + wrapper_class=structlog.make_filtering_bound_logger(level), + context_class=dict, + logger_factory=structlog.PrintLoggerFactory(), + cache_logger_on_first_use=True, + ) + + +def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger: + return structlog.get_logger(name) diff --git a/app/core/queue.py b/app/core/queue.py new file mode 100644 index 0000000..6dee251 --- /dev/null +++ b/app/core/queue.py @@ -0,0 +1,35 @@ +"""arq job queue connection (Redis-backed), shared by the API (enqueuing) +and the worker process (consuming) — see app/workers/worker.py.""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator + +from arq import ArqRedis, create_pool +from arq.connections import RedisSettings + +from app.core.config import get_settings + +_pool: ArqRedis | None = None + + +def redis_settings() -> RedisSettings: + return RedisSettings.from_dsn(get_settings().REDIS_URL) + + +async def get_arq_pool() -> ArqRedis: + global _pool + if _pool is None: + _pool = await create_pool(redis_settings()) + return _pool + + +async def get_queue() -> AsyncGenerator[ArqRedis, None]: + yield await get_arq_pool() + + +async def close_arq_pool() -> None: + global _pool + if _pool is not None: + await _pool.aclose() # type: ignore[attr-defined] # present at runtime; stubs lag redis-py 5.x + _pool = None diff --git a/app/core/redis.py b/app/core/redis.py new file mode 100644 index 0000000..7f31bc6 --- /dev/null +++ b/app/core/redis.py @@ -0,0 +1,50 @@ +"""Redis connection management. + +Redis is disposable acceleration/coordination — caching, rate limiting, +distributed locks, and the arq job queue. It is never the system of record. +If `REDIS_REQUIRED` is false, the application degrades gracefully (caching +becomes a no-op, readiness does not depend on Redis) instead of failing when +Redis is unavailable. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from functools import lru_cache + +import redis.asyncio as redis_asyncio +from redis.asyncio import Redis + +from app.core.config import get_settings + +settings = get_settings() + + +@lru_cache +def get_redis_pool() -> Redis: + return redis_asyncio.from_url( + settings.REDIS_URL, + encoding="utf-8", + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + ) + + +async def get_redis() -> AsyncGenerator[Redis, None]: + """FastAPI dependency yielding the shared Redis client.""" + + yield get_redis_pool() + + +async def redis_is_ready() -> bool: + try: + client = get_redis_pool() + return bool(await client.ping()) + except Exception: + return False + + +async def close_redis() -> None: + client = get_redis_pool() + await client.aclose() # type: ignore[attr-defined] # present at runtime; stubs lag redis-py 5.x diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 0000000..da4d0d0 --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,76 @@ +"""Administrative auth and Redis-backed rate limiting. + +Public read endpoints (`/api/v1/movies`, `/search`, ...) are unauthenticated +but rate-limited. Administrative/provider endpoints (ingestion triggers, +provider config) require an API key via the `X-API-Key` header. +""" + +from __future__ import annotations + +import time + +from fastapi import Depends, HTTPException, Request, status +from fastapi.security import APIKeyHeader +from redis.asyncio import Redis + +from app.core.config import get_settings +from app.core.redis import get_redis + +_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False) + + +async def require_admin_api_key( + api_key: str | None = Depends(_api_key_header), +) -> str: + settings = get_settings() + allowed = settings.admin_api_keys_set + if not allowed: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Administrative API is not configured (AURORA_ADMIN_API_KEYS unset).", + ) + if not api_key or api_key not in allowed: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing or invalid X-API-Key.", + ) + return api_key + + +class RedisRateLimiter: + """Fixed-window rate limiter backed by Redis `INCR` + `EXPIRE`. + + Fails open: if Redis is unreachable, requests are allowed through rather + than rejected, since Redis is disposable infrastructure and must never + become a hard dependency for serving reads. + """ + + def __init__(self, requests_per_minute: int) -> None: + self.requests_per_minute = requests_per_minute + + async def check(self, redis: Redis, key: str) -> tuple[bool, int]: + window = int(time.time() // 60) + redis_key = f"ratelimit:{key}:{window}" + try: + count = await redis.incr(redis_key) + if count == 1: + await redis.expire(redis_key, 60) + remaining = max(0, self.requests_per_minute - count) + return count <= self.requests_per_minute, remaining + except Exception: + return True, self.requests_per_minute + + +async def enforce_rate_limit(request: Request, redis: Redis = Depends(get_redis)) -> None: + settings = get_settings() + if not settings.RATE_LIMIT_ENABLED: + return + limiter = RedisRateLimiter(settings.RATE_LIMIT_PER_MINUTE) + client_id = request.client.host if request.client else "unknown" + allowed, remaining = await limiter.check(redis, client_id) + if not allowed: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Rate limit exceeded.", + headers={"Retry-After": "60"}, + ) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..a1297f7 --- /dev/null +++ b/app/main.py @@ -0,0 +1,164 @@ +"""Aurora Core FastAPI application factory. + +Stateless by design (section 1): nothing here holds request-scoped state in +process memory, so any number of API replicas can run behind Traefik. +""" + +from __future__ import annotations + +import time +import uuid +from collections.abc import Awaitable, Callable +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app.api.v1 import ( + admin, + collections, + companies, + episodes, + health, + movies, + people, + search, + seasons, + series, +) +from app.core.config import get_settings +from app.core.database import dispose_engine +from app.core.logging import configure_logging, get_logger, request_id_ctx +from app.core.queue import close_arq_pool +from app.core.redis import close_redis + +settings = get_settings() + +REQUEST_COUNT = Counter( + "aurora_http_requests_total", "HTTP requests", ["method", "path", "status"] +) +REQUEST_LATENCY = Histogram( + "aurora_http_request_duration_seconds", "HTTP request latency", ["method", "path"] +) + +MAX_REQUEST_BODY_BYTES = 2 * 1024 * 1024 # 2 MiB + + +@asynccontextmanager +async def lifespan(app: FastAPI): + configure_logging() + logger = get_logger(__name__) + logger.info("aurora_core_starting", env=settings.AURORA_ENV) + yield + logger.info("aurora_core_stopping") + await dispose_engine() + await close_redis() + await close_arq_pool() + + +app = FastAPI( + title="Aurora Core", + description="Aurora Dragon Studio's media metadata, discovery, and intelligence backend.", + version="0.1.0", + docs_url=settings.docs_url, + redoc_url=None, + openapi_url=settings.openapi_url, + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins_list, + allow_credentials=True, + allow_methods=["GET", "POST"], + allow_headers=["*"], +) + + +@app.middleware("http") +async def request_context_middleware( + request: Request, call_next: Callable[[Request], Awaitable[Response]] +) -> Response: + request_id = request.headers.get("X-Request-ID", str(uuid.uuid4())) + token = request_id_ctx.set(request_id) + start = time.perf_counter() + try: + response = await call_next(request) + finally: + request_id_ctx.reset(token) + + duration = time.perf_counter() - start + route = request.scope.get("route") + path_template = route.path if route else request.url.path + REQUEST_COUNT.labels(request.method, path_template, "n/a").inc() + REQUEST_LATENCY.labels(request.method, path_template).observe(duration) + + response.headers["X-Request-ID"] = request_id + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["Referrer-Policy"] = "no-referrer" + return response + + +@app.middleware("http") +async def request_size_limit_middleware( + request: Request, call_next: Callable[[Request], Awaitable[Response]] +) -> Response: + content_length = request.headers.get("content-length") + if content_length and int(content_length) > MAX_REQUEST_BODY_BYTES: + return JSONResponse( + status_code=413, + content={"error": {"code": "payload_too_large", "message": "Request body too large."}}, + ) + return await call_next(request) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + detail = exc.detail + if isinstance(detail, dict) and "code" in detail: + error = detail + else: + error = {"code": "http_error", "message": str(detail)} + return JSONResponse(status_code=exc.status_code, content={"error": error}) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, exc: RequestValidationError +) -> JSONResponse: + return JSONResponse( + status_code=422, + content={ + "error": { + "code": "validation_error", + "message": "Request validation failed.", + "details": exc.errors(), + } + }, + ) + + +@app.get("/metrics", include_in_schema=False) +async def metrics() -> Response: + return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST) + + +# Unversioned liveness/readiness for infra probes (Docker/Traefik healthchecks). +app.include_router(health.router) + +API_V1_PREFIX = "/api/v1" +app.include_router(health.router, prefix=API_V1_PREFIX) +app.include_router(search.router, prefix=API_V1_PREFIX) +app.include_router(movies.router, prefix=API_V1_PREFIX) +app.include_router(series.router, prefix=API_V1_PREFIX) +app.include_router(seasons.router, prefix=API_V1_PREFIX) +app.include_router(episodes.router, prefix=API_V1_PREFIX) +app.include_router(people.router, prefix=API_V1_PREFIX) +app.include_router(collections.router, prefix=API_V1_PREFIX) +app.include_router(companies.router, prefix=API_V1_PREFIX) +app.include_router(admin.router, prefix=API_V1_PREFIX) diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..902300a --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,75 @@ +"""SQLAlchemy models. + +Import every model module here so `Base.metadata` is fully populated for +Alembic autogeneration and `Base.metadata.create_all()` (used only in +tests) regardless of which module happens to trigger the import first. +""" + +from app.models.artwork import Artwork +from app.models.base import Base +from app.models.collection import Collection, CollectionMovie, CollectionRelation, CollectionSeries +from app.models.company import Company, MovieCompany, SeriesCompany +from app.models.credit import Credit +from app.models.external_id import ExternalID +from app.models.genre import Genre, MovieGenre, SeriesGenre +from app.models.keyword import Keyword, MovieKeyword, SeriesKeyword +from app.models.language_country import Certification, Country, Language +from app.models.movie import ( + Movie, + MovieAlternateTitle, + MovieProductionCountry, + MovieRecommendation, + MovieSpokenLanguage, + ReleaseDate, +) +from app.models.person import Person +from app.models.provider import IngestionJob, Provider, ProviderRecord +from app.models.tv import ( + Episode, + Season, + Series, + SeriesAlternateTitle, + SeriesProductionCountry, + SeriesSpokenLanguage, +) +from app.models.video import Video + +__all__ = [ + "Base", + "Artwork", + "Collection", + "CollectionMovie", + "CollectionSeries", + "CollectionRelation", + "Company", + "MovieCompany", + "SeriesCompany", + "Credit", + "ExternalID", + "Genre", + "MovieGenre", + "SeriesGenre", + "Keyword", + "MovieKeyword", + "SeriesKeyword", + "Language", + "Country", + "Certification", + "Movie", + "MovieAlternateTitle", + "MovieSpokenLanguage", + "MovieProductionCountry", + "ReleaseDate", + "MovieRecommendation", + "Person", + "Provider", + "ProviderRecord", + "IngestionJob", + "Series", + "SeriesAlternateTitle", + "SeriesSpokenLanguage", + "SeriesProductionCountry", + "Season", + "Episode", + "Video", +] diff --git a/app/models/artwork.py b/app/models/artwork.py new file mode 100644 index 0000000..f05d666 --- /dev/null +++ b/app/models/artwork.py @@ -0,0 +1,52 @@ +"""Artwork as a first-class entity (section 10) — never a `poster_url` +column on media tables, so provenance/licensing can be tracked per asset.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import Boolean, Float, ForeignKey, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import ArtworkType +from app.models.provider import Provider + + +class Artwork(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "artwork" + __table_args__ = ( + Index("ix_artwork_entity", "entity_type", "entity_id", "artwork_type"), + ) + + # Polymorphic target: "movie", "series", "season", "episode", "person", + # "company", "collection". + entity_type: Mapped[str] = mapped_column(String(30), nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(nullable=False) + + artwork_type: Mapped[ArtworkType] = mapped_column(nullable=False) + language_code: Mapped[str | None] = mapped_column( + ForeignKey("languages.code"), nullable=True + ) + + width: Mapped[int | None] = mapped_column(Integer, nullable=True) + height: Mapped[int | None] = mapped_column(Integer, nullable=True) + aspect_ratio: Mapped[float | None] = mapped_column(Float, nullable=True) + vote_average: Mapped[float | None] = mapped_column(Float, nullable=True) + is_primary: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + provider_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("providers.id", ondelete="CASCADE"), nullable=False + ) + provider_asset_id: Mapped[str | None] = mapped_column(String(200), nullable=True) + original_url: Mapped[str] = mapped_column(String(1000), nullable=False) + local_path: Mapped[str | None] = mapped_column(String(1000), nullable=True) + checksum: Mapped[str | None] = mapped_column(String(128), nullable=True) + + attribution: Mapped[str | None] = mapped_column(Text, nullable=True) + license: Mapped[str | None] = mapped_column(Text, nullable=True) + + provider: Mapped[Provider] = relationship() + + +__all__ = ["Artwork"] diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 0000000..e0a5e23 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,80 @@ +"""Declarative base and shared mixins for all Aurora Core models.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum, MetaData, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from uuid6 import uuid7 + +from app.models.enums import ( + ArtworkType, + CompanyType, + IngestionJobStatus, + MatchConfidence, + MovieStatus, + SeriesStatus, + VideoType, +) + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +def _enum_column(enum_cls: type) -> Enum: + """Store our str-Enum members by `.value` (lowercase snake_case), not + `.name`, which is what SQLAlchemy's Enum type does by default. Without + this, the DB would store e.g. "RELEASED" while every part of the + codebase (schemas, ingestion, API filters) works with "released".""" + + return Enum( + enum_cls, + values_callable=lambda obj: [e.value for e in obj], + name=f"{enum_cls.__name__.lower()}_enum", + ) + + +class Base(DeclarativeBase): + """Base class for all ORM models. Naming convention keeps Alembic's + autogenerated constraint names stable across databases.""" + + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + type_annotation_map = { + MovieStatus: _enum_column(MovieStatus), + SeriesStatus: _enum_column(SeriesStatus), + ArtworkType: _enum_column(ArtworkType), + VideoType: _enum_column(VideoType), + CompanyType: _enum_column(CompanyType), + IngestionJobStatus: _enum_column(IngestionJobStatus), + MatchConfidence: _enum_column(MatchConfidence), + } + + +class UUIDPrimaryKeyMixin: + """UUIDv7 primary key. Time-ordered so it stays btree-friendly and can + double as a creation-order cursor, unlike a random UUIDv4.""" + + id: Mapped[uuid.UUID] = mapped_column( + primary_key=True, + default=lambda: uuid.UUID(str(uuid7())), + ) + + +class TimestampMixin: + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) diff --git a/app/models/collection.py b/app/models/collection.py new file mode 100644 index 0000000..09c7c57 --- /dev/null +++ b/app/models/collection.py @@ -0,0 +1,59 @@ +"""Collections/franchises as first-class entities (e.g. "Alien", "MCU").""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class Collection(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "collections" + + name: Mapped[str] = mapped_column(String(300), nullable=False, index=True) + overview: Mapped[str | None] = mapped_column(Text, nullable=True) + slug: Mapped[str] = mapped_column(String(350), unique=True, nullable=False) + + +class CollectionMovie(Base): + __tablename__ = "collection_movies" + + collection_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("collections.id", ondelete="CASCADE"), primary_key=True + ) + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + sort_order: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class CollectionSeries(Base): + __tablename__ = "collection_series" + + collection_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("collections.id", ondelete="CASCADE"), primary_key=True + ) + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + sort_order: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class CollectionRelation(Base): + """Relationships between collections (e.g. shared universe).""" + + __tablename__ = "collection_relations" + + collection_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("collections.id", ondelete="CASCADE"), primary_key=True + ) + related_collection_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("collections.id", ondelete="CASCADE"), primary_key=True + ) + relation_type: Mapped[str] = mapped_column(String(50), default="related", nullable=False) + + +__all__ = ["Collection", "CollectionMovie", "CollectionSeries", "CollectionRelation"] diff --git a/app/models/company.py b/app/models/company.py new file mode 100644 index 0000000..2dd6737 --- /dev/null +++ b/app/models/company.py @@ -0,0 +1,62 @@ +"""Production companies, studios, networks, and distributors. + +Modeled as a single `Company` table distinguished by `company_type` per the +spec (section 8), rather than a separate Network entity — a TV network and a +studio share the same shape (name, logo, country, parent company).""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import CompanyType + + +class Company(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "companies" + + name: Mapped[str] = mapped_column(String(300), nullable=False, index=True) + company_type: Mapped[CompanyType] = mapped_column( + default=CompanyType.PRODUCTION_COMPANY, nullable=False + ) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + country_code: Mapped[str | None] = mapped_column( + ForeignKey("countries.code"), nullable=True + ) + headquarters: Mapped[str | None] = mapped_column(String(300), nullable=True) + homepage: Mapped[str | None] = mapped_column(String(500), nullable=True) + parent_company_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("companies.id", ondelete="SET NULL"), nullable=True + ) + + parent_company: Mapped[Company | None] = relationship(remote_side="Company.id") + + +class MovieCompany(Base): + __tablename__ = "movie_companies" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), primary_key=True + ) + role: Mapped[str | None] = mapped_column(String(50), nullable=True) + + +class SeriesCompany(Base): + __tablename__ = "series_companies" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + company_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("companies.id", ondelete="CASCADE"), primary_key=True + ) + role: Mapped[str | None] = mapped_column(String(50), nullable=True) + + +__all__ = ["Company", "MovieCompany", "SeriesCompany"] diff --git a/app/models/credit.py b/app/models/credit.py new file mode 100644 index 0000000..a67eb4d --- /dev/null +++ b/app/models/credit.py @@ -0,0 +1,50 @@ +"""Flexible, normalized credit system (section 7). + +A `Credit` links a `Person` to any media entity (movie, series, episode) +with a department/job/character, instead of duplicating cast/crew as text +blobs on the media tables. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.person import Person +from app.models.provider import Provider + + +class Credit(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "credits" + __table_args__ = (Index("ix_credits_entity", "entity_type", "entity_id"),) + + person_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("people.id", ondelete="CASCADE"), nullable=False + ) + + # Polymorphic target: "movie", "series", "episode". + entity_type: Mapped[str] = mapped_column(String(30), nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(nullable=False) + + # "Acting", "Directing", "Writing", "Production", "Camera", "Sound", + # "Editing", "Art", "Costume & Make-Up", "Crew", "Creator", ... + department: Mapped[str] = mapped_column(String(100), nullable=False) + job: Mapped[str | None] = mapped_column(String(150), nullable=True) + character: Mapped[str | None] = mapped_column(String(300), nullable=True) + + billing_order: Mapped[int | None] = mapped_column(Integer, nullable=True) + is_guest: Mapped[bool] = mapped_column(default=False, nullable=False) + episode_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + provider_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("providers.id", ondelete="SET NULL"), nullable=True + ) + + person: Mapped[Person] = relationship() + provider: Mapped[Provider | None] = relationship() + + +__all__ = ["Credit"] diff --git a/app/models/enums.py b/app/models/enums.py new file mode 100644 index 0000000..899bf8a --- /dev/null +++ b/app/models/enums.py @@ -0,0 +1,84 @@ +"""Shared enumerations used across Aurora Core models.""" + +from __future__ import annotations + +import enum + + +class MovieStatus(enum.StrEnum): + RUMORED = "rumored" + PLANNED = "planned" + IN_PRODUCTION = "in_production" + POST_PRODUCTION = "post_production" + RELEASED = "released" + CANCELED = "canceled" + + +class SeriesStatus(enum.StrEnum): + PLANNED = "planned" + PILOT = "pilot" + IN_PRODUCTION = "in_production" + RETURNING_SERIES = "returning_series" + ENDED = "ended" + CANCELED = "canceled" + + +class ArtworkType(enum.StrEnum): + POSTER = "poster" + BACKDROP = "backdrop" + LOGO = "logo" + PROFILE = "profile" + STILL = "still" + BANNER = "banner" + THUMBNAIL = "thumbnail" + + +class VideoType(enum.StrEnum): + TRAILER = "trailer" + TEASER = "teaser" + CLIP = "clip" + FEATURETTE = "featurette" + BEHIND_THE_SCENES = "behind_the_scenes" + INTERVIEW = "interview" + OTHER = "other" + + +class CompanyType(enum.StrEnum): + PRODUCTION_COMPANY = "production_company" + NETWORK = "network" + STUDIO = "studio" + DISTRIBUTOR = "distributor" + + +class EntityType(enum.StrEnum): + """Kinds of entity that polymorphic tables (credits, artwork, videos, + external IDs, release dates, provider records) can attach to.""" + + MOVIE = "movie" + SERIES = "series" + SEASON = "season" + EPISODE = "episode" + PERSON = "person" + COMPANY = "company" + COLLECTION = "collection" + + +class IngestionJobStatus(enum.StrEnum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + RETRYING = "retrying" + + +class MatchConfidence(enum.StrEnum): + """How confident the entity matcher was when linking a provider record + to an Aurora entity. Anything below HIGH should be reviewable rather + than silently merged — see docs/architecture.md.""" + + EXACT = "exact" + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + AMBIGUOUS = "ambiguous" + NEW = "new" diff --git a/app/models/external_id.py b/app/models/external_id.py new file mode 100644 index 0000000..bfa5966 --- /dev/null +++ b/app/models/external_id.py @@ -0,0 +1,45 @@ +"""Generalized external identifier system. + +Links an Aurora entity (any type) to its identifier in an external provider +(TMDb, IMDb, TVDb, Wikidata, ...). External IDs are references, never +primary keys — see app/core/ids.py and docs/architecture.md. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.provider import Provider + + +class ExternalID(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "external_ids" + __table_args__ = ( + # A provider's ID for a given entity type must be unique (one Aurora + # entity per provider-native ID). + UniqueConstraint( + "provider_id", "entity_type", "external_id", name="uq_external_id_source" + ), + # An Aurora entity has at most one mapping per provider. + UniqueConstraint( + "entity_type", "entity_id", "provider_id", name="uq_external_id_entity_provider" + ), + ) + + entity_type: Mapped[str] = mapped_column(String(30), nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(nullable=False) + + provider_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("providers.id", ondelete="CASCADE"), nullable=False + ) + external_id: Mapped[str] = mapped_column(String(200), nullable=False) + url: Mapped[str | None] = mapped_column(String(500), nullable=True) + + provider: Mapped[Provider] = relationship() + + +__all__ = ["ExternalID"] diff --git a/app/models/genre.py b/app/models/genre.py new file mode 100644 index 0000000..f1207bf --- /dev/null +++ b/app/models/genre.py @@ -0,0 +1,42 @@ +"""Genres and their movie/series associations.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, UUIDPrimaryKeyMixin + + +class Genre(Base, UUIDPrimaryKeyMixin): + __tablename__ = "genres" + + name: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) + slug: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) + + +class MovieGenre(Base): + __tablename__ = "movie_genres" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + genre_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("genres.id", ondelete="CASCADE"), primary_key=True + ) + + +class SeriesGenre(Base): + __tablename__ = "series_genres" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + genre_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("genres.id", ondelete="CASCADE"), primary_key=True + ) + + +__all__ = ["Genre", "MovieGenre", "SeriesGenre"] diff --git a/app/models/keyword.py b/app/models/keyword.py new file mode 100644 index 0000000..a6f0934 --- /dev/null +++ b/app/models/keyword.py @@ -0,0 +1,42 @@ +"""Keywords/tags and their movie/series associations.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, UUIDPrimaryKeyMixin + + +class Keyword(Base, UUIDPrimaryKeyMixin): + __tablename__ = "keywords" + + name: Mapped[str] = mapped_column(String(150), unique=True, nullable=False) + slug: Mapped[str] = mapped_column(String(150), unique=True, nullable=False) + + +class MovieKeyword(Base): + __tablename__ = "movie_keywords" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + keyword_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("keywords.id", ondelete="CASCADE"), primary_key=True + ) + + +class SeriesKeyword(Base): + __tablename__ = "series_keywords" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + keyword_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("keywords.id", ondelete="CASCADE"), primary_key=True + ) + + +__all__ = ["Keyword", "MovieKeyword", "SeriesKeyword"] diff --git a/app/models/language_country.py b/app/models/language_country.py new file mode 100644 index 0000000..f892284 --- /dev/null +++ b/app/models/language_country.py @@ -0,0 +1,46 @@ +"""Reference data: languages, countries, certifications.""" + +from __future__ import annotations + +from sqlalchemy import ForeignKey, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + + +class Language(Base): + """ISO 639-1 language codes (e.g. 'en', 'ja').""" + + __tablename__ = "languages" + + code: Mapped[str] = mapped_column(String(10), primary_key=True) + english_name: Mapped[str] = mapped_column(String(200), nullable=False) + native_name: Mapped[str | None] = mapped_column(String(200), nullable=True) + + +class Country(Base): + """ISO 3166-1 alpha-2 country codes (e.g. 'US', 'GB').""" + + __tablename__ = "countries" + + code: Mapped[str] = mapped_column(String(2), primary_key=True) + name: Mapped[str] = mapped_column(String(200), nullable=False) + + +class Certification(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """Content rating within a country's rating system, e.g. US/MPA/PG-13.""" + + __tablename__ = "certifications" + __table_args__ = ( + UniqueConstraint("country_code", "system", "rating", name="uq_certification_scope"), + ) + + country_code: Mapped[str] = mapped_column(ForeignKey("countries.code"), nullable=False) + system: Mapped[str] = mapped_column(String(50), nullable=False) # e.g. "MPA", "BBFC" + rating: Mapped[str] = mapped_column(String(20), nullable=False) # e.g. "PG-13" + meaning: Mapped[str | None] = mapped_column(Text, nullable=True) + + country: Mapped[Country] = relationship() + + +__all__ = ["Language", "Country", "Certification"] diff --git a/app/models/movie.py b/app/models/movie.py new file mode 100644 index 0000000..410d567 --- /dev/null +++ b/app/models/movie.py @@ -0,0 +1,127 @@ +"""Movie entity and its directly-owned satellite tables (section 4).""" + +from __future__ import annotations + +import uuid +from datetime import date + +from sqlalchemy import BigInteger, Date, Float, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.collection import Collection +from app.models.enums import MovieStatus + + +class Movie(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "movies" + + title: Mapped[str] = mapped_column(String(500), nullable=False, index=True) + original_title: Mapped[str | None] = mapped_column(String(500), nullable=True) + # Title with leading articles ("The", "A") stripped, for stable sorting. + sort_title: Mapped[str] = mapped_column(String(500), nullable=False, index=True) + + overview: Mapped[str | None] = mapped_column(Text, nullable=True) + tagline: Mapped[str | None] = mapped_column(Text, nullable=True) + + status: Mapped[MovieStatus] = mapped_column(default=MovieStatus.RELEASED, nullable=False) + original_language_code: Mapped[str | None] = mapped_column( + ForeignKey("languages.code"), nullable=True + ) + primary_release_date: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) + runtime_minutes: Mapped[int | None] = mapped_column(Integer, nullable=True) + + budget: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + revenue: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + + popularity: Mapped[float | None] = mapped_column(Float, nullable=True) + vote_average: Mapped[float | None] = mapped_column(Float, nullable=True) + vote_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + adult: Mapped[bool] = mapped_column(default=False, nullable=False) + + collection_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("collections.id", ondelete="SET NULL"), nullable=True + ) + + # Best-known authoritative source for this record at creation time. Full + # field-level provenance lives in provider_records; this is a coarse + # pointer for quick display. See docs/architecture.md. + source_provider_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("providers.id", ondelete="SET NULL"), nullable=True + ) + + collection: Mapped[Collection | None] = relationship() + + +class MovieAlternateTitle(Base, UUIDPrimaryKeyMixin): + __tablename__ = "movie_alternate_titles" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), nullable=False, index=True + ) + title: Mapped[str] = mapped_column(String(500), nullable=False) + country_code: Mapped[str | None] = mapped_column(ForeignKey("countries.code"), nullable=True) + type: Mapped[str | None] = mapped_column(String(50), nullable=True) # e.g. "working_title" + + +class MovieSpokenLanguage(Base): + __tablename__ = "movie_spoken_languages" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + language_code: Mapped[str] = mapped_column( + ForeignKey("languages.code"), primary_key=True + ) + + +class MovieProductionCountry(Base): + __tablename__ = "movie_production_countries" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + country_code: Mapped[str] = mapped_column(ForeignKey("countries.code"), primary_key=True) + + +class ReleaseDate(Base, UUIDPrimaryKeyMixin): + """Per-country/per-type release dates and certifications for a movie.""" + + __tablename__ = "release_dates" + + entity_type: Mapped[str] = mapped_column(String(30), default="movie", nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(nullable=False, index=True) + + country_code: Mapped[str] = mapped_column(ForeignKey("countries.code"), nullable=False) + release_date: Mapped[date] = mapped_column(Date, nullable=False) + # "theatrical", "digital", "physical", "premiere", "tv" + release_type: Mapped[str | None] = mapped_column(String(50), nullable=True) + certification_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("certifications.id", ondelete="SET NULL"), nullable=True + ) + note: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class MovieRecommendation(Base): + """Recommendations/similar-title edges between movies.""" + + __tablename__ = "movie_recommendations" + + movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + recommended_movie_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ) + rank: Mapped[int | None] = mapped_column(Integer, nullable=True) + source: Mapped[str | None] = mapped_column(String(50), nullable=True) + + +__all__ = [ + "Movie", + "MovieAlternateTitle", + "MovieSpokenLanguage", + "MovieProductionCountry", + "ReleaseDate", + "MovieRecommendation", +] diff --git a/app/models/person.py b/app/models/person.py new file mode 100644 index 0000000..cc3e469 --- /dev/null +++ b/app/models/person.py @@ -0,0 +1,32 @@ +"""People — first-class Aurora entities (section 6).""" + +from __future__ import annotations + +from datetime import date + +from sqlalchemy import JSON, Date, Float, String, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin + +JSONVariant = JSONB().with_variant(JSON(), "sqlite") + + +class Person(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "people" + + name: Mapped[str] = mapped_column(String(300), nullable=False, index=True) + aliases: Mapped[list[str]] = mapped_column(JSONVariant, default=list) + biography: Mapped[str | None] = mapped_column(Text, nullable=True) + birthday: Mapped[date | None] = mapped_column(Date, nullable=True) + deathday: Mapped[date | None] = mapped_column(Date, nullable=True) + birthplace: Mapped[str | None] = mapped_column(String(300), nullable=True) + # Free-text, only populated when a source legitimately provides it — + # never inferred. + gender: Mapped[str | None] = mapped_column(String(30), nullable=True) + known_for_department: Mapped[str | None] = mapped_column(String(100), nullable=True) + popularity: Mapped[float | None] = mapped_column(Float, nullable=True) + + +__all__ = ["Person"] diff --git a/app/models/provider.py b/app/models/provider.py new file mode 100644 index 0000000..f8ea1cf --- /dev/null +++ b/app/models/provider.py @@ -0,0 +1,108 @@ +"""Provider abstraction and provenance tables. + +`Provider` identifies a metadata source (TMDb, IMDb dataset export, ...). +`ProviderRecord` is the provenance ledger: one row per (provider, entity +type, provider-native ID) linking to the Aurora entity it was matched to, +retaining the raw payload and match confidence so ambiguous matches stay +reviewable instead of being silently merged (see docs/architecture.md). +`IngestionJob` tracks async ingestion work so jobs are observable, retryable, +and protected against duplicate processing via `idempotency_key`. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import IngestionJobStatus, MatchConfidence + +JSONVariant = JSONB().with_variant(JSON(), "sqlite") + + +class Provider(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """A metadata data source. Providers are replaceable adapters, never + Aurora's identity system — see app/providers/base.py.""" + + __tablename__ = "providers" + + slug: Mapped[str] = mapped_column(String(50), unique=True, nullable=False) # "tmdb", "imdb" + name: Mapped[str] = mapped_column(String(200), nullable=False) + base_url: Mapped[str | None] = mapped_column(String(500), nullable=True) + is_active: Mapped[bool] = mapped_column(default=True, nullable=False) + config: Mapped[dict] = mapped_column(JSONVariant, default=dict) + + +class ProviderRecord(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """Provenance ledger row: this provider's native record, and what Aurora + entity (if any) it has been matched to.""" + + __tablename__ = "provider_records" + __table_args__ = ( + UniqueConstraint( + "provider_id", "entity_type", "external_id", name="uq_provider_record_source" + ), + ) + + provider_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("providers.id", ondelete="CASCADE"), nullable=False + ) + entity_type: Mapped[str] = mapped_column(String(30), nullable=False) + external_id: Mapped[str] = mapped_column(String(200), nullable=False) + + # Polymorphic reference to the Aurora entity this record resolved to. + # No DB-level FK (the target table varies by entity_type); integrity is + # enforced by the entity matcher / repositories, not the schema. + aurora_entity_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True) + + raw_data: Mapped[dict] = mapped_column(JSONVariant, default=dict) + source_version: Mapped[str | None] = mapped_column(String(100), nullable=True) + match_confidence: Mapped[MatchConfidence] = mapped_column( + default=MatchConfidence.NEW, nullable=False + ) + match_notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + ingestion_job_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("ingestion_jobs.id", ondelete="SET NULL"), nullable=True + ) + last_synced_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + provider: Mapped[Provider] = relationship() + + +class IngestionJob(Base, UUIDPrimaryKeyMixin, TimestampMixin): + """A unit of asynchronous ingestion work processed by workers.""" + + __tablename__ = "ingestion_jobs" + __table_args__ = (UniqueConstraint("idempotency_key", name="uq_ingestion_job_idempotency"),) + + provider_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("providers.id", ondelete="CASCADE"), nullable=False + ) + job_type: Mapped[str] = mapped_column(String(50), nullable=False) # "movie_sync", "full_sync" + entity_type: Mapped[str | None] = mapped_column(String(30), nullable=True) + external_id: Mapped[str | None] = mapped_column(String(200), nullable=True) + + idempotency_key: Mapped[str] = mapped_column(String(300), nullable=False) + status: Mapped[IngestionJobStatus] = mapped_column( + default=IngestionJobStatus.PENDING, nullable=False + ) + attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + max_attempts: Mapped[int] = mapped_column(Integer, default=5, nullable=False) + + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + error: Mapped[str | None] = mapped_column(Text, nullable=True) + result_aurora_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True) + + provider: Mapped[Provider] = relationship() + + +__all__ = ["Provider", "ProviderRecord", "IngestionJob"] diff --git a/app/models/tv.py b/app/models/tv.py new file mode 100644 index 0000000..37d2c7f --- /dev/null +++ b/app/models/tv.py @@ -0,0 +1,118 @@ +"""Television model: Series -> Seasons -> Episodes (section 5).""" + +from __future__ import annotations + +import uuid +from datetime import date + +from sqlalchemy import Date, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import SeriesStatus + + +class Series(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "series" + + title: Mapped[str] = mapped_column(String(500), nullable=False, index=True) + original_title: Mapped[str | None] = mapped_column(String(500), nullable=True) + sort_title: Mapped[str] = mapped_column(String(500), nullable=False, index=True) + + overview: Mapped[str | None] = mapped_column(Text, nullable=True) + tagline: Mapped[str | None] = mapped_column(Text, nullable=True) + + status: Mapped[SeriesStatus] = mapped_column( + default=SeriesStatus.RETURNING_SERIES, nullable=False + ) + original_language_code: Mapped[str | None] = mapped_column( + ForeignKey("languages.code"), nullable=True + ) + first_air_date: Mapped[date | None] = mapped_column(Date, nullable=True, index=True) + last_air_date: Mapped[date | None] = mapped_column(Date, nullable=True) + + number_of_seasons: Mapped[int | None] = mapped_column(Integer, nullable=True) + number_of_episodes: Mapped[int | None] = mapped_column(Integer, nullable=True) + + popularity: Mapped[float | None] = mapped_column(nullable=True) + vote_average: Mapped[float | None] = mapped_column(nullable=True) + vote_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + source_provider_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("providers.id", ondelete="SET NULL"), nullable=True + ) + + +class SeriesAlternateTitle(Base, UUIDPrimaryKeyMixin): + __tablename__ = "series_alternate_titles" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), nullable=False, index=True + ) + title: Mapped[str] = mapped_column(String(500), nullable=False) + country_code: Mapped[str | None] = mapped_column(ForeignKey("countries.code"), nullable=True) + type: Mapped[str | None] = mapped_column(String(50), nullable=True) + + +class SeriesSpokenLanguage(Base): + __tablename__ = "series_spoken_languages" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + language_code: Mapped[str] = mapped_column(ForeignKey("languages.code"), primary_key=True) + + +class SeriesProductionCountry(Base): + __tablename__ = "series_production_countries" + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ) + country_code: Mapped[str] = mapped_column(ForeignKey("countries.code"), primary_key=True) + + +class Season(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "seasons" + __table_args__ = (UniqueConstraint("series_id", "season_number", name="uq_season_number"),) + + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), nullable=False, index=True + ) + season_number: Mapped[int] = mapped_column(Integer, nullable=False) + title: Mapped[str | None] = mapped_column(String(500), nullable=True) + overview: Mapped[str | None] = mapped_column(Text, nullable=True) + air_date: Mapped[date | None] = mapped_column(Date, nullable=True) + episode_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class Episode(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "episodes" + __table_args__ = ( + UniqueConstraint("season_id", "episode_number", name="uq_episode_number"), + ) + + season_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("seasons.id", ondelete="CASCADE"), nullable=False, index=True + ) + # Denormalized for direct filtering/joins without hopping through seasons. + series_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("series.id", ondelete="CASCADE"), nullable=False, index=True + ) + season_number: Mapped[int] = mapped_column(Integer, nullable=False) + episode_number: Mapped[int] = mapped_column(Integer, nullable=False) + + title: Mapped[str | None] = mapped_column(String(500), nullable=True) + overview: Mapped[str | None] = mapped_column(Text, nullable=True) + air_date: Mapped[date | None] = mapped_column(Date, nullable=True) + runtime_minutes: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +__all__ = [ + "Series", + "SeriesAlternateTitle", + "SeriesSpokenLanguage", + "SeriesProductionCountry", + "Season", + "Episode", +] diff --git a/app/models/video.py b/app/models/video.py new file mode 100644 index 0000000..ac18e05 --- /dev/null +++ b/app/models/video.py @@ -0,0 +1,50 @@ +"""Videos/trailers as normalized entities (section 11). + +Aurora never downloads copyrighted video media — only stores the +provider's reference/URL and metadata about it. +""" + +from __future__ import annotations + +import uuid +from datetime import date + +from sqlalchemy import Boolean, Date, ForeignKey, Index, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin +from app.models.enums import VideoType +from app.models.provider import Provider + + +class Video(Base, UUIDPrimaryKeyMixin, TimestampMixin): + __tablename__ = "videos" + __table_args__ = ( + Index("ix_videos_entity", "entity_type", "entity_id"), + UniqueConstraint("provider_id", "provider_video_id", name="uq_video_provider_source"), + ) + + # Polymorphic target: "movie", "series", "season", "episode". + entity_type: Mapped[str] = mapped_column(String(30), nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(nullable=False) + + video_type: Mapped[VideoType] = mapped_column(nullable=False) + + provider_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("providers.id", ondelete="CASCADE"), nullable=False + ) + provider_video_id: Mapped[str] = mapped_column(String(200), nullable=False) + site: Mapped[str | None] = mapped_column(String(100), nullable=True) # "YouTube", "Vimeo" + url: Mapped[str] = mapped_column(String(1000), nullable=False) + + language_code: Mapped[str | None] = mapped_column( + ForeignKey("languages.code"), nullable=True + ) + country_code: Mapped[str | None] = mapped_column(ForeignKey("countries.code"), nullable=True) + is_official: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + published_at: Mapped[date | None] = mapped_column(Date, nullable=True) + + provider: Mapped[Provider] = relationship() + + +__all__ = ["Video"] diff --git a/app/providers/__init__.py b/app/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/providers/base.py b/app/providers/base.py new file mode 100644 index 0000000..f11a559 --- /dev/null +++ b/app/providers/base.py @@ -0,0 +1,58 @@ +"""Provider abstraction (section 13). + +A `MetadataProvider` fetches **raw**, unnormalized data from one external +source. It never writes to the database and never returns Aurora entities — +that's the normalizer/matcher/ingestion pipeline's job (see +`app/services/metadata/` and `app/services/ingestion/`). This keeps adapters +replaceable: swapping TMDb for another provider means writing a new class +here, not touching ingestion logic. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + + +class ProviderError(Exception): + """Raised for any provider-side failure (auth, not-found, rate limit, + network). Ingestion jobs catch this and record it, they don't crash.""" + + +class ProviderNotConfiguredError(ProviderError): + """Raised when a required credential (e.g. TMDB_API_KEY) is missing.""" + + +class ProviderNotFoundError(ProviderError): + """Raised when the provider has no record for the given external ID.""" + + +class MetadataProvider(ABC): + slug: str + name: str + + @abstractmethod + async def get_movie(self, external_id: str) -> dict[str, Any]: + """Fetch raw movie data, including credits/images/videos/keywords/ + alternate titles/external IDs/release dates where the provider + supports fetching them in one call.""" + + @abstractmethod + async def get_series(self, external_id: str) -> dict[str, Any]: + """Fetch raw series data (without season/episode bodies).""" + + @abstractmethod + async def get_season(self, series_external_id: str, season_number: int) -> dict[str, Any]: + """Fetch raw season data, including its episodes.""" + + @abstractmethod + async def get_person(self, external_id: str) -> dict[str, Any]: + """Fetch raw person data, including external IDs and images.""" + + @abstractmethod + async def search_movie(self, query: str) -> list[dict[str, Any]]: + """Search this provider's catalog for movies matching `query`. Used + to seed ingestion, not exposed directly to API clients.""" + + async def aclose(self) -> None: + """Release any held resources (HTTP clients, ...). Optional.""" diff --git a/app/providers/tmdb.py b/app/providers/tmdb.py new file mode 100644 index 0000000..a5f4165 --- /dev/null +++ b/app/providers/tmdb.py @@ -0,0 +1,89 @@ +"""TMDb metadata provider adapter. + +Requires the `TMDB_API_KEY` environment variable (a TMDb v3 API key — +https://www.themoviedb.org/settings/api). If it is not configured, every +method raises `ProviderNotConfiguredError` rather than returning fake data. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential + +from app.core.config import get_settings +from app.providers.base import ( + MetadataProvider, + ProviderError, + ProviderNotConfiguredError, + ProviderNotFoundError, +) + +_MOVIE_APPEND = "credits,images,videos,keywords,alternative_titles,external_ids,release_dates,recommendations" +_SERIES_APPEND = "credits,images,videos,keywords,alternative_titles,external_ids,content_ratings" +_PERSON_APPEND = "external_ids,images,combined_credits" + + +class TMDbProvider(MetadataProvider): + slug = "tmdb" + name = "The Movie Database (TMDb)" + base_url = "https://api.themoviedb.org/3" + + def __init__(self, api_key: str | None = None, client: httpx.AsyncClient | None = None) -> None: + settings = get_settings() + self._api_key = api_key if api_key is not None else settings.TMDB_API_KEY + self._client = client or httpx.AsyncClient(base_url=self.base_url, timeout=15.0) + self._owns_client = client is None + + def _require_api_key(self) -> None: + if not self._api_key: + raise ProviderNotConfiguredError( + "TMDB_API_KEY is not set. Configure it in the environment to use the TMDb " + "provider — see .env.example." + ) + + @retry( + reraise=True, + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=0.5, max=5), + retry=retry_if_exception_type(httpx.TransportError), + ) + async def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + self._require_api_key() + query = {"api_key": self._api_key, **(params or {})} + try: + response = await self._client.get(path, params=query) + except httpx.TransportError as exc: + raise ProviderError(f"TMDb request failed: {exc}") from exc + + if response.status_code == 404: + raise ProviderNotFoundError(f"TMDb has no record at {path}") + if response.status_code == 401: + raise ProviderNotConfiguredError("TMDb rejected the configured TMDB_API_KEY.") + if response.status_code >= 400: + raise ProviderError(f"TMDb error {response.status_code} at {path}: {response.text[:300]}") + return response.json() + + async def get_movie(self, external_id: str) -> dict[str, Any]: + return await self._get(f"/movie/{external_id}", {"append_to_response": _MOVIE_APPEND}) + + async def get_series(self, external_id: str) -> dict[str, Any]: + return await self._get(f"/tv/{external_id}", {"append_to_response": _SERIES_APPEND}) + + async def get_season(self, series_external_id: str, season_number: int) -> dict[str, Any]: + return await self._get( + f"/tv/{series_external_id}/season/{season_number}", + {"append_to_response": "credits,images,external_ids"}, + ) + + async def get_person(self, external_id: str) -> dict[str, Any]: + return await self._get(f"/person/{external_id}", {"append_to_response": _PERSON_APPEND}) + + async def search_movie(self, query: str) -> list[dict[str, Any]]: + data = await self._get("/search/movie", {"query": query}) + return data.get("results", []) + + async def aclose(self) -> None: + if self._owns_client: + await self._client.aclose() diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/repositories/collection_repository.py b/app/repositories/collection_repository.py new file mode 100644 index 0000000..d1d1a53 --- /dev/null +++ b/app/repositories/collection_repository.py @@ -0,0 +1,39 @@ +"""Data access for Collection/franchise.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.collection import Collection, CollectionMovie, CollectionSeries +from app.models.movie import Movie +from app.models.tv import Series + + +async def get_collection_by_id(session: AsyncSession, collection_id: uuid.UUID) -> Collection | None: + result = await session.execute(select(Collection).where(Collection.id == collection_id)) + return result.scalar_one_or_none() + + +async def get_collection_movies(session: AsyncSession, collection_id: uuid.UUID) -> list[Movie]: + stmt = ( + select(Movie) + .join(CollectionMovie, CollectionMovie.movie_id == Movie.id) + .where(CollectionMovie.collection_id == collection_id) + .order_by(CollectionMovie.sort_order.asc().nulls_last(), Movie.primary_release_date.asc()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_collection_series(session: AsyncSession, collection_id: uuid.UUID) -> list[Series]: + stmt = ( + select(Series) + .join(CollectionSeries, CollectionSeries.series_id == Series.id) + .where(CollectionSeries.collection_id == collection_id) + .order_by(CollectionSeries.sort_order.asc().nulls_last()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) diff --git a/app/repositories/company_repository.py b/app/repositories/company_repository.py new file mode 100644 index 0000000..ee55e04 --- /dev/null +++ b/app/repositories/company_repository.py @@ -0,0 +1,15 @@ +"""Data access for Company.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.company import Company + + +async def get_company_by_id(session: AsyncSession, company_id: uuid.UUID) -> Company | None: + result = await session.execute(select(Company).where(Company.id == company_id)) + return result.scalar_one_or_none() diff --git a/app/repositories/movie_repository.py b/app/repositories/movie_repository.py new file mode 100644 index 0000000..4a2779f --- /dev/null +++ b/app/repositories/movie_repository.py @@ -0,0 +1,61 @@ +"""Data access for Movie and its directly-owned satellite tables.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.company import Company, MovieCompany +from app.models.genre import Genre, MovieGenre +from app.models.keyword import Keyword, MovieKeyword +from app.models.movie import Movie + + +async def get_movie_by_id(session: AsyncSession, movie_id: uuid.UUID) -> Movie | None: + stmt = ( + select(Movie) + .options(selectinload(Movie.collection)) + .where(Movie.id == movie_id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def get_movies_by_ids(session: AsyncSession, movie_ids: list[uuid.UUID]) -> list[Movie]: + if not movie_ids: + return [] + stmt = select(Movie).where(Movie.id.in_(movie_ids)) + result = await session.execute(stmt) + by_id = {m.id: m for m in result.scalars().all()} + return [by_id[i] for i in movie_ids if i in by_id] + + +async def get_movie_genres(session: AsyncSession, movie_id: uuid.UUID) -> list[Genre]: + stmt = select(Genre).join(MovieGenre, MovieGenre.genre_id == Genre.id).where( + MovieGenre.movie_id == movie_id + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_movie_keywords(session: AsyncSession, movie_id: uuid.UUID) -> list[Keyword]: + stmt = select(Keyword).join(MovieKeyword, MovieKeyword.keyword_id == Keyword.id).where( + MovieKeyword.movie_id == movie_id + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_movie_companies( + session: AsyncSession, movie_id: uuid.UUID +) -> list[tuple[Company, str | None]]: + stmt = ( + select(Company, MovieCompany.role) + .join(MovieCompany, MovieCompany.company_id == Company.id) + .where(MovieCompany.movie_id == movie_id) + ) + result = await session.execute(stmt) + return [(company, role) for company, role in result.all()] diff --git a/app/repositories/person_repository.py b/app/repositories/person_repository.py new file mode 100644 index 0000000..a1a6180 --- /dev/null +++ b/app/repositories/person_repository.py @@ -0,0 +1,26 @@ +"""Data access for Person.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.credit import Credit +from app.models.person import Person + + +async def get_person_by_id(session: AsyncSession, person_id: uuid.UUID) -> Person | None: + result = await session.execute(select(Person).where(Person.id == person_id)) + return result.scalar_one_or_none() + + +async def get_person_credits(session: AsyncSession, person_id: uuid.UUID) -> list[Credit]: + stmt = ( + select(Credit) + .where(Credit.person_id == person_id) + .order_by(Credit.entity_type.asc()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) diff --git a/app/repositories/polymorphic.py b/app/repositories/polymorphic.py new file mode 100644 index 0000000..a95d5f6 --- /dev/null +++ b/app/repositories/polymorphic.py @@ -0,0 +1,90 @@ +"""Shared data access for the polymorphic satellite tables — credits, +artwork, videos, external IDs — that attach to more than one entity type via +an (entity_type, entity_id) pair rather than a typed foreign key. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.artwork import Artwork +from app.models.credit import Credit +from app.models.enums import ArtworkType +from app.models.external_id import ExternalID +from app.models.provider import Provider +from app.models.video import Video + + +async def get_external_ids( + session: AsyncSession, entity_type: str, entity_id: uuid.UUID +) -> dict[str, str]: + stmt = ( + select(Provider.slug, ExternalID.external_id) + .join(Provider, Provider.id == ExternalID.provider_id) + .where(ExternalID.entity_type == entity_type, ExternalID.entity_id == entity_id) + ) + result = await session.execute(stmt) + return {slug: external_id for slug, external_id in result.all()} + + +async def get_external_ids_bulk( + session: AsyncSession, entity_type: str, entity_ids: list[uuid.UUID] +) -> dict[uuid.UUID, dict[str, str]]: + if not entity_ids: + return {} + stmt = ( + select(ExternalID.entity_id, Provider.slug, ExternalID.external_id) + .join(Provider, Provider.id == ExternalID.provider_id) + .where(ExternalID.entity_type == entity_type, ExternalID.entity_id.in_(entity_ids)) + ) + result = await session.execute(stmt) + out: dict[uuid.UUID, dict[str, str]] = {eid: {} for eid in entity_ids} + for entity_id, slug, external_id in result.all(): + out.setdefault(entity_id, {})[slug] = external_id + return out + + +async def get_artwork( + session: AsyncSession, + entity_type: str, + entity_id: uuid.UUID, + artwork_type: ArtworkType | None = None, +) -> list[Artwork]: + stmt = ( + select(Artwork) + .options(selectinload(Artwork.provider)) + .where(Artwork.entity_type == entity_type, Artwork.entity_id == entity_id) + .order_by(Artwork.is_primary.desc(), Artwork.vote_average.desc().nulls_last()) + ) + if artwork_type is not None: + stmt = stmt.where(Artwork.artwork_type == artwork_type) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_videos(session: AsyncSession, entity_type: str, entity_id: uuid.UUID) -> list[Video]: + stmt = ( + select(Video) + .options(selectinload(Video.provider)) + .where(Video.entity_type == entity_type, Video.entity_id == entity_id) + .order_by(Video.is_official.desc(), Video.published_at.desc().nulls_last()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_credits( + session: AsyncSession, entity_type: str, entity_id: uuid.UUID +) -> list[Credit]: + stmt = ( + select(Credit) + .options(selectinload(Credit.person)) + .where(Credit.entity_type == entity_type, Credit.entity_id == entity_id) + .order_by(Credit.billing_order.asc().nulls_last()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) diff --git a/app/repositories/provider_repository.py b/app/repositories/provider_repository.py new file mode 100644 index 0000000..e6d94a9 --- /dev/null +++ b/app/repositories/provider_repository.py @@ -0,0 +1,145 @@ +"""Data access for Provider, ProviderRecord, and IngestionJob.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import IngestionJobStatus, MatchConfidence +from app.models.provider import IngestionJob, Provider, ProviderRecord + + +async def get_provider_by_slug(session: AsyncSession, slug: str) -> Provider | None: + result = await session.execute(select(Provider).where(Provider.slug == slug)) + return result.scalar_one_or_none() + + +async def get_or_create_provider( + session: AsyncSession, slug: str, name: str, base_url: str | None = None +) -> Provider: + provider = await get_provider_by_slug(session, slug) + if provider is not None: + return provider + provider = Provider(slug=slug, name=name, base_url=base_url) + session.add(provider) + await session.flush() + return provider + + +async def get_provider_record( + session: AsyncSession, provider_id: uuid.UUID, entity_type: str, external_id: str +) -> ProviderRecord | None: + stmt = select(ProviderRecord).where( + ProviderRecord.provider_id == provider_id, + ProviderRecord.entity_type == entity_type, + ProviderRecord.external_id == external_id, + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def upsert_provider_record( + session: AsyncSession, + *, + provider_id: uuid.UUID, + entity_type: str, + external_id: str, + raw_data: dict, + aurora_entity_id: uuid.UUID | None, + match_confidence: MatchConfidence, + source_version: str | None = None, + ingestion_job_id: uuid.UUID | None = None, +) -> ProviderRecord: + record = await get_provider_record(session, provider_id, entity_type, external_id) + now = datetime.now(UTC) + if record is None: + record = ProviderRecord( + provider_id=provider_id, + entity_type=entity_type, + external_id=external_id, + raw_data=raw_data, + aurora_entity_id=aurora_entity_id, + match_confidence=match_confidence, + source_version=source_version, + ingestion_job_id=ingestion_job_id, + last_synced_at=now, + ) + session.add(record) + else: + record.raw_data = raw_data + record.aurora_entity_id = aurora_entity_id + record.match_confidence = match_confidence + record.source_version = source_version + record.ingestion_job_id = ingestion_job_id + record.last_synced_at = now + await session.flush() + return record + + +async def get_ingestion_job_by_idempotency_key( + session: AsyncSession, idempotency_key: str +) -> IngestionJob | None: + stmt = select(IngestionJob).where(IngestionJob.idempotency_key == idempotency_key) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def create_ingestion_job( + session: AsyncSession, + *, + provider_id: uuid.UUID, + job_type: str, + idempotency_key: str, + entity_type: str | None = None, + external_id: str | None = None, +) -> IngestionJob: + existing = await get_ingestion_job_by_idempotency_key(session, idempotency_key) + if existing is not None: + return existing + job = IngestionJob( + provider_id=provider_id, + job_type=job_type, + entity_type=entity_type, + external_id=external_id, + idempotency_key=idempotency_key, + status=IngestionJobStatus.PENDING, + ) + session.add(job) + await session.flush() + return job + + +async def get_ingestion_job(session: AsyncSession, job_id: uuid.UUID) -> IngestionJob | None: + result = await session.execute(select(IngestionJob).where(IngestionJob.id == job_id)) + return result.scalar_one_or_none() + + +async def mark_job_running(session: AsyncSession, job: IngestionJob) -> None: + job.status = IngestionJobStatus.RUNNING + job.attempts += 1 + job.started_at = datetime.now(UTC) + await session.flush() + + +async def mark_job_succeeded( + session: AsyncSession, job: IngestionJob, result_aurora_id: uuid.UUID | None +) -> None: + job.status = IngestionJobStatus.SUCCEEDED + job.finished_at = datetime.now(UTC) + job.result_aurora_id = result_aurora_id + job.error = None + await session.flush() + + +async def mark_job_failed(session: AsyncSession, job: IngestionJob, error: str) -> None: + job.error = error[:4000] + job.finished_at = datetime.now(UTC) + job.status = ( + IngestionJobStatus.RETRYING + if job.attempts < job.max_attempts + else IngestionJobStatus.FAILED + ) + await session.flush() diff --git a/app/repositories/reference_repository.py b/app/repositories/reference_repository.py new file mode 100644 index 0000000..dacec98 --- /dev/null +++ b/app/repositories/reference_repository.py @@ -0,0 +1,270 @@ +"""Get-or-create helpers for shared reference/lookup data used during +ingestion: genres, keywords, languages, countries, companies, collections, +people, and external ID links.""" + +from __future__ import annotations + +import re +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.collection import Collection +from app.models.company import Company +from app.models.enums import CompanyType +from app.models.external_id import ExternalID +from app.models.genre import Genre +from app.models.keyword import Keyword +from app.models.language_country import Country, Language +from app.models.person import Person + + +def slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return slug or "n-a" + + +async def get_or_create_genre(session: AsyncSession, name: str) -> Genre: + slug = slugify(name) + result = await session.execute(select(Genre).where(Genre.slug == slug)) + genre = result.scalar_one_or_none() + if genre is not None: + return genre + genre = Genre(name=name, slug=slug) + session.add(genre) + await session.flush() + return genre + + +async def get_or_create_keyword(session: AsyncSession, name: str) -> Keyword: + slug = slugify(name) + result = await session.execute(select(Keyword).where(Keyword.slug == slug)) + keyword = result.scalar_one_or_none() + if keyword is not None: + return keyword + keyword = Keyword(name=name, slug=slug) + session.add(keyword) + await session.flush() + return keyword + + +async def get_or_create_language(session: AsyncSession, code: str, english_name: str) -> Language: + result = await session.execute(select(Language).where(Language.code == code)) + language = result.scalar_one_or_none() + if language is not None: + return language + language = Language(code=code, english_name=english_name) + session.add(language) + await session.flush() + return language + + +async def get_or_create_country(session: AsyncSession, code: str, name: str) -> Country: + result = await session.execute(select(Country).where(Country.code == code)) + country = result.scalar_one_or_none() + if country is not None: + return country + country = Country(code=code, name=name) + session.add(country) + await session.flush() + return country + + +async def find_company_by_external_id( + session: AsyncSession, provider_id: uuid.UUID, external_id: str +) -> Company | None: + stmt = ( + select(Company) + .join( + ExternalID, + (ExternalID.entity_id == Company.id) & (ExternalID.entity_type == "company"), + ) + .where(ExternalID.provider_id == provider_id, ExternalID.external_id == external_id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def get_or_create_company( + session: AsyncSession, + *, + provider_id: uuid.UUID, + external_id: str, + name: str, + company_type: CompanyType = CompanyType.PRODUCTION_COMPANY, + country_code: str | None = None, +) -> Company: + company = await find_company_by_external_id(session, provider_id, external_id) + if company is not None: + return company + company = Company(name=name, company_type=company_type, country_code=country_code) + session.add(company) + await session.flush() + session.add( + ExternalID( + entity_type="company", + entity_id=company.id, + provider_id=provider_id, + external_id=external_id, + ) + ) + await session.flush() + return company + + +async def find_collection_by_external_id( + session: AsyncSession, provider_id: uuid.UUID, external_id: str +) -> Collection | None: + stmt = ( + select(Collection) + .join( + ExternalID, + (ExternalID.entity_id == Collection.id) & (ExternalID.entity_type == "collection"), + ) + .where(ExternalID.provider_id == provider_id, ExternalID.external_id == external_id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def get_or_create_collection( + session: AsyncSession, + *, + provider_id: uuid.UUID, + external_id: str, + name: str, + overview: str | None = None, +) -> Collection: + collection = await find_collection_by_external_id(session, provider_id, external_id) + if collection is not None: + return collection + base_slug = slugify(name) + slug = base_slug + suffix = 1 + while ( + await session.execute(select(Collection.id).where(Collection.slug == slug)) + ).scalar_one_or_none() is not None: + suffix += 1 + slug = f"{base_slug}-{suffix}" + collection = Collection(name=name, overview=overview, slug=slug) + session.add(collection) + await session.flush() + session.add( + ExternalID( + entity_type="collection", + entity_id=collection.id, + provider_id=provider_id, + external_id=external_id, + ) + ) + await session.flush() + return collection + + +async def find_person_by_external_id( + session: AsyncSession, provider_id: uuid.UUID, external_id: str +) -> Person | None: + stmt = ( + select(Person) + .join( + ExternalID, + (ExternalID.entity_id == Person.id) & (ExternalID.entity_type == "person"), + ) + .where(ExternalID.provider_id == provider_id, ExternalID.external_id == external_id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def get_or_create_person( + session: AsyncSession, + *, + provider_id: uuid.UUID, + external_id: str, + name: str, + **extra_fields: object, +) -> Person: + person = await find_person_by_external_id(session, provider_id, external_id) + if person is not None: + for key, value in extra_fields.items(): + if value is not None: + setattr(person, key, value) + await session.flush() + return person + person = Person(name=name, **extra_fields) + session.add(person) + await session.flush() + session.add( + ExternalID( + entity_type="person", + entity_id=person.id, + provider_id=provider_id, + external_id=external_id, + ) + ) + await session.flush() + return person + + +async def find_entity_by_external_id( + session: AsyncSession, provider_id: uuid.UUID, entity_type: str, external_id: str +) -> uuid.UUID | None: + stmt = select(ExternalID.entity_id).where( + ExternalID.provider_id == provider_id, + ExternalID.entity_type == entity_type, + ExternalID.external_id == external_id, + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def find_entities_by_external_ids( + session: AsyncSession, provider_id: uuid.UUID, entity_type: str, external_ids: list[str] +) -> list[uuid.UUID]: + """Bulk lookup preserving `external_ids` order; unmatched IDs are + dropped rather than raising, since a recommendation pointing at a movie + Aurora hasn't ingested yet is expected, not an error.""" + + if not external_ids: + return [] + stmt = select(ExternalID.external_id, ExternalID.entity_id).where( + ExternalID.provider_id == provider_id, + ExternalID.entity_type == entity_type, + ExternalID.external_id.in_(external_ids), + ) + result = await session.execute(stmt) + found = {external_id: entity_id for external_id, entity_id in result.all()} + return [found[eid] for eid in external_ids if eid in found] + + +async def link_external_id( + session: AsyncSession, + *, + entity_type: str, + entity_id: uuid.UUID, + provider_id: uuid.UUID, + external_id: str, + url: str | None = None, +) -> None: + stmt = select(ExternalID).where( + ExternalID.entity_type == entity_type, + ExternalID.entity_id == entity_id, + ExternalID.provider_id == provider_id, + ) + result = await session.execute(stmt) + existing = result.scalar_one_or_none() + if existing is not None: + existing.external_id = external_id + existing.url = url + else: + session.add( + ExternalID( + entity_type=entity_type, + entity_id=entity_id, + provider_id=provider_id, + external_id=external_id, + url=url, + ) + ) + await session.flush() diff --git a/app/repositories/tv_repository.py b/app/repositories/tv_repository.py new file mode 100644 index 0000000..9614579 --- /dev/null +++ b/app/repositories/tv_repository.py @@ -0,0 +1,79 @@ +"""Data access for Series -> Season -> Episode.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.company import Company, SeriesCompany +from app.models.enums import CompanyType +from app.models.genre import Genre, SeriesGenre +from app.models.keyword import Keyword, SeriesKeyword +from app.models.tv import Episode, Season, Series + + +async def get_series_by_id(session: AsyncSession, series_id: uuid.UUID) -> Series | None: + result = await session.execute(select(Series).where(Series.id == series_id)) + return result.scalar_one_or_none() + + +async def get_seasons_for_series(session: AsyncSession, series_id: uuid.UUID) -> list[Season]: + stmt = ( + select(Season) + .where(Season.series_id == series_id) + .order_by(Season.season_number.asc()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_season_by_id(session: AsyncSession, season_id: uuid.UUID) -> Season | None: + result = await session.execute(select(Season).where(Season.id == season_id)) + return result.scalar_one_or_none() + + +async def get_episodes_for_season(session: AsyncSession, season_id: uuid.UUID) -> list[Episode]: + stmt = ( + select(Episode) + .where(Episode.season_id == season_id) + .order_by(Episode.episode_number.asc()) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_episode_by_id(session: AsyncSession, episode_id: uuid.UUID) -> Episode | None: + result = await session.execute(select(Episode).where(Episode.id == episode_id)) + return result.scalar_one_or_none() + + +async def get_series_genres(session: AsyncSession, series_id: uuid.UUID) -> list[Genre]: + stmt = select(Genre).join(SeriesGenre, SeriesGenre.genre_id == Genre.id).where( + SeriesGenre.series_id == series_id + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_series_keywords(session: AsyncSession, series_id: uuid.UUID) -> list[Keyword]: + stmt = select(Keyword).join(SeriesKeyword, SeriesKeyword.keyword_id == Keyword.id).where( + SeriesKeyword.series_id == series_id + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def get_series_companies( + session: AsyncSession, series_id: uuid.UUID, company_type: CompanyType | None = None +) -> list[tuple[Company, str | None]]: + stmt = ( + select(Company, SeriesCompany.role) + .join(SeriesCompany, SeriesCompany.company_id == Company.id) + .where(SeriesCompany.series_id == series_id) + ) + if company_type is not None: + stmt = stmt.where(Company.company_type == company_type) + result = await session.execute(stmt) + return [(company, role) for company, role in result.all()] diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/schemas/collection.py b/app/schemas/collection.py new file mode 100644 index 0000000..b1565ab --- /dev/null +++ b/app/schemas/collection.py @@ -0,0 +1,24 @@ +"""Collection/franchise response schemas.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + +from app.schemas.movie import MovieSummaryOut +from app.schemas.tv import SeriesSummaryOut + + +class CollectionDetailOut(BaseModel): + id: str + type: str = "collection" + name: str + overview: str | None = None + poster_path: str | None = None + backdrop_path: str | None = None + movies: list[MovieSummaryOut] = [] + series: list[SeriesSummaryOut] = [] + + created_at: datetime + updated_at: datetime diff --git a/app/schemas/common.py b/app/schemas/common.py new file mode 100644 index 0000000..2318ce4 --- /dev/null +++ b/app/schemas/common.py @@ -0,0 +1,119 @@ +"""Shared response building blocks used across entity schemas. + +These are hand-written Pydantic models, never SQLAlchemy models serialized +directly — see section 18 of the build spec. +""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel, ConfigDict + + +class ORMModel(BaseModel): + """Base for schemas built from ORM rows via `model_validate(obj)`.""" + + model_config = ConfigDict(from_attributes=True) + + +class GenreOut(ORMModel): + id: str + name: str + + +class KeywordOut(ORMModel): + id: str + name: str + + +class ArtworkOut(BaseModel): + id: str + type: str + language: str | None = None + width: int | None = None + height: int | None = None + aspect_ratio: float | None = None + url: str + provider: str | None = None + is_primary: bool = False + vote_average: float | None = None + + +class ImagesOut(BaseModel): + posters: list[ArtworkOut] = [] + backdrops: list[ArtworkOut] = [] + logos: list[ArtworkOut] = [] + + +class VideoOut(BaseModel): + id: str + type: str + site: str | None = None + url: str + language: str | None = None + country: str | None = None + is_official: bool = False + published_at: date | None = None + provider: str | None = None + + +class ExternalIDsOut(BaseModel): + """Provider slug -> native ID, e.g. {"tmdb": "550", "imdb": "tt0137523"}.""" + + model_config = ConfigDict(extra="allow") + + +class PersonBriefOut(BaseModel): + id: str + name: str + profile_path: str | None = None + + +class CastCreditOut(BaseModel): + id: str + person: PersonBriefOut + character: str | None = None + billing_order: int | None = None + episode_count: int | None = None + + +class CrewCreditOut(BaseModel): + id: str + person: PersonBriefOut + department: str + job: str | None = None + episode_count: int | None = None + + +class CreditsOut(BaseModel): + cast: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + + +class CompanyBriefOut(BaseModel): + id: str + name: str + company_type: str + logo_path: str | None = None + + +class CollectionBriefOut(BaseModel): + id: str + name: str + + +class PageMeta(BaseModel): + page: int + page_size: int + total: int + total_pages: int + + +class ErrorDetail(BaseModel): + code: str + message: str + + +class ErrorResponse(BaseModel): + error: ErrorDetail diff --git a/app/schemas/company.py b/app/schemas/company.py new file mode 100644 index 0000000..17af1cc --- /dev/null +++ b/app/schemas/company.py @@ -0,0 +1,24 @@ +"""Company response schemas.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + + +class CompanyDetailOut(BaseModel): + id: str + type: str = "company" + name: str + company_type: str + description: str | None = None + country: str | None = None + headquarters: str | None = None + homepage: str | None = None + logo_path: str | None = None + parent_company_id: str | None = None + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime diff --git a/app/schemas/health.py b/app/schemas/health.py new file mode 100644 index 0000000..c7ab2eb --- /dev/null +++ b/app/schemas/health.py @@ -0,0 +1,20 @@ +"""Health/readiness response schemas.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class HealthOut(BaseModel): + status: str = "ok" + + +class DependencyStatus(BaseModel): + name: str + ok: bool + required: bool + + +class ReadyOut(BaseModel): + status: str + dependencies: list[DependencyStatus] diff --git a/app/schemas/movie.py b/app/schemas/movie.py new file mode 100644 index 0000000..bac78b5 --- /dev/null +++ b/app/schemas/movie.py @@ -0,0 +1,80 @@ +"""Movie response schemas.""" + +from __future__ import annotations + +from datetime import date, datetime + +from pydantic import BaseModel + +from app.schemas.common import ( + CastCreditOut, + CollectionBriefOut, + CompanyBriefOut, + CrewCreditOut, + GenreOut, + ImagesOut, + KeywordOut, + VideoOut, +) + + +class MovieSummaryOut(BaseModel): + """Compact movie representation used in lists and search results.""" + + id: str + type: str = "movie" + title: str + original_title: str | None = None + overview: str | None = None + release_date: date | None = None + poster_path: str | None = None + popularity: float | None = None + vote_average: float | None = None + + +class MovieDetailOut(BaseModel): + id: str + type: str = "movie" + title: str + original_title: str | None = None + overview: str | None = None + tagline: str | None = None + status: str + original_language: str | None = None + release_date: date | None = None + runtime: int | None = None + budget: int | None = None + revenue: int | None = None + popularity: float | None = None + vote_average: float | None = None + vote_count: int | None = None + adult: bool = False + + genres: list[GenreOut] = [] + keywords: list[KeywordOut] = [] + cast: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + companies: list[CompanyBriefOut] = [] + collection: CollectionBriefOut | None = None + images: ImagesOut = ImagesOut() + videos: list[VideoOut] = [] + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime + + +class MovieCreditsOut(BaseModel): + id: str + cast: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + + +class MovieImagesOut(BaseModel): + id: str + images: ImagesOut = ImagesOut() + + +class MovieVideosOut(BaseModel): + id: str + videos: list[VideoOut] = [] diff --git a/app/schemas/person.py b/app/schemas/person.py new file mode 100644 index 0000000..1c51419 --- /dev/null +++ b/app/schemas/person.py @@ -0,0 +1,55 @@ +"""Person response schemas.""" + +from __future__ import annotations + +from datetime import date, datetime + +from pydantic import BaseModel + +from app.schemas.common import ArtworkOut + + +class PersonSummaryOut(BaseModel): + id: str + type: str = "person" + name: str + known_for_department: str | None = None + profile_path: str | None = None + popularity: float | None = None + + +class FilmographyCreditOut(BaseModel): + id: str + media_id: str + media_type: str + title: str + department: str + job: str | None = None + character: str | None = None + release_date: date | None = None + poster_path: str | None = None + + +class PersonDetailOut(BaseModel): + id: str + type: str = "person" + name: str + aliases: list[str] = [] + biography: str | None = None + birthday: date | None = None + deathday: date | None = None + birthplace: str | None = None + gender: str | None = None + known_for_department: str | None = None + popularity: float | None = None + profile_images: list[ArtworkOut] = [] + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime + + +class PersonCreditsOut(BaseModel): + id: str + cast: list[FilmographyCreditOut] = [] + crew: list[FilmographyCreditOut] = [] diff --git a/app/schemas/search.py b/app/schemas/search.py new file mode 100644 index 0000000..e5a7d02 --- /dev/null +++ b/app/schemas/search.py @@ -0,0 +1,20 @@ +"""Search response schemas.""" + +from __future__ import annotations + +from pydantic import BaseModel + +from app.schemas.common import CollectionBriefOut, CompanyBriefOut +from app.schemas.movie import MovieSummaryOut +from app.schemas.person import PersonSummaryOut +from app.schemas.tv import SeriesSummaryOut + + +class SearchResultsOut(BaseModel): + query: str + movies: list[MovieSummaryOut] = [] + series: list[SeriesSummaryOut] = [] + people: list[PersonSummaryOut] = [] + collections: list[CollectionBriefOut] = [] + companies: list[CompanyBriefOut] = [] + total_results: int = 0 diff --git a/app/schemas/tv.py b/app/schemas/tv.py new file mode 100644 index 0000000..63cd5e6 --- /dev/null +++ b/app/schemas/tv.py @@ -0,0 +1,123 @@ +"""Series/season/episode response schemas.""" + +from __future__ import annotations + +from datetime import date, datetime + +from pydantic import BaseModel + +from app.schemas.common import ( + CastCreditOut, + CompanyBriefOut, + CrewCreditOut, + GenreOut, + ImagesOut, + KeywordOut, + VideoOut, +) + + +class SeriesSummaryOut(BaseModel): + id: str + type: str = "series" + title: str + original_title: str | None = None + overview: str | None = None + first_air_date: date | None = None + poster_path: str | None = None + popularity: float | None = None + vote_average: float | None = None + + +class SeasonBriefOut(BaseModel): + id: str + season_number: int + title: str | None = None + episode_count: int | None = None + air_date: date | None = None + poster_path: str | None = None + + +class SeriesDetailOut(BaseModel): + id: str + type: str = "series" + title: str + original_title: str | None = None + overview: str | None = None + tagline: str | None = None + status: str + original_language: str | None = None + first_air_date: date | None = None + last_air_date: date | None = None + number_of_seasons: int | None = None + number_of_episodes: int | None = None + popularity: float | None = None + vote_average: float | None = None + vote_count: int | None = None + + genres: list[GenreOut] = [] + keywords: list[KeywordOut] = [] + creators: list[CrewCreditOut] = [] + cast: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + networks: list[CompanyBriefOut] = [] + companies: list[CompanyBriefOut] = [] + seasons: list[SeasonBriefOut] = [] + images: ImagesOut = ImagesOut() + videos: list[VideoOut] = [] + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime + + +class SeriesSeasonsOut(BaseModel): + id: str + seasons: list[SeasonBriefOut] = [] + + +class EpisodeBriefOut(BaseModel): + id: str + episode_number: int + title: str | None = None + air_date: date | None = None + still_path: str | None = None + + +class SeasonDetailOut(BaseModel): + id: str + type: str = "season" + series_id: str + season_number: int + title: str | None = None + overview: str | None = None + air_date: date | None = None + episode_count: int | None = None + episodes: list[EpisodeBriefOut] = [] + images: ImagesOut = ImagesOut() + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime + + +class EpisodeDetailOut(BaseModel): + id: str + type: str = "episode" + series_id: str + season_id: str + season_number: int + episode_number: int + title: str | None = None + overview: str | None = None + air_date: date | None = None + runtime: int | None = None + + cast: list[CastCreditOut] = [] + guest_stars: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + images: ImagesOut = ImagesOut() + external_ids: dict[str, str] = {} + + created_at: datetime + updated_at: datetime diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/cache.py b/app/services/cache.py new file mode 100644 index 0000000..9d378d0 --- /dev/null +++ b/app/services/cache.py @@ -0,0 +1,61 @@ +"""Cache abstraction over Redis (section 21). + +Redis is disposable acceleration — every function here fails open: if Redis +is unreachable, reads report a cache miss and writes are silently skipped, +so the API keeps serving from PostgreSQL instead of failing. +""" + +from __future__ import annotations + +import orjson +from redis.asyncio import Redis + +from app.core.config import get_settings + +settings = get_settings() + +CACHE_VERSION = "v1" + + +def build_key(*parts: str) -> str: + return ":".join(("aurora", CACHE_VERSION, *parts)) + + +async def cache_get_json(redis: Redis, key: str) -> dict | list | None: + try: + raw = await redis.get(key) + except Exception: + return None + if raw is None: + return None + try: + return orjson.loads(raw) + except orjson.JSONDecodeError: + return None + + +async def cache_set_json( + redis: Redis, key: str, value: dict | list, ttl: int | None = None +) -> None: + ttl = ttl if ttl is not None else settings.CACHE_DEFAULT_TTL + try: + await redis.set(key, orjson.dumps(value), ex=ttl) + except Exception: + return + + +async def cache_delete(redis: Redis, *keys: str) -> None: + if not keys: + return + try: + await redis.delete(*keys) + except Exception: + return + + +async def cache_delete_prefix(redis: Redis, prefix: str) -> None: + try: + async for key in redis.scan_iter(match=f"{prefix}*"): + await redis.delete(key) + except Exception: + return diff --git a/app/services/collection_service.py b/app/services/collection_service.py new file mode 100644 index 0000000..4d4333c --- /dev/null +++ b/app/services/collection_service.py @@ -0,0 +1,75 @@ +"""Assembles Collection/franchise API responses from repository data.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id, parse_aurora_id +from app.models.enums import ArtworkType +from app.repositories import collection_repository, polymorphic +from app.schemas.collection import CollectionDetailOut +from app.schemas.common import CollectionBriefOut +from app.services.common import primary_artwork_path +from app.services.movie_service import build_movie_summaries +from app.services.tv_service import build_series_summaries + + +class CollectionNotFoundError(Exception): + pass + + +def _resolve_collection_id(aid: str) -> uuid.UUID: + try: + return uuid.UUID(parse_aurora_id(aid, expected_type="collection")) + except ValueError as exc: + raise CollectionNotFoundError(str(exc)) from exc + + +async def get_collection_detail( + session: AsyncSession, collection_aurora_id: str +) -> CollectionDetailOut: + collection_id = _resolve_collection_id(collection_aurora_id) + collection = await collection_repository.get_collection_by_id(session, collection_id) + if collection is None: + raise CollectionNotFoundError(collection_aurora_id) + + movies = await collection_repository.get_collection_movies(session, collection_id) + series = await collection_repository.get_collection_series(session, collection_id) + posters = await polymorphic.get_artwork(session, "collection", collection_id, ArtworkType.POSTER) + backdrops = await polymorphic.get_artwork( + session, "collection", collection_id, ArtworkType.BACKDROP + ) + + return CollectionDetailOut( + id=aurora_id("collection", str(collection.id)), + name=collection.name, + overview=collection.overview, + poster_path=primary_artwork_path(posters, ArtworkType.POSTER), + backdrop_path=primary_artwork_path(backdrops, ArtworkType.BACKDROP), + movies=await build_movie_summaries(session, [m.id for m in movies]), + series=await build_series_summaries(session, [s.id for s in series]), + created_at=collection.created_at, + updated_at=collection.updated_at, + ) + + +async def build_collection_summary( + session: AsyncSession, collection_id: uuid.UUID +) -> CollectionBriefOut | None: + collection = await collection_repository.get_collection_by_id(session, collection_id) + if collection is None: + return None + return CollectionBriefOut(id=aurora_id("collection", str(collection.id)), name=collection.name) + + +async def build_collection_summaries( + session: AsyncSession, collection_ids: list[uuid.UUID] +) -> list[CollectionBriefOut]: + out: list[CollectionBriefOut] = [] + for collection_id in collection_ids: + summary = await build_collection_summary(session, collection_id) + if summary is not None: + out.append(summary) + return out diff --git a/app/services/common.py b/app/services/common.py new file mode 100644 index 0000000..f36e68e --- /dev/null +++ b/app/services/common.py @@ -0,0 +1,105 @@ +"""Shared helpers for turning ORM rows into response schemas.""" + +from __future__ import annotations + +from app.core.ids import aurora_id +from app.models.artwork import Artwork +from app.models.credit import Credit +from app.models.enums import ArtworkType +from app.models.video import Video +from app.schemas.common import ( + ArtworkOut, + CastCreditOut, + CrewCreditOut, + ImagesOut, + PersonBriefOut, + VideoOut, +) + + +def artwork_url(artwork: Artwork) -> str: + return artwork.local_path or artwork.original_url + + +def to_artwork_out(artwork: Artwork) -> ArtworkOut: + return ArtworkOut( + id=aurora_id("artwork", str(artwork.id)), + type=artwork.artwork_type.value, + language=artwork.language_code, + width=artwork.width, + height=artwork.height, + aspect_ratio=artwork.aspect_ratio, + url=artwork_url(artwork), + provider=artwork.provider.slug if artwork.provider else None, + is_primary=artwork.is_primary, + vote_average=artwork.vote_average, + ) + + +def to_images_out(artwork_list: list[Artwork]) -> ImagesOut: + images = ImagesOut() + for artwork in artwork_list: + out = to_artwork_out(artwork) + if artwork.artwork_type == ArtworkType.POSTER: + images.posters.append(out) + elif artwork.artwork_type == ArtworkType.BACKDROP: + images.backdrops.append(out) + elif artwork.artwork_type == ArtworkType.LOGO: + images.logos.append(out) + return images + + +def primary_artwork_path(artwork_list: list[Artwork], artwork_type: ArtworkType) -> str | None: + for artwork in artwork_list: + if artwork.artwork_type == artwork_type: + return artwork_url(artwork) + return None + + +def to_video_out(video: Video) -> VideoOut: + return VideoOut( + id=aurora_id("video", str(video.id)), + type=video.video_type.value, + site=video.site, + url=video.url, + language=video.language_code, + country=video.country_code, + is_official=video.is_official, + published_at=video.published_at, + provider=video.provider.slug if video.provider else None, + ) + + +def split_credits( + credits_: list[Credit], profile_paths: dict | None = None +) -> tuple[list[CastCreditOut], list[CrewCreditOut]]: + profile_paths = profile_paths or {} + cast: list[CastCreditOut] = [] + crew: list[CrewCreditOut] = [] + for credit in credits_: + person_brief = PersonBriefOut( + id=aurora_id("person", str(credit.person_id)), + name=credit.person.name, + profile_path=profile_paths.get(credit.person_id), + ) + if credit.department == "Acting": + cast.append( + CastCreditOut( + id=aurora_id("credit", str(credit.id)), + person=person_brief, + character=credit.character, + billing_order=credit.billing_order, + episode_count=credit.episode_count, + ) + ) + else: + crew.append( + CrewCreditOut( + id=aurora_id("credit", str(credit.id)), + person=person_brief, + department=credit.department, + job=credit.job, + episode_count=credit.episode_count, + ) + ) + return cast, crew diff --git a/app/services/company_service.py b/app/services/company_service.py new file mode 100644 index 0000000..b24f1c1 --- /dev/null +++ b/app/services/company_service.py @@ -0,0 +1,80 @@ +"""Assembles Company API responses from repository data.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id, parse_aurora_id +from app.models.enums import ArtworkType +from app.repositories import company_repository, polymorphic +from app.schemas.common import CompanyBriefOut +from app.schemas.company import CompanyDetailOut +from app.services.common import primary_artwork_path + + +class CompanyNotFoundError(Exception): + pass + + +def _resolve_company_id(aid: str) -> uuid.UUID: + try: + return uuid.UUID(parse_aurora_id(aid, expected_type="company")) + except ValueError as exc: + raise CompanyNotFoundError(str(exc)) from exc + + +async def get_company_detail(session: AsyncSession, company_aurora_id: str) -> CompanyDetailOut: + company_id = _resolve_company_id(company_aurora_id) + company = await company_repository.get_company_by_id(session, company_id) + if company is None: + raise CompanyNotFoundError(company_aurora_id) + + logos = await polymorphic.get_artwork(session, "company", company_id, ArtworkType.LOGO) + external_ids = await polymorphic.get_external_ids(session, "company", company_id) + + return CompanyDetailOut( + id=aurora_id("company", str(company.id)), + name=company.name, + company_type=company.company_type.value, + description=company.description, + country=company.country_code, + headquarters=company.headquarters, + homepage=company.homepage, + logo_path=primary_artwork_path(logos, ArtworkType.LOGO), + parent_company_id=( + aurora_id("company", str(company.parent_company_id)) + if company.parent_company_id + else None + ), + external_ids=external_ids, + created_at=company.created_at, + updated_at=company.updated_at, + ) + + +async def build_company_summary( + session: AsyncSession, company_id: uuid.UUID +) -> CompanyBriefOut | None: + company = await company_repository.get_company_by_id(session, company_id) + if company is None: + return None + logos = await polymorphic.get_artwork(session, "company", company_id, ArtworkType.LOGO) + return CompanyBriefOut( + id=aurora_id("company", str(company.id)), + name=company.name, + company_type=company.company_type.value, + logo_path=primary_artwork_path(logos, ArtworkType.LOGO), + ) + + +async def build_company_summaries( + session: AsyncSession, company_ids: list[uuid.UUID] +) -> list[CompanyBriefOut]: + out: list[CompanyBriefOut] = [] + for company_id in company_ids: + summary = await build_company_summary(session, company_id) + if summary is not None: + out.append(summary) + return out diff --git a/app/services/ingestion/__init__.py b/app/services/ingestion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/ingestion/movie_ingestion.py b/app/services/ingestion/movie_ingestion.py new file mode 100644 index 0000000..40c1ae1 --- /dev/null +++ b/app/services/ingestion/movie_ingestion.py @@ -0,0 +1,314 @@ +"""Movie ingestion pipeline (section 15): + +Provider -> raw data -> normalize -> match -> upsert Aurora entity -> +satellite tables (genres, keywords, companies, credits, artwork, videos, +external IDs) -> provenance record. + +Idempotent: re-running for the same provider + external_id updates the same +Aurora movie rather than creating a duplicate (zero duplicate ingestion +records — section 2). Satellite associations are replaced wholesale on each +sync rather than diffed, which is simple and safe but means their Aurora +IDs (e.g. individual credit IDs) are not stable across re-syncs — see +docs/architecture.md for the tradeoff. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.models.artwork import Artwork +from app.models.company import MovieCompany +from app.models.credit import Credit +from app.models.enums import ArtworkType, MatchConfidence, MovieStatus, VideoType +from app.models.genre import MovieGenre +from app.models.keyword import MovieKeyword +from app.models.movie import Movie, MovieRecommendation +from app.models.video import Video +from app.providers.base import MetadataProvider +from app.repositories import reference_repository as refs +from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record +from app.services.metadata.matcher import match_entity +from app.services.metadata.normalizer import NormalizedMovie, normalize_movie + +logger = get_logger(__name__) + + +async def _replace_movie_genres(session: AsyncSession, movie_id: uuid.UUID, names: list[str]) -> None: + await session.execute(delete(MovieGenre).where(MovieGenre.movie_id == movie_id)) + for name in names: + genre = await refs.get_or_create_genre(session, name) + session.add(MovieGenre(movie_id=movie_id, genre_id=genre.id)) + + +async def _replace_movie_keywords(session: AsyncSession, movie_id: uuid.UUID, names: list[str]) -> None: + await session.execute(delete(MovieKeyword).where(MovieKeyword.movie_id == movie_id)) + for name in names: + keyword = await refs.get_or_create_keyword(session, name) + session.add(MovieKeyword(movie_id=movie_id, keyword_id=keyword.id)) + + +async def _replace_movie_companies( + session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, companies +) -> None: + await session.execute(delete(MovieCompany).where(MovieCompany.movie_id == movie_id)) + for ref in companies: + country_code = None + if ref.country_code: + country = await refs.get_or_create_country(session, ref.country_code, ref.country_code) + country_code = country.code + company = await refs.get_or_create_company( + session, + provider_id=provider_id, + external_id=ref.external_id, + name=ref.name, + country_code=country_code, + ) + session.add(MovieCompany(movie_id=movie_id, company_id=company.id, role=ref.role)) + + +async def _replace_movie_credits( + session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, cast, crew +) -> None: + await session.execute( + delete(Credit).where(Credit.entity_type == "movie", Credit.entity_id == movie_id) + ) + for order, ref in enumerate(cast): + person = await refs.get_or_create_person( + session, + provider_id=provider_id, + external_id=ref.external_id, + name=ref.name, + gender=ref.gender, + ) + session.add( + Credit( + person_id=person.id, + entity_type="movie", + entity_id=movie_id, + department="Acting", + character=ref.character, + billing_order=ref.billing_order if ref.billing_order is not None else order, + provider_id=provider_id, + ) + ) + for ref in crew: + person = await refs.get_or_create_person( + session, + provider_id=provider_id, + external_id=ref.external_id, + name=ref.name, + gender=ref.gender, + ) + session.add( + Credit( + person_id=person.id, + entity_type="movie", + entity_id=movie_id, + department=ref.department or "Crew", + job=ref.job, + provider_id=provider_id, + ) + ) + + +_ARTWORK_TYPES = {t.value for t in ArtworkType} +_VIDEO_TYPES = {t.value for t in VideoType} + + +async def _replace_movie_artwork( + session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, artwork_list +) -> None: + await session.execute( + delete(Artwork).where(Artwork.entity_type == "movie", Artwork.entity_id == movie_id) + ) + for art in artwork_list: + art_type = art.artwork_type if art.artwork_type in _ARTWORK_TYPES else "poster" + session.add( + Artwork( + entity_type="movie", + entity_id=movie_id, + artwork_type=ArtworkType(art_type), + language_code=None, + width=art.width, + height=art.height, + aspect_ratio=art.aspect_ratio, + provider_id=provider_id, + provider_asset_id=art.provider_asset_id, + original_url=art.url, + vote_average=art.vote_average, + is_primary=False, + ) + ) + + +async def _replace_movie_videos( + session: AsyncSession, movie_id: uuid.UUID, provider_id: uuid.UUID, videos +) -> None: + await session.execute( + delete(Video).where(Video.entity_type == "movie", Video.entity_id == movie_id) + ) + for v in videos: + video_type = v.video_type if v.video_type in _VIDEO_TYPES else "other" + session.add( + Video( + entity_type="movie", + entity_id=movie_id, + video_type=VideoType(video_type), + provider_id=provider_id, + provider_video_id=v.provider_video_id, + site=v.site, + url=v.url, + language_code=None, + country_code=None, + is_official=v.is_official, + published_at=v.published_at, + ) + ) + + +async def _link_external_ids( + session: AsyncSession, movie_id: uuid.UUID, tmdb_provider_id: uuid.UUID, normalized: NormalizedMovie +) -> None: + await refs.link_external_id( + session, + entity_type="movie", + entity_id=movie_id, + provider_id=tmdb_provider_id, + external_id=normalized.external_id, + ) + for slug, external_id in normalized.external_ids.items(): + provider = await get_or_create_provider(session, slug, slug.upper()) + await refs.link_external_id( + session, + entity_type="movie", + entity_id=movie_id, + provider_id=provider.id, + external_id=external_id, + ) + + +async def ingest_movie( + session: AsyncSession, + provider: MetadataProvider, + external_id: str, + *, + ingestion_job_id: uuid.UUID | None = None, +) -> uuid.UUID: + """Fetch, normalize, and upsert a single movie. Returns the Aurora + movie UUID (new or existing).""" + + tmdb_provider = await get_or_create_provider(session, provider.slug, provider.name) + + aurora_id_existing, confidence = await match_entity( + session, provider_id=tmdb_provider.id, entity_type="movie", external_id=external_id + ) + + raw = await provider.get_movie(external_id) + normalized = normalize_movie(raw) + + existing_movie: Movie | None = None + if aurora_id_existing is not None: + existing_movie = await session.get(Movie, aurora_id_existing) + if existing_movie is None: + aurora_id_existing = None + + movie: Movie + if existing_movie is not None: + movie = existing_movie + else: + movie = Movie(title=normalized.title, sort_title=_sort_title(normalized.title)) + session.add(movie) + await session.flush() + + movie.title = normalized.title + movie.original_title = normalized.original_title + movie.sort_title = _sort_title(normalized.title) + movie.overview = normalized.overview + movie.tagline = normalized.tagline + movie.status = MovieStatus(normalized.status) + movie.primary_release_date = normalized.release_date + movie.runtime_minutes = normalized.runtime_minutes + movie.budget = normalized.budget + movie.revenue = normalized.revenue + movie.popularity = normalized.popularity + movie.vote_average = normalized.vote_average + movie.vote_count = normalized.vote_count + movie.adult = normalized.adult + movie.source_provider_id = tmdb_provider.id + + if normalized.original_language: + language = await refs.get_or_create_language( + session, normalized.original_language, normalized.original_language.upper() + ) + movie.original_language_code = language.code + + if normalized.collection: + ext_id, name = normalized.collection + collection = await refs.get_or_create_collection( + session, provider_id=tmdb_provider.id, external_id=ext_id, name=name + ) + movie.collection_id = collection.id + + await session.flush() + + await _replace_movie_genres(session, movie.id, normalized.genres) + await _replace_movie_keywords(session, movie.id, normalized.keywords) + await _replace_movie_companies(session, movie.id, tmdb_provider.id, normalized.companies) + await _replace_movie_credits(session, movie.id, tmdb_provider.id, normalized.cast, normalized.crew) + await _replace_movie_artwork(session, movie.id, tmdb_provider.id, normalized.artwork) + await _replace_movie_videos(session, movie.id, tmdb_provider.id, normalized.videos) + await _link_external_ids(session, movie.id, tmdb_provider.id, normalized) + + await session.execute( + delete(MovieRecommendation).where(MovieRecommendation.movie_id == movie.id) + ) + if normalized.recommendation_external_ids: + rec_ids = await refs.find_entities_by_external_ids( + session, tmdb_provider.id, "movie", normalized.recommendation_external_ids + ) + for rank, rec_movie_id in enumerate(rec_ids): + session.add( + MovieRecommendation( + movie_id=movie.id, + recommended_movie_id=rec_movie_id, + rank=rank, + source="tmdb", + ) + ) + + await refs.link_external_id( + session, + entity_type="movie", + entity_id=movie.id, + provider_id=tmdb_provider.id, + external_id=external_id, + ) + + await upsert_provider_record( + session, + provider_id=tmdb_provider.id, + entity_type="movie", + external_id=external_id, + raw_data=raw, + aurora_entity_id=movie.id, + match_confidence=confidence if aurora_id_existing else MatchConfidence.NEW, + ingestion_job_id=ingestion_job_id, + ) + + logger.info("movie_ingested", movie_id=str(movie.id), external_id=external_id) + return movie.id + + +_LEADING_ARTICLES = ("the ", "a ", "an ") + + +def _sort_title(title: str) -> str: + lowered = title.lower() + for article in _LEADING_ARTICLES: + if lowered.startswith(article): + return title[len(article) :].strip() + return title diff --git a/app/services/ingestion/person_ingestion.py b/app/services/ingestion/person_ingestion.py new file mode 100644 index 0000000..8839fb0 --- /dev/null +++ b/app/services/ingestion/person_ingestion.py @@ -0,0 +1,115 @@ +"""Person ingestion pipeline (section 15). + +Movie/series ingestion already creates a stub `Person` row (name + gender) +for each cast/crew credit via `get_or_create_person`. This module fetches +and upserts a *full* profile (biography, birthday, images, ...) for one +person, either standalone (admin-triggered) or as a follow-up job. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.models.artwork import Artwork +from app.models.enums import ArtworkType, MatchConfidence +from app.models.person import Person +from app.providers.base import MetadataProvider +from app.repositories import reference_repository as refs +from app.repositories.provider_repository import get_or_create_provider, upsert_provider_record +from app.services.metadata.matcher import match_entity +from app.services.metadata.normalizer import normalize_person + +logger = get_logger(__name__) + + +async def ingest_person( + session: AsyncSession, + provider: MetadataProvider, + external_id: str, + *, + ingestion_job_id: uuid.UUID | None = None, +) -> uuid.UUID: + tmdb_provider = await get_or_create_provider(session, provider.slug, provider.name) + + aurora_id_existing, confidence = await match_entity( + session, provider_id=tmdb_provider.id, entity_type="person", external_id=external_id + ) + + raw = await provider.get_person(external_id) + normalized = normalize_person(raw) + + person: Person | None = None + if aurora_id_existing is not None: + person = await session.get(Person, aurora_id_existing) + + if person is None: + person = Person(name=normalized.name) + session.add(person) + await session.flush() + aurora_id_existing = None + + person.name = normalized.name + person.biography = normalized.biography + person.birthday = normalized.birthday + person.deathday = normalized.deathday + person.birthplace = normalized.birthplace + person.gender = normalized.gender + person.known_for_department = normalized.known_for_department + person.popularity = normalized.popularity + person.aliases = normalized.aliases + + await session.flush() + + await session.execute( + delete(Artwork).where(Artwork.entity_type == "person", Artwork.entity_id == person.id) + ) + for art in normalized.artwork: + session.add( + Artwork( + entity_type="person", + entity_id=person.id, + artwork_type=ArtworkType.PROFILE, + width=art.width, + height=art.height, + aspect_ratio=art.aspect_ratio, + provider_id=tmdb_provider.id, + provider_asset_id=art.provider_asset_id, + original_url=art.url, + vote_average=art.vote_average, + ) + ) + + await refs.link_external_id( + session, + entity_type="person", + entity_id=person.id, + provider_id=tmdb_provider.id, + external_id=external_id, + ) + for slug, ext_id in normalized.external_ids.items(): + ext_provider = await get_or_create_provider(session, slug, slug.upper()) + await refs.link_external_id( + session, + entity_type="person", + entity_id=person.id, + provider_id=ext_provider.id, + external_id=ext_id, + ) + + await upsert_provider_record( + session, + provider_id=tmdb_provider.id, + entity_type="person", + external_id=external_id, + raw_data=raw, + aurora_entity_id=person.id, + match_confidence=confidence if aurora_id_existing else MatchConfidence.NEW, + ingestion_job_id=ingestion_job_id, + ) + + logger.info("person_ingested", person_id=str(person.id), external_id=external_id) + return person.id diff --git a/app/services/ingestion/runner.py b/app/services/ingestion/runner.py new file mode 100644 index 0000000..5414be3 --- /dev/null +++ b/app/services/ingestion/runner.py @@ -0,0 +1,103 @@ +"""Orchestrates one ingestion job end-to-end: create/track the `IngestionJob` +row, run the provider fetch + normalize + upsert pipeline, and record +success/failure. Used by both the admin trigger endpoint (section 17) and +the arq worker task (section 15) so the two entry points share identical +job-lifecycle and idempotency behavior. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import session_scope +from app.core.logging import get_logger +from app.models.provider import IngestionJob +from app.providers.tmdb import TMDbProvider +from app.repositories.provider_repository import ( + create_ingestion_job, + get_ingestion_job, + get_or_create_provider, + mark_job_failed, + mark_job_running, + mark_job_succeeded, +) +from app.services.ingestion.movie_ingestion import ingest_movie +from app.services.ingestion.person_ingestion import ingest_person + +logger = get_logger(__name__) + + +async def _require_job(session: AsyncSession, job_id: uuid.UUID) -> IngestionJob: + job = await get_ingestion_job(session, job_id) + if job is None: + # Only possible if the row was deleted out-of-band; the job was + # created moments earlier in this same call chain. + raise RuntimeError(f"ingestion job {job_id} vanished mid-run") + return job + + +async def _create_job(job_type: str, entity_type: str, external_id: str) -> uuid.UUID: + idempotency_key = f"tmdb:{entity_type}:{external_id}" + async with session_scope() as session: + provider_row = await get_or_create_provider(session, "tmdb", "The Movie Database (TMDb)") + job = await create_ingestion_job( + session, + provider_id=provider_row.id, + job_type=job_type, + idempotency_key=idempotency_key, + entity_type=entity_type, + external_id=external_id, + ) + return job.id + + +async def run_movie_ingestion(external_id: str) -> uuid.UUID: + job_id = await _create_job("movie_sync", "movie", external_id) + + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_running(session, job) + + provider = TMDbProvider() + try: + async with session_scope() as session: + movie_id = await ingest_movie(session, provider, external_id, ingestion_job_id=job_id) + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_succeeded(session, job, movie_id) + return movie_id + except Exception as exc: + logger.error("movie_ingestion_failed", external_id=external_id, error=str(exc)) + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_failed(session, job, str(exc)) + raise + finally: + await provider.aclose() + + +async def run_person_ingestion(external_id: str) -> uuid.UUID: + job_id = await _create_job("person_sync", "person", external_id) + + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_running(session, job) + + provider = TMDbProvider() + try: + async with session_scope() as session: + person_id = await ingest_person(session, provider, external_id, ingestion_job_id=job_id) + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_succeeded(session, job, person_id) + return person_id + except Exception as exc: + logger.error("person_ingestion_failed", external_id=external_id, error=str(exc)) + async with session_scope() as session: + job = await _require_job(session, job_id) + await mark_job_failed(session, job, str(exc)) + raise + finally: + await provider.aclose() diff --git a/app/services/metadata/__init__.py b/app/services/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/metadata/matcher.py b/app/services/metadata/matcher.py new file mode 100644 index 0000000..83e3bb8 --- /dev/null +++ b/app/services/metadata/matcher.py @@ -0,0 +1,37 @@ +"""Entity matching (section 14): decides whether an incoming provider +record corresponds to an existing Aurora entity. + +Core 0.1 policy — deliberately conservative: + +- An exact external-ID match (this provider has already been linked to an + Aurora entity of this type) is the only case treated as MatchConfidence + EXACT and reused. +- Anything else creates a *new* Aurora entity (MatchConfidence NEW) rather + than guessing via fuzzy title matching. Silently merging two providers' + differing descriptions of "the same" title is exactly what section 14 + prohibits. + +Fuzzy candidate matching (title + year similarity, surfaced for human +review) is genuinely deferred — see docs/architecture.md — until Core has +an admin review workflow to act on AMBIGUOUS matches. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.enums import MatchConfidence +from app.repositories.reference_repository import find_entity_by_external_id + + +async def match_entity( + session: AsyncSession, *, provider_id: uuid.UUID, entity_type: str, external_id: str +) -> tuple[uuid.UUID | None, MatchConfidence]: + """Return (existing_aurora_id_or_None, confidence).""" + + existing = await find_entity_by_external_id(session, provider_id, entity_type, external_id) + if existing is not None: + return existing, MatchConfidence.EXACT + return None, MatchConfidence.NEW diff --git a/app/services/metadata/normalizer.py b/app/services/metadata/normalizer.py new file mode 100644 index 0000000..8892b0a --- /dev/null +++ b/app/services/metadata/normalizer.py @@ -0,0 +1,341 @@ +"""Normalizes raw TMDb payloads into provider-agnostic dataclasses. + +This is the seam between "provider shape" and "Aurora shape" (section 13). +A future provider adapter only needs its own `normalize_movie`-equivalent +that produces these same dataclasses; nothing downstream (matcher, +ingestion) needs to know which provider the data came from. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date + +TMDB_IMAGE_BASE = "https://image.tmdb.org/t/p/original" + + +def _parse_date(value: str | None) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value) + except ValueError: + return None + + +def _image_url(path: str | None) -> str | None: + if not path: + return None + return f"{TMDB_IMAGE_BASE}{path}" + + +def _image_url_required(path: str) -> str: + """Same as `_image_url`, for call sites already guarded by a truthy + check on `path` — keeps `NormalizedArtwork.url: str` honest.""" + + return f"{TMDB_IMAGE_BASE}{path}" + + +@dataclass +class NormalizedPersonRef: + external_id: str + name: str + profile_path: str | None = None + department: str | None = None + job: str | None = None + character: str | None = None + billing_order: int | None = None + gender: str | None = None + + +@dataclass +class NormalizedCompanyRef: + external_id: str + name: str + country_code: str | None = None + role: str = "production" + + +@dataclass +class NormalizedArtwork: + artwork_type: str # "poster" | "backdrop" | "logo" | "profile" | "still" + url: str + provider_asset_id: str | None + width: int | None + height: int | None + aspect_ratio: float | None + language: str | None + vote_average: float | None = None + + +@dataclass +class NormalizedVideo: + video_type: str + provider_video_id: str + site: str | None + url: str + language: str | None + country: str | None + is_official: bool + published_at: date | None + + +@dataclass +class NormalizedMovie: + external_id: str + title: str + original_title: str | None + overview: str | None + tagline: str | None + status: str + original_language: str | None + release_date: date | None + runtime_minutes: int | None + budget: int | None + revenue: int | None + popularity: float | None + vote_average: float | None + vote_count: int | None + adult: bool + genres: list[str] = field(default_factory=list) + keywords: list[str] = field(default_factory=list) + alternate_titles: list[str] = field(default_factory=list) + companies: list[NormalizedCompanyRef] = field(default_factory=list) + cast: list[NormalizedPersonRef] = field(default_factory=list) + crew: list[NormalizedPersonRef] = field(default_factory=list) + artwork: list[NormalizedArtwork] = field(default_factory=list) + videos: list[NormalizedVideo] = field(default_factory=list) + external_ids: dict[str, str] = field(default_factory=dict) + collection: tuple[str, str] | None = None # (external_id, name) + recommendation_external_ids: list[str] = field(default_factory=list) + + +_STATUS_MAP = { + "Rumored": "rumored", + "Planned": "planned", + "In Production": "in_production", + "Post Production": "post_production", + "Released": "released", + "Canceled": "canceled", +} + + +def normalize_movie(raw: dict) -> NormalizedMovie: + credits_ = raw.get("credits") or {} + images = raw.get("images") or {} + videos = (raw.get("videos") or {}).get("results") or [] + keywords = (raw.get("keywords") or {}).get("keywords") or [] + alt_titles = (raw.get("alternative_titles") or {}).get("titles") or [] + ext_ids = raw.get("external_ids") or {} + collection = raw.get("belongs_to_collection") + recommendations = (raw.get("recommendations") or {}).get("results") or [] + + external_ids: dict[str, str] = {} + if raw.get("imdb_id"): + external_ids["imdb"] = raw["imdb_id"] + elif ext_ids.get("imdb_id"): + external_ids["imdb"] = ext_ids["imdb_id"] + if ext_ids.get("wikidata_id"): + external_ids["wikidata"] = ext_ids["wikidata_id"] + + cast = [ + NormalizedPersonRef( + external_id=str(c["id"]), + name=c["name"], + profile_path=_image_url(c.get("profile_path")), + department="Acting", + character=c.get("character"), + billing_order=c.get("order"), + gender=_gender_label(c.get("gender")), + ) + for c in credits_.get("cast", []) + ] + crew = [ + NormalizedPersonRef( + external_id=str(c["id"]), + name=c["name"], + profile_path=_image_url(c.get("profile_path")), + department=c.get("department") or "Crew", + job=c.get("job"), + gender=_gender_label(c.get("gender")), + ) + for c in credits_.get("crew", []) + ] + + companies = [ + NormalizedCompanyRef( + external_id=str(pc["id"]), + name=pc["name"], + country_code=pc.get("origin_country") or None, + ) + for pc in raw.get("production_companies", []) + ] + + artwork: list[NormalizedArtwork] = [] + if raw.get("poster_path"): + artwork.append( + NormalizedArtwork( + "poster", + _image_url_required(raw["poster_path"]), + raw["poster_path"], + None, + None, + None, + None, + ) + ) + if raw.get("backdrop_path"): + artwork.append( + NormalizedArtwork( + "backdrop", + _image_url_required(raw["backdrop_path"]), + raw["backdrop_path"], + None, + None, + None, + None, + ) + ) + for kind, tmdb_key in (("poster", "posters"), ("backdrop", "backdrops"), ("logo", "logos")): + for img in images.get(tmdb_key, []): + artwork.append( + NormalizedArtwork( + kind, + _image_url_required(img["file_path"]), + img["file_path"], + img.get("width"), + img.get("height"), + img.get("aspect_ratio"), + img.get("iso_639_1"), + img.get("vote_average"), + ) + ) + + normalized_videos = [ + NormalizedVideo( + video_type=(v.get("type") or "other").lower().replace(" ", "_"), + provider_video_id=v["id"], + site=v.get("site"), + url=_video_url(v), + language=v.get("iso_639_1"), + country=v.get("iso_3166_1"), + is_official=bool(v.get("official")), + published_at=_parse_date(v.get("published_at", "")[:10] if v.get("published_at") else None), + ) + for v in videos + ] + + return NormalizedMovie( + external_id=str(raw["id"]), + title=raw.get("title") or raw.get("original_title") or "", + original_title=raw.get("original_title"), + overview=raw.get("overview") or None, + tagline=raw.get("tagline") or None, + status=_STATUS_MAP.get(raw.get("status", "Released"), "released"), + original_language=raw.get("original_language"), + release_date=_parse_date(raw.get("release_date")), + runtime_minutes=raw.get("runtime"), + budget=raw.get("budget") or None, + revenue=raw.get("revenue") or None, + popularity=raw.get("popularity"), + vote_average=raw.get("vote_average"), + vote_count=raw.get("vote_count"), + adult=bool(raw.get("adult", False)), + genres=[g["name"] for g in raw.get("genres", [])], + keywords=[k["name"] for k in keywords], + alternate_titles=[t["title"] for t in alt_titles if t.get("title")], + companies=companies, + cast=cast, + crew=crew, + artwork=artwork, + videos=normalized_videos, + external_ids=external_ids, + collection=(str(collection["id"]), collection["name"]) if collection else None, + recommendation_external_ids=[str(m["id"]) for m in recommendations], + ) + + +@dataclass +class NormalizedPerson: + external_id: str + name: str + biography: str | None + birthday: date | None + deathday: date | None + birthplace: str | None + gender: str | None + known_for_department: str | None + popularity: float | None + aliases: list[str] + artwork: list[NormalizedArtwork] + external_ids: dict[str, str] + + +def _gender_label(tmdb_gender: int | None) -> str | None: + return {1: "female", 2: "male", 3: "non_binary"}.get(tmdb_gender or 0) + + +def _video_url(video: dict) -> str: + site = (video.get("site") or "").lower() + key = video.get("key", "") + if site == "youtube": + return f"https://www.youtube.com/watch?v={key}" + if site == "vimeo": + return f"https://vimeo.com/{key}" + return key + + +def normalize_person(raw: dict) -> NormalizedPerson: + images = (raw.get("images") or {}).get("profiles") or [] + ext_ids = raw.get("external_ids") or {} + external_ids: dict[str, str] = {} + if raw.get("imdb_id"): + external_ids["imdb"] = raw["imdb_id"] + elif ext_ids.get("imdb_id"): + external_ids["imdb"] = ext_ids["imdb_id"] + if ext_ids.get("wikidata_id"): + external_ids["wikidata"] = ext_ids["wikidata_id"] + + artwork = [ + NormalizedArtwork( + "profile", + _image_url_required(img["file_path"]), + img["file_path"], + img.get("width"), + img.get("height"), + img.get("aspect_ratio"), + img.get("iso_639_1"), + img.get("vote_average"), + ) + for img in images + ] + if raw.get("profile_path") and not any(a.provider_asset_id == raw["profile_path"] for a in artwork): + artwork.insert( + 0, + NormalizedArtwork( + "profile", + _image_url_required(raw["profile_path"]), + raw["profile_path"], + None, + None, + None, + None, + ), + ) + + also_known_as = raw.get("also_known_as") or [] + + return NormalizedPerson( + external_id=str(raw["id"]), + name=raw.get("name", ""), + biography=raw.get("biography") or None, + birthday=_parse_date(raw.get("birthday")), + deathday=_parse_date(raw.get("deathday")), + birthplace=raw.get("place_of_birth") or None, + gender=_gender_label(raw.get("gender")), + known_for_department=raw.get("known_for_department"), + popularity=raw.get("popularity"), + aliases=list(also_known_as), + artwork=artwork, + external_ids=external_ids, + ) diff --git a/app/services/movie_service.py b/app/services/movie_service.py new file mode 100644 index 0000000..7cd6b09 --- /dev/null +++ b/app/services/movie_service.py @@ -0,0 +1,147 @@ +"""Assembles Movie API responses from repository data.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id, parse_aurora_id +from app.models.enums import ArtworkType +from app.repositories import movie_repository, polymorphic +from app.schemas.common import CollectionBriefOut, CompanyBriefOut, GenreOut, KeywordOut +from app.schemas.movie import ( + MovieCreditsOut, + MovieDetailOut, + MovieImagesOut, + MovieSummaryOut, + MovieVideosOut, +) +from app.services.common import primary_artwork_path, split_credits, to_images_out, to_video_out + + +class MovieNotFoundError(Exception): + pass + + +async def _resolve_movie_id(movie_aurora_id: str) -> uuid.UUID: + try: + return uuid.UUID(parse_aurora_id(movie_aurora_id, expected_type="movie")) + except ValueError as exc: + raise MovieNotFoundError(str(exc)) from exc + + +async def get_movie_detail(session: AsyncSession, movie_aurora_id: str) -> MovieDetailOut: + movie_id = await _resolve_movie_id(movie_aurora_id) + movie = await movie_repository.get_movie_by_id(session, movie_id) + if movie is None: + raise MovieNotFoundError(movie_aurora_id) + + genres = await movie_repository.get_movie_genres(session, movie_id) + keywords = await movie_repository.get_movie_keywords(session, movie_id) + companies = await movie_repository.get_movie_companies(session, movie_id) + credits_ = await polymorphic.get_credits(session, "movie", movie_id) + artwork = await polymorphic.get_artwork(session, "movie", movie_id) + videos = await polymorphic.get_videos(session, "movie", movie_id) + external_ids = await polymorphic.get_external_ids(session, "movie", movie_id) + + cast, crew = split_credits(credits_) + + return MovieDetailOut( + id=aurora_id("movie", str(movie.id)), + title=movie.title, + original_title=movie.original_title, + overview=movie.overview, + tagline=movie.tagline, + status=movie.status.value, + original_language=movie.original_language_code, + release_date=movie.primary_release_date, + runtime=movie.runtime_minutes, + budget=movie.budget, + revenue=movie.revenue, + popularity=movie.popularity, + vote_average=movie.vote_average, + vote_count=movie.vote_count, + adult=movie.adult, + genres=[GenreOut(id=aurora_id("genre", str(g.id)), name=g.name) for g in genres], + keywords=[KeywordOut(id=aurora_id("keyword", str(k.id)), name=k.name) for k in keywords], + cast=cast, + crew=crew, + companies=[ + CompanyBriefOut( + id=aurora_id("company", str(company.id)), + name=company.name, + company_type=company.company_type.value, + logo_path=None, + ) + for company, _role in companies + ], + collection=( + CollectionBriefOut( + id=aurora_id("collection", str(movie.collection.id)), name=movie.collection.name + ) + if movie.collection + else None + ), + images=to_images_out(artwork), + videos=[to_video_out(v) for v in videos], + external_ids=external_ids, + created_at=movie.created_at, + updated_at=movie.updated_at, + ) + + +async def get_movie_credits(session: AsyncSession, movie_aurora_id: str) -> MovieCreditsOut: + movie_id = await _resolve_movie_id(movie_aurora_id) + movie = await movie_repository.get_movie_by_id(session, movie_id) + if movie is None: + raise MovieNotFoundError(movie_aurora_id) + credits_ = await polymorphic.get_credits(session, "movie", movie_id) + cast, crew = split_credits(credits_) + return MovieCreditsOut(id=aurora_id("movie", str(movie.id)), cast=cast, crew=crew) + + +async def get_movie_images(session: AsyncSession, movie_aurora_id: str) -> MovieImagesOut: + movie_id = await _resolve_movie_id(movie_aurora_id) + movie = await movie_repository.get_movie_by_id(session, movie_id) + if movie is None: + raise MovieNotFoundError(movie_aurora_id) + artwork = await polymorphic.get_artwork(session, "movie", movie_id) + return MovieImagesOut(id=aurora_id("movie", str(movie.id)), images=to_images_out(artwork)) + + +async def get_movie_videos(session: AsyncSession, movie_aurora_id: str) -> MovieVideosOut: + movie_id = await _resolve_movie_id(movie_aurora_id) + movie = await movie_repository.get_movie_by_id(session, movie_id) + if movie is None: + raise MovieNotFoundError(movie_aurora_id) + videos = await polymorphic.get_videos(session, "movie", movie_id) + return MovieVideosOut(id=aurora_id("movie", str(movie.id)), videos=[to_video_out(v) for v in videos]) + + +async def build_movie_summary(session: AsyncSession, movie_id: uuid.UUID) -> MovieSummaryOut | None: + movie = await movie_repository.get_movie_by_id(session, movie_id) + if movie is None: + return None + posters = await polymorphic.get_artwork(session, "movie", movie_id, ArtworkType.POSTER) + return MovieSummaryOut( + id=aurora_id("movie", str(movie.id)), + title=movie.title, + original_title=movie.original_title, + overview=movie.overview, + release_date=movie.primary_release_date, + poster_path=primary_artwork_path(posters, ArtworkType.POSTER), + popularity=movie.popularity, + vote_average=movie.vote_average, + ) + + +async def build_movie_summaries( + session: AsyncSession, movie_ids: list[uuid.UUID] +) -> list[MovieSummaryOut]: + out: list[MovieSummaryOut] = [] + for movie_id in movie_ids: + summary = await build_movie_summary(session, movie_id) + if summary is not None: + out.append(summary) + return out diff --git a/app/services/person_service.py b/app/services/person_service.py new file mode 100644 index 0000000..dee1b7f --- /dev/null +++ b/app/services/person_service.py @@ -0,0 +1,127 @@ +"""Assembles Person API responses from repository data.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id, parse_aurora_id +from app.models.enums import ArtworkType +from app.repositories import movie_repository, person_repository, polymorphic +from app.schemas.person import ( + FilmographyCreditOut, + PersonCreditsOut, + PersonDetailOut, + PersonSummaryOut, +) +from app.services.common import primary_artwork_path, to_artwork_out + + +class PersonNotFoundError(Exception): + pass + + +def _resolve_person_id(aid: str) -> uuid.UUID: + try: + return uuid.UUID(parse_aurora_id(aid, expected_type="person")) + except ValueError as exc: + raise PersonNotFoundError(str(exc)) from exc + + +async def get_person_detail(session: AsyncSession, person_aurora_id: str) -> PersonDetailOut: + person_id = _resolve_person_id(person_aurora_id) + person = await person_repository.get_person_by_id(session, person_id) + if person is None: + raise PersonNotFoundError(person_aurora_id) + + profiles = await polymorphic.get_artwork(session, "person", person_id, ArtworkType.PROFILE) + external_ids = await polymorphic.get_external_ids(session, "person", person_id) + + return PersonDetailOut( + id=aurora_id("person", str(person.id)), + name=person.name, + aliases=list(person.aliases or []), + biography=person.biography, + birthday=person.birthday, + deathday=person.deathday, + birthplace=person.birthplace, + gender=person.gender, + known_for_department=person.known_for_department, + popularity=person.popularity, + profile_images=[to_artwork_out(a) for a in profiles], + external_ids=external_ids, + created_at=person.created_at, + updated_at=person.updated_at, + ) + + +async def get_person_credits(session: AsyncSession, person_aurora_id: str) -> PersonCreditsOut: + person_id = _resolve_person_id(person_aurora_id) + person = await person_repository.get_person_by_id(session, person_id) + if person is None: + raise PersonNotFoundError(person_aurora_id) + + credits_ = await person_repository.get_person_credits(session, person_id) + + movie_ids = [c.entity_id for c in credits_ if c.entity_type == "movie"] + movies = {m.id: m for m in await movie_repository.get_movies_by_ids(session, movie_ids)} + + cast: list[FilmographyCreditOut] = [] + crew: list[FilmographyCreditOut] = [] + + for credit in credits_: + if credit.entity_type != "movie": + # Series/episode filmography follows the same shape; movies are + # implemented first per the Core 0.1 milestone (section 34). + continue + movie = movies.get(credit.entity_id) + if movie is None: + continue + posters = await polymorphic.get_artwork( + session, "movie", movie.id, ArtworkType.POSTER + ) + item = FilmographyCreditOut( + id=aurora_id("credit", str(credit.id)), + media_id=aurora_id("movie", str(movie.id)), + media_type="movie", + title=movie.title, + department=credit.department, + job=credit.job, + character=credit.character, + release_date=movie.primary_release_date, + poster_path=primary_artwork_path(posters, ArtworkType.POSTER), + ) + if credit.department == "Acting": + cast.append(item) + else: + crew.append(item) + + return PersonCreditsOut(id=aurora_id("person", str(person.id)), cast=cast, crew=crew) + + +async def build_person_summary( + session: AsyncSession, person_id: uuid.UUID +) -> PersonSummaryOut | None: + person = await person_repository.get_person_by_id(session, person_id) + if person is None: + return None + profiles = await polymorphic.get_artwork(session, "person", person_id, ArtworkType.PROFILE) + return PersonSummaryOut( + id=aurora_id("person", str(person.id)), + name=person.name, + known_for_department=person.known_for_department, + profile_path=primary_artwork_path(profiles, ArtworkType.PROFILE), + popularity=person.popularity, + ) + + +async def build_person_summaries( + session: AsyncSession, person_ids: list[uuid.UUID] +) -> list[PersonSummaryOut]: + out: list[PersonSummaryOut] = [] + for person_id in person_ids: + summary = await build_person_summary(session, person_id) + if summary is not None: + out.append(summary) + return out diff --git a/app/services/search/__init__.py b/app/services/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/search/base.py b/app/services/search/base.py new file mode 100644 index 0000000..5646c5b --- /dev/null +++ b/app/services/search/base.py @@ -0,0 +1,40 @@ +"""Search provider abstraction (section 16). + +API clients and the rest of the service layer talk to `SearchProvider`, not +to PostgreSQL directly, so the underlying engine can be swapped later (e.g. +for an `OpenSearchProvider`) without touching callers. +""" + +from __future__ import annotations + +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + +from sqlalchemy.ext.asyncio import AsyncSession + + +@dataclass +class SearchMatch: + entity_id: uuid.UUID + score: float + + +@dataclass +class SearchResultIDs: + movies: list[SearchMatch] = field(default_factory=list) + series: list[SearchMatch] = field(default_factory=list) + people: list[SearchMatch] = field(default_factory=list) + collections: list[SearchMatch] = field(default_factory=list) + companies: list[SearchMatch] = field(default_factory=list) + + +class SearchProvider(ABC): + """Returns ranked Aurora entity IDs matching a query. Never returns + fully-hydrated entities — that's the SearchService's job, so the + provider stays a thin, swappable ranking layer.""" + + @abstractmethod + async def search( + self, session: AsyncSession, query: str, *, limit_per_type: int = 10 + ) -> SearchResultIDs: ... diff --git a/app/services/search/postgresql_provider.py b/app/services/search/postgresql_provider.py new file mode 100644 index 0000000..225814d --- /dev/null +++ b/app/services/search/postgresql_provider.py @@ -0,0 +1,121 @@ +"""PostgreSQL-backed search: full-text search ranking blended with trigram +similarity for typo tolerance, over primary titles/names and alternate +titles. Requires the `pg_trgm` extension and the functional GIN indexes +created in the initial migration. +""" + +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.search.base import SearchMatch, SearchProvider, SearchResultIDs + +_MOVIE_SQL = text( + """ + WITH scored AS ( + SELECT m.id AS entity_id, + GREATEST( + ts_rank_cd(to_tsvector('simple', coalesce(m.title, '') || ' ' || coalesce(m.original_title, '')), plainto_tsquery('simple', :q)), + similarity(m.title, :q) + ) AS score + FROM movies m + WHERE to_tsvector('simple', coalesce(m.title, '') || ' ' || coalesce(m.original_title, '')) @@ plainto_tsquery('simple', :q) + OR m.title % :q + UNION ALL + SELECT mat.movie_id AS entity_id, similarity(mat.title, :q) AS score + FROM movie_alternate_titles mat + WHERE mat.title % :q + ) + SELECT entity_id, MAX(score) AS score + FROM scored + GROUP BY entity_id + ORDER BY score DESC + LIMIT :limit + """ +) + +_SERIES_SQL = text( + """ + WITH scored AS ( + SELECT s.id AS entity_id, + GREATEST( + ts_rank_cd(to_tsvector('simple', coalesce(s.title, '') || ' ' || coalesce(s.original_title, '')), plainto_tsquery('simple', :q)), + similarity(s.title, :q) + ) AS score + FROM series s + WHERE to_tsvector('simple', coalesce(s.title, '') || ' ' || coalesce(s.original_title, '')) @@ plainto_tsquery('simple', :q) + OR s.title % :q + UNION ALL + SELECT sat.series_id AS entity_id, similarity(sat.title, :q) AS score + FROM series_alternate_titles sat + WHERE sat.title % :q + ) + SELECT entity_id, MAX(score) AS score + FROM scored + GROUP BY entity_id + ORDER BY score DESC + LIMIT :limit + """ +) + +_PEOPLE_SQL = text( + """ + SELECT p.id AS entity_id, + GREATEST( + ts_rank_cd(to_tsvector('simple', p.name), plainto_tsquery('simple', :q)), + similarity(p.name, :q) + ) AS score + FROM people p + WHERE to_tsvector('simple', p.name) @@ plainto_tsquery('simple', :q) + OR p.name % :q + ORDER BY score DESC + LIMIT :limit + """ +) + +_COLLECTIONS_SQL = text( + """ + SELECT c.id AS entity_id, similarity(c.name, :q) AS score + FROM collections c + WHERE c.name % :q OR to_tsvector('simple', c.name) @@ plainto_tsquery('simple', :q) + ORDER BY score DESC + LIMIT :limit + """ +) + +_COMPANIES_SQL = text( + """ + SELECT co.id AS entity_id, similarity(co.name, :q) AS score + FROM companies co + WHERE co.name % :q OR to_tsvector('simple', co.name) @@ plainto_tsquery('simple', :q) + ORDER BY score DESC + LIMIT :limit + """ +) + + +class PostgreSQLSearchProvider(SearchProvider): + async def search( + self, session: AsyncSession, query: str, *, limit_per_type: int = 10 + ) -> SearchResultIDs: + params = {"q": query, "limit": limit_per_type} + movies = await session.execute(_MOVIE_SQL, params) + series = await session.execute(_SERIES_SQL, params) + people = await session.execute(_PEOPLE_SQL, params) + collections = await session.execute(_COLLECTIONS_SQL, params) + companies = await session.execute(_COMPANIES_SQL, params) + + def to_matches(rows) -> list[SearchMatch]: + return [ + SearchMatch(entity_id=row.entity_id, score=float(row.score or 0.0)) + for row in rows + ] + + return SearchResultIDs( + movies=to_matches(movies), + series=to_matches(series), + people=to_matches(people), + collections=to_matches(collections), + companies=to_matches(companies), + ) diff --git a/app/services/search/service.py b/app/services/search/service.py new file mode 100644 index 0000000..d04f1c0 --- /dev/null +++ b/app/services/search/service.py @@ -0,0 +1,49 @@ +"""Search service: runs a `SearchProvider` and hydrates matches into full +response summaries via the entity services. API clients never see the +underlying search engine (section 16).""" + +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.schemas.search import SearchResultsOut +from app.services.collection_service import build_collection_summaries +from app.services.company_service import build_company_summaries +from app.services.movie_service import build_movie_summaries +from app.services.person_service import build_person_summaries +from app.services.search.base import SearchProvider +from app.services.tv_service import build_series_summaries + + +class SearchService: + def __init__(self, provider: SearchProvider) -> None: + self._provider = provider + + async def search( + self, session: AsyncSession, query: str, *, limit_per_type: int = 10 + ) -> SearchResultsOut: + query = query.strip() + if not query: + return SearchResultsOut(query=query) + + results = await self._provider.search(session, query, limit_per_type=limit_per_type) + + movies = await build_movie_summaries(session, [m.entity_id for m in results.movies]) + series = await build_series_summaries(session, [s.entity_id for s in results.series]) + people = await build_person_summaries(session, [p.entity_id for p in results.people]) + collections = await build_collection_summaries( + session, [c.entity_id for c in results.collections] + ) + companies = await build_company_summaries( + session, [c.entity_id for c in results.companies] + ) + + return SearchResultsOut( + query=query, + movies=movies, + series=series, + people=people, + collections=collections, + companies=companies, + total_results=len(movies) + len(series) + len(people) + len(collections) + len(companies), + ) diff --git a/app/services/tv_service.py b/app/services/tv_service.py new file mode 100644 index 0000000..2300282 --- /dev/null +++ b/app/services/tv_service.py @@ -0,0 +1,229 @@ +"""Assembles Series/Season/Episode API responses from repository data.""" + +from __future__ import annotations + +import uuid + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id, parse_aurora_id +from app.models.enums import ArtworkType, CompanyType +from app.repositories import polymorphic, tv_repository +from app.schemas.common import CompanyBriefOut, GenreOut, KeywordOut +from app.schemas.tv import ( + EpisodeBriefOut, + EpisodeDetailOut, + SeasonBriefOut, + SeasonDetailOut, + SeriesDetailOut, + SeriesSeasonsOut, + SeriesSummaryOut, +) +from app.services.common import primary_artwork_path, split_credits, to_images_out, to_video_out + + +class SeriesNotFoundError(Exception): + pass + + +class SeasonNotFoundError(Exception): + pass + + +class EpisodeNotFoundError(Exception): + pass + + +def _resolve_id(aid: str, expected_type: str, error_cls: type[Exception]) -> uuid.UUID: + try: + return uuid.UUID(parse_aurora_id(aid, expected_type=expected_type)) + except ValueError as exc: + raise error_cls(str(exc)) from exc + + +async def _season_brief(session: AsyncSession, season) -> SeasonBriefOut: + posters = await polymorphic.get_artwork(session, "season", season.id, ArtworkType.POSTER) + return SeasonBriefOut( + id=aurora_id("season", str(season.id)), + season_number=season.season_number, + title=season.title, + episode_count=season.episode_count, + air_date=season.air_date, + poster_path=primary_artwork_path(posters, ArtworkType.POSTER), + ) + + +async def get_series_detail(session: AsyncSession, series_aurora_id: str) -> SeriesDetailOut: + series_id = _resolve_id(series_aurora_id, "series", SeriesNotFoundError) + series = await tv_repository.get_series_by_id(session, series_id) + if series is None: + raise SeriesNotFoundError(series_aurora_id) + + genres = await tv_repository.get_series_genres(session, series_id) + keywords = await tv_repository.get_series_keywords(session, series_id) + networks = await tv_repository.get_series_companies(session, series_id, CompanyType.NETWORK) + companies = await tv_repository.get_series_companies( + session, series_id, CompanyType.PRODUCTION_COMPANY + ) + seasons = await tv_repository.get_seasons_for_series(session, series_id) + credits_ = await polymorphic.get_credits(session, "series", series_id) + artwork = await polymorphic.get_artwork(session, "series", series_id) + videos = await polymorphic.get_videos(session, "series", series_id) + external_ids = await polymorphic.get_external_ids(session, "series", series_id) + + cast, crew = split_credits([c for c in credits_ if c.department != "Creator"]) + _, creators = split_credits([c for c in credits_ if c.department == "Creator"]) + + season_briefs = [await _season_brief(session, s) for s in seasons] + + return SeriesDetailOut( + id=aurora_id("series", str(series.id)), + title=series.title, + original_title=series.original_title, + overview=series.overview, + tagline=series.tagline, + status=series.status.value, + original_language=series.original_language_code, + first_air_date=series.first_air_date, + last_air_date=series.last_air_date, + number_of_seasons=series.number_of_seasons, + number_of_episodes=series.number_of_episodes, + popularity=series.popularity, + vote_average=series.vote_average, + vote_count=series.vote_count, + genres=[GenreOut(id=aurora_id("genre", str(g.id)), name=g.name) for g in genres], + keywords=[KeywordOut(id=aurora_id("keyword", str(k.id)), name=k.name) for k in keywords], + creators=creators, + cast=cast, + crew=crew, + networks=[ + CompanyBriefOut( + id=aurora_id("company", str(c.id)), name=c.name, company_type=c.company_type.value + ) + for c, _role in networks + ], + companies=[ + CompanyBriefOut( + id=aurora_id("company", str(c.id)), name=c.name, company_type=c.company_type.value + ) + for c, _role in companies + ], + seasons=season_briefs, + images=to_images_out(artwork), + videos=[to_video_out(v) for v in videos], + external_ids=external_ids, + created_at=series.created_at, + updated_at=series.updated_at, + ) + + +async def get_series_seasons(session: AsyncSession, series_aurora_id: str) -> SeriesSeasonsOut: + series_id = _resolve_id(series_aurora_id, "series", SeriesNotFoundError) + series = await tv_repository.get_series_by_id(session, series_id) + if series is None: + raise SeriesNotFoundError(series_aurora_id) + seasons = await tv_repository.get_seasons_for_series(session, series_id) + briefs = [await _season_brief(session, s) for s in seasons] + return SeriesSeasonsOut(id=aurora_id("series", str(series.id)), seasons=briefs) + + +async def get_season_detail(session: AsyncSession, season_aurora_id: str) -> SeasonDetailOut: + season_id = _resolve_id(season_aurora_id, "season", SeasonNotFoundError) + season = await tv_repository.get_season_by_id(session, season_id) + if season is None: + raise SeasonNotFoundError(season_aurora_id) + + episodes = await tv_repository.get_episodes_for_season(session, season_id) + artwork = await polymorphic.get_artwork(session, "season", season_id) + external_ids = await polymorphic.get_external_ids(session, "season", season_id) + + episode_briefs: list[EpisodeBriefOut] = [] + for ep in episodes: + stills = await polymorphic.get_artwork(session, "episode", ep.id, ArtworkType.STILL) + episode_briefs.append( + EpisodeBriefOut( + id=aurora_id("episode", str(ep.id)), + episode_number=ep.episode_number, + title=ep.title, + air_date=ep.air_date, + still_path=primary_artwork_path(stills, ArtworkType.STILL), + ) + ) + + return SeasonDetailOut( + id=aurora_id("season", str(season.id)), + series_id=aurora_id("series", str(season.series_id)), + season_number=season.season_number, + title=season.title, + overview=season.overview, + air_date=season.air_date, + episode_count=season.episode_count, + episodes=episode_briefs, + images=to_images_out(artwork), + external_ids=external_ids, + created_at=season.created_at, + updated_at=season.updated_at, + ) + + +async def get_episode_detail(session: AsyncSession, episode_aurora_id: str) -> EpisodeDetailOut: + episode_id = _resolve_id(episode_aurora_id, "episode", EpisodeNotFoundError) + episode = await tv_repository.get_episode_by_id(session, episode_id) + if episode is None: + raise EpisodeNotFoundError(episode_aurora_id) + + credits_ = await polymorphic.get_credits(session, "episode", episode_id) + artwork = await polymorphic.get_artwork(session, "episode", episode_id) + external_ids = await polymorphic.get_external_ids(session, "episode", episode_id) + + main_cast, crew = split_credits([c for c in credits_ if not c.is_guest]) + guest_cast, _ = split_credits([c for c in credits_ if c.is_guest]) + + return EpisodeDetailOut( + id=aurora_id("episode", str(episode.id)), + series_id=aurora_id("series", str(episode.series_id)), + season_id=aurora_id("season", str(episode.season_id)), + season_number=episode.season_number, + episode_number=episode.episode_number, + title=episode.title, + overview=episode.overview, + air_date=episode.air_date, + runtime=episode.runtime_minutes, + cast=main_cast, + guest_stars=guest_cast, + crew=crew, + images=to_images_out(artwork), + external_ids=external_ids, + created_at=episode.created_at, + updated_at=episode.updated_at, + ) + + +async def build_series_summary( + session: AsyncSession, series_id: uuid.UUID +) -> SeriesSummaryOut | None: + series = await tv_repository.get_series_by_id(session, series_id) + if series is None: + return None + posters = await polymorphic.get_artwork(session, "series", series_id, ArtworkType.POSTER) + return SeriesSummaryOut( + id=aurora_id("series", str(series.id)), + title=series.title, + original_title=series.original_title, + overview=series.overview, + first_air_date=series.first_air_date, + poster_path=primary_artwork_path(posters, ArtworkType.POSTER), + popularity=series.popularity, + vote_average=series.vote_average, + ) + + +async def build_series_summaries( + session: AsyncSession, series_ids: list[uuid.UUID] +) -> list[SeriesSummaryOut]: + out: list[SeriesSummaryOut] = [] + for series_id in series_ids: + summary = await build_series_summary(session, series_id) + if summary is not None: + out.append(summary) + return out diff --git a/app/workers/__init__.py b/app/workers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/workers/tasks.py b/app/workers/tasks.py new file mode 100644 index 0000000..7e03d07 --- /dev/null +++ b/app/workers/tasks.py @@ -0,0 +1,22 @@ +"""arq task functions. Thin wrappers around the ingestion runner so retry/ +concurrency/observability behavior lives in arq's worker loop, not here. +""" + +from __future__ import annotations + +from typing import Any + +from app.core.logging import get_logger +from app.services.ingestion.runner import run_movie_ingestion, run_person_ingestion + +logger = get_logger(__name__) + + +async def ingest_movie_task(ctx: dict[str, Any], external_id: str) -> str: + movie_id = await run_movie_ingestion(external_id) + return str(movie_id) + + +async def ingest_person_task(ctx: dict[str, Any], external_id: str) -> str: + person_id = await run_person_ingestion(external_id) + return str(person_id) diff --git a/app/workers/worker.py b/app/workers/worker.py new file mode 100644 index 0000000..8e762a0 --- /dev/null +++ b/app/workers/worker.py @@ -0,0 +1,37 @@ +"""arq worker process entrypoint. + +Run with: `arq app.workers.worker.WorkerSettings` + +Workers scale independently of the API (section 1) — run as many worker +processes/containers as needed; arq's Redis-backed queue coordinates them so +each job is picked up by exactly one worker. +""" + +from __future__ import annotations + +from arq.connections import RedisSettings + +from app.core.config import get_settings +from app.core.logging import configure_logging, get_logger +from app.workers.tasks import ingest_movie_task, ingest_person_task + +logger = get_logger(__name__) + + +async def startup(ctx: dict) -> None: + configure_logging() + logger.info("worker_started") + + +async def shutdown(ctx: dict) -> None: + logger.info("worker_stopped") + + +class WorkerSettings: + functions = [ingest_movie_task, ingest_person_task] + on_startup = startup + on_shutdown = shutdown + redis_settings = RedisSettings.from_dsn(get_settings().REDIS_URL) + max_jobs = get_settings().WORKER_CONCURRENCY + job_timeout = 300 + max_tries = 5 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..8660c87 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,137 @@ +# Production-oriented topology (section 1/28): Traefik -> N stateless API +# replicas -> PgBouncer -> PostgreSQL, plus Redis and independently-scaled +# workers. Requires a pre-existing external Traefik network named +# `traefik-public` and a populated `.env` (never commit real secrets). +name: aurora-core + +networks: + traefik-public: + external: true + aurora-internal: + driver: bridge + +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + env_file: .env + environment: + POSTGRES_USER: ${POSTGRES_USER:-aurora} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB:-aurora_core} + volumes: + - aurora_postgres_data:/var/lib/postgresql/data + networks: [aurora-internal] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-aurora}"] + interval: 5s + timeout: 5s + retries: 10 + deploy: + resources: + limits: { cpus: "2", memory: 2G } + + pgbouncer: + image: edoburu/pgbouncer:latest + restart: unless-stopped + environment: + DATABASE_URL: postgres://${POSTGRES_USER:-aurora}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-aurora_core} + POOL_MODE: transaction + MAX_CLIENT_CONN: "1000" + DEFAULT_POOL_SIZE: "25" + depends_on: + postgres: + condition: service_healthy + networks: [aurora-internal] + healthcheck: + test: ["CMD", "pg_isready", "-h", "localhost", "-p", "5432"] + interval: 10s + timeout: 5s + retries: 10 + + redis: + image: redis:7-alpine + restart: unless-stopped + command: ["redis-server", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"] + volumes: + - aurora_redis_data:/data + networks: [aurora-internal] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + + migrate: + build: + context: . + dockerfile: docker/api.Dockerfile + env_file: .env + environment: + DATABASE_URL_SYNC: postgresql+psycopg2://${POSTGRES_USER:-aurora}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-aurora_core} + depends_on: + postgres: + condition: service_healthy + networks: [aurora-internal] + entrypoint: ["alembic"] + command: ["upgrade", "head"] + + api: + build: + context: . + dockerfile: docker/api.Dockerfile + restart: unless-stopped + env_file: .env + environment: + AURORA_ENV: production + DATABASE_BEHIND_TRANSACTION_POOLER: "true" + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-aurora}:${POSTGRES_PASSWORD}@pgbouncer:5432/${POSTGRES_DB:-aurora_core} + REDIS_URL: redis://redis:6379/0 + depends_on: + pgbouncer: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + networks: [traefik-public, aurora-internal] + deploy: + replicas: 3 + resources: + limits: { cpus: "1", memory: 1G } + labels: + - "traefik.enable=true" + - "traefik.docker.network=traefik-public" + - "traefik.http.routers.aurora-core-api.rule=Host(`${AURORA_API_HOST:-api.aurora.local}`)" + - "traefik.http.routers.aurora-core-api.entrypoints=websecure" + - "traefik.http.routers.aurora-core-api.tls.certresolver=letsencrypt" + - "traefik.http.services.aurora-core-api.loadbalancer.server.port=8000" + - "traefik.http.services.aurora-core-api.loadbalancer.healthcheck.path=/health" + + worker: + build: + context: . + dockerfile: docker/worker.Dockerfile + restart: unless-stopped + env_file: .env + environment: + AURORA_ENV: production + DATABASE_BEHIND_TRANSACTION_POOLER: "true" + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-aurora}:${POSTGRES_PASSWORD}@pgbouncer:5432/${POSTGRES_DB:-aurora_core} + REDIS_URL: redis://redis:6379/0 + depends_on: + pgbouncer: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + networks: [aurora-internal] + deploy: + replicas: 2 + resources: + limits: { cpus: "1", memory: 1G } + +volumes: + aurora_postgres_data: + aurora_redis_data: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..679b09e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,92 @@ +# Local development environment. For a production-shaped topology +# (PgBouncer, multiple API replicas, Traefik) see docker-compose.prod.yml. +name: aurora-core + +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_USER: aurora + POSTGRES_PASSWORD: aurora + POSTGRES_DB: aurora_core + ports: + - "5432:5432" + volumes: + - aurora_postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U aurora -d aurora_core"] + interval: 5s + timeout: 5s + retries: 10 + + redis: + image: redis:7-alpine + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + + migrate: + build: + context: . + dockerfile: docker/api.Dockerfile + env_file: .env + environment: + DATABASE_URL_SYNC: postgresql+psycopg2://aurora:aurora@postgres:5432/aurora_core + depends_on: + postgres: + condition: service_healthy + entrypoint: ["alembic"] + command: ["upgrade", "head"] + + api: + build: + context: . + dockerfile: docker/api.Dockerfile + restart: unless-stopped + env_file: .env + environment: + DATABASE_URL: postgresql+asyncpg://aurora:aurora@postgres:5432/aurora_core + DATABASE_URL_SYNC: postgresql+psycopg2://aurora:aurora@postgres:5432/aurora_core + REDIS_URL: redis://redis:6379/0 + ports: + - "8000:8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + command: + ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + volumes: + - ./app:/app/app + + worker: + build: + context: . + dockerfile: docker/worker.Dockerfile + restart: unless-stopped + env_file: .env + environment: + DATABASE_URL: postgresql+asyncpg://aurora:aurora@postgres:5432/aurora_core + DATABASE_URL_SYNC: postgresql+psycopg2://aurora:aurora@postgres:5432/aurora_core + REDIS_URL: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + volumes: + - ./app:/app/app + +volumes: + aurora_postgres_data: diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile new file mode 100644 index 0000000..1bf3af5 --- /dev/null +++ b/docker/api.Dockerfile @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl libpq5 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +FROM base AS builder +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential libpq-dev \ + && rm -rf /var/lib/apt/lists/* +COPY pyproject.toml ./ +COPY app ./app +RUN pip install --no-cache-dir . + +FROM base AS runtime +RUN useradd --create-home --uid 1000 aurora +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin +COPY app ./app +COPY migrations ./migrations +COPY alembic.ini ./alembic.ini + +USER aurora + +EXPOSE 8000 + +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=5 \ + CMD curl -fsS http://localhost:8000/health || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] diff --git a/docker/worker.Dockerfile b/docker/worker.Dockerfile new file mode 100644 index 0000000..a525904 --- /dev/null +++ b/docker/worker.Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libpq5 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +FROM base AS builder +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential libpq-dev \ + && rm -rf /var/lib/apt/lists/* +COPY pyproject.toml ./ +COPY app ./app +RUN pip install --no-cache-dir . + +FROM base AS runtime +RUN useradd --create-home --uid 1000 aurora +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin +COPY app ./app + +USER aurora + +CMD ["arq", "app.workers.worker.WorkerSettings"] diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..9520205 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,143 @@ +# Aurora Core — API + +Base path: `/api/v1`. Interactive OpenAPI docs are available at `/docs` in +any environment where `AURORA_API_DOCS_ENABLED=true` (default; set to +`false` to hide docs in production — section 32). + +All responses are JSON, built from Pydantic response schemas +(`app/schemas/`), never from serialized ORM objects (section 18). Errors use +a consistent envelope: + +```json +{ "error": { "code": "movie_not_found", "message": "No such movie: aur_movie_..." } } +``` + +## Health + +| Method | Path | Notes | +|---|---|---| +| GET | `/health` and `/api/v1/health` | Liveness only. | +| GET | `/ready` and `/api/v1/ready` | Checks PostgreSQL (always) and Redis (if `REDIS_REQUIRED=true`). | + +## Search + +| Method | Path | Notes | +|---|---|---| +| GET | `/api/v1/search?q=...&limit=10` | Searches movies, series, people, collections, companies. `limit` (1-50) caps results **per type**. | + +```json +GET /api/v1/search?q=the%20matrix + +{ + "query": "the matrix", + "movies": [ { "id": "aur_movie_...", "type": "movie", "title": "The Matrix", "release_date": "1999-03-30", "poster_path": "https://..." } ], + "series": [], + "people": [], + "collections": [ { "id": "aur_collection_...", "name": "The Matrix Collection" } ], + "companies": [], + "total_results": 2 +} +``` + +## Movies + +| Method | Path | +|---|---| +| GET | `/api/v1/movies/{aurora_id}` | +| GET | `/api/v1/movies/{aurora_id}/credits` | +| GET | `/api/v1/movies/{aurora_id}/images` | +| GET | `/api/v1/movies/{aurora_id}/videos` | + +`GET /api/v1/movies/{aurora_id}` (cached, see `docs/architecture.md#caching`): + +```json +{ + "id": "aur_movie_01m1q7p1c...", + "type": "movie", + "title": "The Matrix", + "original_title": "The Matrix", + "overview": "...", + "tagline": "Welcome to the Real World", + "status": "released", + "original_language": "en", + "release_date": "1999-03-30", + "runtime": 136, + "budget": 63000000, + "revenue": 463517383, + "popularity": 45.5, + "vote_average": 8.2, + "vote_count": 24000, + "adult": false, + "genres": [{ "id": "aur_genre_...", "name": "Action" }], + "keywords": [{ "id": "aur_keyword_...", "name": "dystopia" }], + "cast": [{ "id": "aur_credit_...", "person": { "id": "aur_person_...", "name": "Keanu Reeves", "profile_path": null }, "character": "Neo", "billing_order": 0 }], + "crew": [{ "id": "aur_credit_...", "person": { "id": "aur_person_...", "name": "Lana Wachowski" }, "department": "Directing", "job": "Director" }], + "companies": [{ "id": "aur_company_...", "name": "Village Roadshow Pictures", "company_type": "production_company" }], + "collection": { "id": "aur_collection_...", "name": "The Matrix Collection" }, + "images": { "posters": [...], "backdrops": [...], "logos": [...] }, + "videos": [{ "id": "aur_vid_...", "type": "trailer", "site": "YouTube", "url": "https://www.youtube.com/watch?v=...", "is_official": true }], + "external_ids": { "tmdb": "603", "imdb": "tt0133093" }, + "created_at": "...", + "updated_at": "..." +} +``` + +## Series / Seasons / Episodes + +| Method | Path | +|---|---| +| GET | `/api/v1/series/{aurora_id}` | +| GET | `/api/v1/series/{aurora_id}/seasons` | +| GET | `/api/v1/seasons/{aurora_id}` | +| GET | `/api/v1/episodes/{aurora_id}` | + +Series detail includes `networks` and `companies` as separate lists (a TV +network and a production company are both `Company` rows distinguished by +`company_type` — see `docs/database.md`), plus `creators` (series-level +"Creator" credits) alongside `cast`/`crew`. + +## People + +| Method | Path | +|---|---| +| GET | `/api/v1/people/{aurora_id}` | +| GET | `/api/v1/people/{aurora_id}/credits` (movie filmography in 0.1; series/episode credits are stored but not yet surfaced here — see Known limitations) | + +## Collections / Companies + +| Method | Path | +|---|---| +| GET | `/api/v1/collections/{aurora_id}` | +| GET | `/api/v1/companies/{aurora_id}` | + +## Administrative (section 17: "separately protected") + +Require header `X-API-Key: `. Never exposed to +public clients — mount behind a separate Traefik router/ACL in production if +you want network-level isolation too. + +| Method | Path | Body | Notes | +|---|---|---|---| +| POST | `/api/v1/admin/ingest/movie` | `{"tmdb_id": "603"}` | Enqueues an arq job; returns `{"job_id": "...", "status": "enqueued"}` immediately (section 15: ingestion never runs inline in a request). | +| POST | `/api/v1/admin/ingest/person` | `{"tmdb_id": "6384"}` | Same, for a person profile. | +| GET | `/api/v1/admin/ingest/jobs/{job_id}` | — | Poll job status: `pending` → `running` → `succeeded`/`failed`/`retrying`. | + +## Errors + +| Status | When | +|---|---| +| 404 | Entity not found, or the ID is well-formed but the wrong entity type (`aur_person_...` passed to `/movies/{id}`) | +| 422 | Request validation failure (e.g. missing/invalid query params) | +| 429 | Rate limit exceeded (`Retry-After` header set) | +| 401 | Missing/invalid `X-API-Key` on an admin endpoint | +| 503 | Admin API called with `AURORA_ADMIN_API_KEYS` unset, or the job queue is unavailable | + +## Known limitations + +- No pagination on list-shaped responses yet beyond search's `limit` + (movie/series lists like recommendations return everything). Cursor-based + pagination using the Aurora ID's inherent creation-order sort (see + `docs/database.md#aurora-ids-vs-primary-keys`) is the natural next step. +- `/people/{id}/credits` only returns movie credits today; series/episode + credits exist in the `credits` table but the response assembly for them + is not yet wired up (`app/services/person_service.py`). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..67a9591 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,154 @@ +# Aurora Core — Architecture + +Aurora Core is a production-grade **modular monolith** (section 1/9 of +`docs/aurora-core-spec.md`) — one deployable API codebase with clear internal +module boundaries, not a set of microservices. It is the canonical metadata +layer for the Aurora ecosystem; Plex, Overseerr, Radarr, Sonarr, and future +Aurora applications are consumers of it, never dependencies of it. + +``` +Internet + | +Traefik + | +Aurora Core API replicas (stateless) + | + +--- PostgreSQL (authoritative) + | + +--- PgBouncer (production connection pooling) + | + +--- Redis (cache / rate limit / job queue — disposable) + | + +--- Background Workers (arq, scale independently) + | + +--- Metadata ingestion + +--- Search index maintenance (implicit: Postgres FTS/trigram indexes) +``` + +## Module boundaries (`app/`) + +| Module | Responsibility | +|---|---| +| `app/core/` | Config, DB engine, Redis client, structured logging, security/rate-limiting, job queue connection. No business logic. | +| `app/models/` | SQLAlchemy 2.0 ORM models — the schema. See `docs/database.md`. | +| `app/schemas/` | Pydantic **response** models. API responses are always built from these, never from serialized ORM objects directly (section 18). | +| `app/repositories/` | Raw data access (SQLAlchemy queries) per aggregate, plus `polymorphic.py` for the shared credit/artwork/video/external-id tables. No response shaping. | +| `app/services/` | Business logic: assembling API responses from repositories (`movie_service.py`, ...), caching (`cache.py`), search (`search/`), and the ingestion pipeline (`metadata/`, `ingestion/`). | +| `app/providers/` | Provider adapters (`MetadataProvider` ABC + `TMDbProvider`). Fetch **raw** data only — never write to the database. | +| `app/api/v1/` | FastAPI routers. Thin: parse request, call a service, translate service exceptions to HTTP errors. | +| `app/workers/` | arq worker process + task functions, thin wrappers around `app/services/ingestion/runner.py`. | + +Data flows in one direction: `providers` → `services/metadata` (normalize) → +`services/metadata/matcher` (match) → `repositories` (persist) → +`services/*_service` (assemble response) → `schemas` (validate/serialize) → +`api/v1` (HTTP). Nothing skips a layer; in particular, provider adapters +never touch the database, and routers never touch SQLAlchemy models +directly. + +## Provider abstraction & ingestion pipeline (sections 13–15) + +``` +Provider.get_movie(external_id) # raw dict, provider-shaped + -> normalize_movie() # -> NormalizedMovie, provider-agnostic + -> match_entity() # exact external-ID match, or NEW + -> ingest_movie() # upsert Movie + satellite tables + -> upsert_provider_record() # provenance: raw payload + match confidence +``` + +`MetadataProvider` (`app/providers/base.py`) is the only thing that knows +TMDb-specific request shapes. A second provider (IMDb dataset export, +Wikidata, ...) means writing a new adapter that returns the same raw shape +TMDb's endpoints return, or a new `normalize_*` sibling if its shape differs +— nothing in `app/services/ingestion/` or downstream needs to change. + +**Matching policy is deliberately conservative** (section 14): only an exact +external-ID match reuses an existing Aurora entity. Everything else creates +a new entity. Aurora never guesses via fuzzy title matching that two +providers' records are "the same movie" — that risk (wrongly merging two +different films) is worse than the cost of a duplicate until a human- +reviewable matching workflow exists (tracked in `docs/database.md#known-limitations`). + +Ingestion always runs through workers, never inline in an HTTP request +(section 15). The admin API (`POST /api/v1/admin/ingest/movie`) only +enqueues an arq job and returns immediately; `app/workers/tasks.py` picks it +up and calls `app/services/ingestion/runner.py`, which tracks the job's +lifecycle in `ingestion_jobs` (idempotent via a unique `idempotency_key` — +re-triggering ingestion for the same provider + external ID reuses the same +job record rather than double-processing). + +## Search (section 16) + +`app/services/search/base.py:SearchProvider` is the abstraction; API +handlers and `SearchService` only depend on it, never on SQL directly. +`PostgreSQLSearchProvider` blends `ts_rank_cd` (full-text) with `pg_trgm` +`similarity()` (typo tolerance) across primary titles/names and alternate +titles. Swapping in `OpenSearchProvider` later means implementing the same +`SearchProvider.search()` contract — no caller changes. + +## Caching (section 21) + +`app/services/cache.py` wraps Redis with fail-open semantics: every read/ +write function catches Redis errors and behaves as a cache miss / no-op +rather than raising, so Redis being down degrades performance, not +correctness. `app/api/deps.py:cached_response()` is the read-through helper +used by the movie/person detail endpoints (the read-heaviest, per section +21). Cache keys are versioned (`aurora:v1:...`) so a response-shape change +(bumping `CACHE_VERSION`) doesn't require a manual flush. + +Cache invalidation on write: 0.1 has no write API yet (ingestion writes +directly to Postgres via workers, not through the cached read endpoints), so +there is nothing to invalidate today. The moment a write endpoint exists, +it must call `app/services/cache.py:cache_delete` for the affected keys — +tracked as a Known limitation. + +## Security (section 22) + +- CORS restricted to `CORS_ORIGINS` (comma-separated env var), not `*`. +- Every response carries `X-Content-Type-Options`, `X-Frame-Options`, + `Referrer-Policy` (see `app/main.py` middleware). +- Request bodies over 2 MiB are rejected before parsing. +- Redis-backed fixed-window rate limiting (`app/core/security.py`), fails + open if Redis is unreachable (same reasoning as caching). +- Admin/provider endpoints (`/api/v1/admin/*`) require `X-API-Key` matching + `AURORA_ADMIN_API_KEYS`; public read endpoints require no auth but are + rate-limited. +- No provider credentials are hard-coded; `TMDB_API_KEY` is read from the + environment and the TMDb adapter raises a clear + `ProviderNotConfiguredError` (not a fake success) if it's missing. +- Secrets never committed — `.env.example` holds placeholders only. + +Not yet implemented (tracked): encryption-at-rest for provider credentials +beyond "not committed to Git" (there's currently exactly one credential, +`TMDB_API_KEY`, sourced from the environment — a secrets manager integration +is the natural next step once there are more); audit logging for admin +actions beyond structured request logs. + +## Observability (section 23) + +Structured logging via `structlog`, JSON in non-development environments. +Every request gets a correlation ID (`X-Request-ID`, generated if the client +doesn't send one) bound into the log context for that request's lifetime +(`app/core/logging.py`, `app/main.py:request_context_middleware`). +`GET /metrics` exposes Prometheus counters/histograms (HTTP request count, +latency by route) — compatible with Prometheus/Grafana; nothing in local +development requires the observability stack to be running. + +## Health checks (section 24) + +`GET /health` — liveness only (process is up). `GET /ready` — checks +PostgreSQL connectivity (always required) and Redis (required only if +`REDIS_REQUIRED=true`, since Redis is disposable — see `docs/database.md`). +Both are mounted twice: unprefixed (`/health`, `/ready`, for +Docker/Traefik health checks) and under `/api/v1` (per section 17's endpoint +list). + +## Statelessness & horizontal scaling (section 1/2) + +The API holds no in-process state across requests — no server-side +sessions, no in-memory caches keyed by request. Every dependency +(`get_db`, `get_redis`, `get_queue`) is request-scoped and backed by a +shared pool. This is what makes `docker-compose.prod.yml`'s `api: replicas: +3` behind Traefik correct without sticky sessions. Workers +(`docker-compose.prod.yml`'s `worker: replicas: 2`) scale independently of +the API — arq's Redis-backed queue ensures each job is claimed by exactly +one worker. diff --git a/docs/aurora-core-spec.md b/docs/aurora-core-spec.md new file mode 100644 index 0000000..006d22a --- /dev/null +++ b/docs/aurora-core-spec.md @@ -0,0 +1,1359 @@ +# Aurora Core — Codex Build Specification + +## Project + +**Name:** Aurora Core +**Company:** Aurora Dragon Studio +**Purpose:** Production-grade media metadata, discovery, and intelligence backend. + +Aurora Core will become the canonical data layer for the Aurora media ecosystem. + +It should function conceptually like our own TMDb/IMDb-style platform, providing normalized information about: + +- Movies +- Television series +- Seasons +- Episodes +- People +- Cast and crew +- Companies +- Collections/franchises +- Genres +- Keywords +- Artwork +- Videos/trailers +- Release information +- Ratings/certifications +- External provider IDs +- Relationships between all of the above + +Aurora Core is NOT a Plex application. + +Plex, Overseerr, Radarr, Sonarr, Jellyfin, Emby, and future Aurora streaming software will be consumers or integrations built around Core. + +Do NOT make Aurora Core dependent on Plex. + +--- + +# 1. Primary Architecture + +Build Aurora Core as a production-grade **modular monolith**. + +Do NOT prematurely break the application into microservices. + +Architecture: + +Internet + | +Traefik + | +Aurora Core API replicas + | + +--- PostgreSQL + | + +--- PgBouncer + | + +--- Redis + | + +--- Background Workers + | + +--- Metadata ingestion + +--- Artwork processing + +--- Search indexing + +--- Provider synchronization + +Primary technology stack: + +- Python +- FastAPI +- PostgreSQL +- SQLAlchemy 2.x +- Alembic +- Pydantic +- Redis +- PgBouncer +- Background job workers +- Docker +- Docker Compose +- Traefik-compatible deployment + +The API must remain stateless so multiple replicas can run simultaneously. + +Example: + +Traefik + | + +--- aurora-core-api-1 + +--- aurora-core-api-2 + +--- aurora-core-api-3 + +Workers must also scale independently. + +--- + +# 2. Production Requirements + +This is NOT a prototype. + +Design Aurora Core for production use from the beginning. + +Initial engineering target: + +- 500 concurrent interactive users +- Horizontal API scaling +- Horizontal worker scaling +- Stateless API containers +- Database connection pooling +- Redis caching +- Background processing +- Idempotent jobs +- Structured logging +- Health checks +- Readiness checks +- Metrics +- Rate limiting +- Graceful shutdown +- Proper database migrations +- Automated tests + +Initial performance goals: + +| Requirement | Target | +|---|---| +| Concurrent interactive users | 500 | +| Normal API p95 | < 250 ms | +| Search p95 | < 500 ms | +| Core 5xx rate | < 0.1% | +| API horizontal scaling | Required | +| Worker horizontal scaling | Required | +| Lost jobs | 0 | +| Duplicate ingestion records | 0 | + +These targets must ultimately be verified using load testing. + +--- + +# 3. Aurora-Owned IDs + +Aurora Core MUST own its identifiers. + +Do NOT use TMDb IDs, IMDb IDs, TVDb IDs, or other provider IDs as primary keys. + +Example: + +aur_movie_01K5A8XXXXXXXXXXXX + +or another suitable sortable globally unique ID scheme such as UUIDv7. + +Internal database primary keys may use UUIDv7. + +External IDs must be separate records. + +Example: + +Aurora Movie: + Aurora ID: aur_movie_xxxxx + +External IDs: + TMDb: 550 + IMDb: tt0137523 + TVDb: ... + +This allows Aurora Core to survive provider changes. + +External providers are DATA SOURCES. + +They are not Aurora Core's identity system. + +--- + +# 4. Core Media Model + +## Movies + +Movie records should support: + +- Aurora ID +- title +- original title +- alternate titles +- overview +- tagline +- release dates +- runtime +- status +- original language +- spoken languages +- countries +- genres +- keywords +- certifications +- budget +- revenue +- popularity data +- cast +- crew +- directors +- writers +- producers +- production companies +- distributors +- collections/franchises +- posters +- backdrops +- logos +- artwork +- trailers +- videos +- external IDs +- recommendations +- similar titles +- source/provenance information +- created timestamp +- updated timestamp + +--- + +# 5. Television Model + +Support: + +Series + | + +--- Seasons + | + +--- Episodes + +Series should support: + +- title +- original title +- alternate titles +- overview +- tagline +- first air date +- last air date +- status +- networks +- production companies +- creators +- cast +- crew +- genres +- keywords +- languages +- countries +- artwork +- videos +- external IDs + +Season should support: + +- season number +- title +- overview +- air date +- episode count +- artwork +- external IDs + +Episode should support: + +- episode number +- season number +- title +- overview +- air date +- runtime +- cast +- guest stars +- crew +- directors +- writers +- still images +- external IDs + +--- + +# 6. People + +People are first-class Aurora entities. + +Person records should support: + +- Aurora ID +- name +- aliases +- biography +- birthday +- death date +- birthplace +- gender where legitimately provided by metadata sources +- profile images +- external IDs +- known-for department +- acting credits +- directing credits +- writing credits +- production credits +- other crew credits + +Credits should NOT be stored as duplicated text blobs. + +Use normalized relationship tables. + +--- + +# 7. Credits + +Create a flexible credit system. + +Examples: + +Person + | + +--- Movie + | role = actor + | character = "..." + | + +--- Movie + | role = director + | + +--- Episode + role = guest_star + +Credits should support: + +- person +- media entity +- department +- job +- character +- billing order +- episode count where appropriate +- source information + +--- + +# 8. Companies + +Support entities such as: + +- production companies +- networks +- studios +- distributors + +Fields should include: + +- Aurora ID +- company name +- description +- country +- logo +- external IDs +- parent company where applicable + +--- + +# 9. Collections / Franchises + +Collections should be first-class entities. + +Examples: + +Alien +Star Wars +Fast & Furious +Marvel Cinematic Universe + +Support: + +Collection + | + +--- Movies + +--- Series + +--- Related collections + +Do not assume every provider's concept of "collection" is sufficient for Aurora's future requirements. + +--- + +# 10. Artwork System + +Do NOT simply place poster_url columns on movie tables. + +Artwork must be its own entity. + +Artwork fields: + +- Aurora artwork ID +- associated entity +- artwork type +- language +- width +- height +- aspect ratio +- provider +- provider asset ID +- original URL +- cached/local location where permitted +- checksum +- attribution +- licensing information +- source/provenance +- created timestamp + +Artwork types: + +- poster +- backdrop +- logo +- profile +- still +- banner +- thumbnail + +Aurora must retain sufficient provider/licensing information to know where artwork originated. + +Do not assume externally sourced artwork can automatically be redistributed. + +--- + +# 11. Videos + +Videos should be normalized entities. + +Support: + +- trailer +- teaser +- clip +- featurette +- behind-the-scenes +- interview +- other + +Store: + +- provider +- provider video ID +- URL/reference +- language +- country +- official flag +- publication date +- source/provenance + +Do NOT download copyrighted video media unless explicitly supported by the provider's licensing/terms. + +--- + +# 12. External IDs + +Create a generalized external identifier system. + +Example providers: + +- TMDb +- IMDb +- TVDb +- Wikidata +- other future sources + +External IDs should support multiple Aurora entity types. + +Enforce uniqueness where appropriate. + +--- + +# 13. Provider System + +Build a provider abstraction. + +Example: + +MetadataProvider + + TMDbProvider + IMDbDatasetProvider + FutureProvider + +Providers should NOT write directly into arbitrary database tables. + +Flow: + +Provider + | +Raw Provider Data + | +Normalizer + | +Entity Matching + | +Deduplication + | +Aurora Entity + | +PostgreSQL + +Provider adapters should be replaceable. + +--- + +# 14. Provenance + +Aurora Core needs to know where metadata originated. + +Track: + +- provider +- provider entity ID +- ingestion date +- last synchronization +- source version if available +- field provenance where practical +- ingestion job +- confidence/match information + +Never silently merge questionable entities. + +Ambiguous matches should be detectable and reviewable. + +--- + +# 15. Ingestion + +Metadata ingestion MUST run asynchronously. + +Do NOT perform large provider synchronization jobs inside HTTP requests. + +Use workers. + +Example: + +Provider + | +Ingestion Job + | +Raw Record + | +Normalizer + | +Entity Matcher + | +Deduplication + | +Database + | +Search Index Update + +Jobs must be: + +- retryable +- idempotent +- observable +- resumable where practical +- protected against duplicate processing + +--- + +# 16. Search + +Search is a major Aurora Core feature. + +Initial search endpoint: + +GET /api/v1/search?q=alien + +Search across: + +- movies +- TV series +- episodes where appropriate +- people +- collections +- companies + +Initial implementation may use PostgreSQL: + +- full-text search +- trigram indexes +- normalized title search +- alternate titles + +Design a search abstraction. + +Example: + +SearchProvider + + PostgreSQLSearchProvider + +Future: + + OpenSearchProvider + +Do NOT make API clients aware of the underlying search engine. + +--- + +# 17. Core 0.1 API + +Initial endpoints should include at minimum: + +GET /api/v1/health +GET /api/v1/ready + +GET /api/v1/search + +GET /api/v1/movies/{aurora_id} +GET /api/v1/movies/{aurora_id}/credits +GET /api/v1/movies/{aurora_id}/images +GET /api/v1/movies/{aurora_id}/videos + +GET /api/v1/series/{aurora_id} +GET /api/v1/series/{aurora_id}/seasons + +GET /api/v1/seasons/{aurora_id} + +GET /api/v1/episodes/{aurora_id} + +GET /api/v1/people/{aurora_id} +GET /api/v1/people/{aurora_id}/credits + +GET /api/v1/collections/{aurora_id} + +GET /api/v1/companies/{aurora_id} + +Administrative/provider endpoints should be separately protected. + +--- + +# 18. API Response Design + +Responses should be versioned and consistent. + +Example: + +GET /api/v1/movies/{id} + +Response concept: + +{ + "id": "aur_movie_...", + "type": "movie", + "title": "Example Movie", + "original_title": "Example Movie", + "overview": "...", + "release_date": "2026-01-01", + "runtime": 120, + "genres": [], + "cast": [], + "crew": [], + "companies": [], + "collection": null, + "images": {}, + "videos": [], + "external_ids": {}, + "created_at": "...", + "updated_at": "..." +} + +Use proper Pydantic response schemas. + +Do NOT serialize SQLAlchemy models directly. + +--- + +# 19. PostgreSQL + +PostgreSQL is authoritative. + +Important application data must NEVER exist only in Redis. + +Use: + +- foreign keys +- unique constraints +- check constraints +- appropriate indexes +- partial indexes where beneficial +- transaction boundaries +- PostgreSQL full-text capabilities + +Use PgBouncer for production connection pooling. + +Avoid N+1 queries. + +Use eager/select-in loading appropriately. + +API endpoints should be profiled for query count. + +--- + +# 20. Redis + +Redis may provide: + +- API caching +- search-result caching +- rate limiting +- distributed locks +- temporary state +- worker coordination +- job queues + +Redis must NOT become the permanent source of truth. + +Aurora Core should recover gracefully from Redis loss. + +--- + +# 21. Caching + +Create a cache abstraction. + +Cache suitable read-heavy endpoints such as: + +- movie details +- person details +- series details +- common search queries + +Use sensible TTLs. + +Implement cache invalidation when records change. + +Do not cache authorization-sensitive responses without appropriate cache keys. + +--- + +# 22. Security + +Production security requirements: + +- secrets never committed to Git +- `.env.example` only contains examples +- validate all API inputs +- secure database credentials +- secure Redis configuration +- restricted CORS +- security headers +- request-size limits +- rate limiting +- administrative authorization +- API-key support where required +- audit logging for administrative actions +- dependency vulnerability scanning +- container security considerations +- no hard-coded provider tokens + +Sensitive provider credentials should be encrypted at rest where appropriate. + +--- + +# 23. Observability + +Implement structured JSON-compatible logging. + +Every request should have a correlation/request ID. + +Log useful context without logging secrets. + +Prepare metrics for: + +- HTTP request count +- response latency +- HTTP errors +- database latency +- Redis latency +- queue depth +- worker jobs +- worker failures +- provider failures +- ingestion throughput + +Architecture should be compatible with: + +- Prometheus +- Grafana +- Loki +- OpenTelemetry + +Do not require the full observability stack merely to run local development. + +--- + +# 24. Health Checks + +Implement: + +GET /health + +Used for liveness. + +GET /ready + +Readiness should validate required dependencies such as PostgreSQL. + +Redis readiness behavior should reflect whether Redis is mandatory for the configured deployment. + +Docker services must have health checks. + +--- + +# 25. Repository Structure + +Create approximately: + +aurora-core/ +├── app/ +│ ├── api/ +│ │ └── v1/ +│ │ ├── movies.py +│ │ ├── series.py +│ │ ├── seasons.py +│ │ ├── episodes.py +│ │ ├── people.py +│ │ ├── collections.py +│ │ ├── companies.py +│ │ ├── search.py +│ │ └── health.py +│ │ +│ ├── core/ +│ │ ├── config.py +│ │ ├── database.py +│ │ ├── redis.py +│ │ ├── logging.py +│ │ └── security.py +│ │ +│ ├── models/ +│ ├── schemas/ +│ ├── repositories/ +│ ├── services/ +│ │ ├── search/ +│ │ ├── metadata/ +│ │ └── ingestion/ +│ │ +│ ├── providers/ +│ ├── workers/ +│ └── main.py +│ +├── migrations/ +├── tests/ +│ ├── unit/ +│ ├── integration/ +│ └── load/ +│ +├── docs/ +│ ├── architecture.md +│ ├── database.md +│ ├── api.md +│ └── erd.md +│ +├── docker/ +│ ├── api.Dockerfile +│ └── worker.Dockerfile +│ +├── .github/ +│ └── workflows/ +│ +├── docker-compose.yml +├── docker-compose.prod.yml +├── pyproject.toml +├── alembic.ini +├── .env.example +├── .gitignore +└── README.md + +Codex may improve this layout where technically justified. + +Keep module boundaries clear. + +--- + +# 26. Database ERD + +Create a documented ERD before considering Core 0.1 complete. + +Include relationships between: + +- movies +- series +- seasons +- episodes +- people +- credits +- companies +- genres +- keywords +- collections +- languages +- countries +- certifications +- release dates +- alternate titles +- artwork +- videos +- providers +- provider records +- external IDs +- ingestion jobs +- provenance records + +Avoid unnecessary JSON blobs when normalized relational data is appropriate. + +JSONB may be used for provider-specific raw data and genuinely flexible metadata. + +--- + +# 27. Migrations + +Use Alembic. + +The application must NOT create production database tables automatically on startup. + +Database changes require migrations. + +CI should test: + +fresh database + | +run migrations + | +start application + | +run tests + +Also test migration upgrades. + +--- + +# 28. Docker + +Provide: + +docker-compose.yml + +for development. + +Provide: + +docker-compose.prod.yml + +for production-oriented deployment. + +Services should include as appropriate: + +- Aurora Core API +- Aurora worker +- PostgreSQL +- PgBouncer +- Redis + +Traefik integration should be supported. + +Do not bundle unrelated applications into the Core repository. + +--- + +# 29. Configuration + +Use environment-based configuration. + +Provide `.env.example`. + +Possible variables: + +AURORA_ENV +AURORA_LOG_LEVEL + +DATABASE_URL + +REDIS_URL + +AURORA_SECRET_KEY + +TMDB_API_KEY + +CORS_ORIGINS + +WORKER_CONCURRENCY + +CACHE_DEFAULT_TTL + +RATE_LIMIT_* + +Do NOT place actual credentials in examples. + +--- + +# 30. Testing + +Testing is mandatory. + +Create: + +## Unit tests + +Test: + +- normalization +- matching +- ID generation +- services +- schema validation +- provider adapters +- search logic + +## Integration tests + +Test: + +- PostgreSQL +- Redis +- API endpoints +- migrations +- ingestion +- entity relationships + +## API tests + +Test: + +- success responses +- invalid IDs +- missing entities +- pagination +- filtering +- validation +- error format + +## Load tests + +Create k6 or Locust tests. + +Simulate at least: + +- searching +- opening movies +- opening TV series +- opening people +- pagination + +Target: + +500 concurrent interactive users. + +Document how to run the load test. + +--- + +# 31. CI + +Create CI workflow. + +Pipeline should include: + +lint + | +type checking + | +unit tests + | +integration tests + | +migration test + | +container build + | +security/dependency checks + +Use appropriate Python tooling. + +Suggested tools may include: + +- Ruff +- mypy or Pyright +- pytest + +Codex may select equivalent mature tooling if justified. + +--- + +# 32. Documentation + +Create: + +README.md + +docs/architecture.md +docs/database.md +docs/api.md +docs/erd.md + +README should explain: + +- project purpose +- architecture +- prerequisites +- local setup +- environment configuration +- Docker startup +- migrations +- tests +- API documentation +- worker operation + +FastAPI OpenAPI documentation should remain available in development. + +Production exposure of API documentation should be configurable. + +--- + +# 33. Core 0.1 Scope + +DO build: + +- database foundation +- Aurora IDs +- movies +- television +- seasons +- episodes +- people +- credits +- companies +- genres +- keywords +- collections +- artwork metadata +- videos +- external IDs +- provider abstraction +- ingestion pipeline +- normalization +- deduplication foundations +- search +- REST API +- caching +- workers +- production Docker architecture +- tests +- documentation + +DO NOT build into Core 0.1: + +- Plex integration +- Overseerr integration +- Radarr integration +- Sonarr integration +- video streaming +- transcoding +- CouchSurfer UI +- Aurora Media website UI + +Those are separate applications/integrations. + +--- + +# 34. Initial Milestone + +The first meaningful milestone is: + +A client can search Aurora Core for a movie, TV series, or person and retrieve a complete normalized Aurora-owned record. + +Example: + +GET /api/v1/search?q=the%20matrix + +Aurora returns matching Aurora entities. + +The client can then call: + +GET /api/v1/movies/{aurora_id} + +and receive: + +- title +- overview +- release information +- runtime +- genres +- cast +- crew +- directors +- writers +- companies +- collection +- artwork metadata +- videos +- external IDs + +Likewise: + +GET /api/v1/people/{aurora_id} + +returns: + +- person information +- images +- external IDs +- known credits/filmography + +--- + +# 35. Future Aurora Ecosystem + +Aurora Core should eventually support: + +Aurora Core + | + +--- Aurora Media + | IMDb-style web experience + | + +--- Aurora Curator + | Media identification/organization + | + +--- CouchSurfer + | Personal media front end + | + +--- Future Aurora streaming server + | + +--- Mobile applications + | + +--- Aurora AI/intelligence + +Applications should consume Core APIs. + +Do not tightly couple Core to any one application. + +--- + +# 36. Important Engineering Principles + +1. Production first. + +2. Aurora owns its entity identities. + +3. External metadata services are providers, not Aurora's database. + +4. PostgreSQL is authoritative. + +5. Redis is disposable acceleration/coordination. + +6. Expensive work runs asynchronously. + +7. APIs remain stateless and horizontally scalable. + +8. Maintain clear module boundaries. + +9. Avoid premature microservices. + +10. Every schema change uses a migration. + +11. Everything important is testable. + +12. Provider provenance must be retained. + +13. Do not duplicate entities because two providers describe them differently. + +14. Do not silently merge uncertain matches. + +15. Design interfaces so external providers can be replaced. + +16. Core must function independently of Plex. + +--- + +# 37. Codex Instructions + +Begin implementing the repository. + +Do not merely produce another architecture proposal. + +Create the actual files and working code. + +Implementation order: + +1. Create repository structure. +2. Create Python project configuration. +3. Create application configuration. +4. Create database connection layer. +5. Create Redis layer. +6. Create Aurora ID system. +7. Implement SQLAlchemy models. +8. Create initial Alembic migration. +9. Create Pydantic schemas. +10. Create repositories. +11. Create service layer. +12. Create provider abstraction. +13. Implement initial metadata provider adapter. +14. Create ingestion/normalization pipeline. +15. Implement PostgreSQL search. +16. Implement REST API. +17. Implement workers. +18. Add caching. +19. Add logging/observability foundations. +20. Add Docker environment. +21. Add tests. +22. Add load tests. +23. Add CI. +24. Complete documentation. + +After each major stage: + +- run tests +- run linting +- run type checking +- fix failures before proceeding + +Do not leave fake implementations silently returning successful responses. + +If an external credential is required, create the integration interface and clearly document the required environment variable. + +Use TODO comments only for genuinely deferred Core features. + +--- + +# 38. Definition of Done — Aurora Core 0.1 + +Core 0.1 is complete when: + +- Docker Compose can start the complete development environment. +- PostgreSQL migrations run successfully from an empty database. +- Aurora IDs are generated independently of providers. +- Movies can be ingested and normalized. +- TV entities can be ingested and normalized. +- People can be ingested and normalized. +- External IDs are linked without becoming primary IDs. +- Credits and relationships work. +- Artwork provenance is retained. +- Search returns movies, TV series, and people. +- Detail APIs return normalized records. +- Redis caching works. +- Background jobs work. +- Duplicate ingestion is protected. +- API instances can run horizontally. +- Automated tests pass. +- CI passes. +- Health/readiness checks work. +- Load-test scripts exist and are documented. +- The architecture and ERD are documented. +- No Plex dependency exists. +- No real secrets exist in Git. +- README provides enough information for another developer to launch the project. + +At completion, output: + +1. Complete repository tree. +2. Database/ERD summary. +3. API endpoint list. +4. Docker service list. +5. Test results. +6. Known limitations. +7. Commands required to launch Aurora Core. +8. Recommended next milestone. + +Do not begin CouchSurfer or other Aurora applications until the Core 0.1 foundation is operational. diff --git a/docs/database.md b/docs/database.md new file mode 100644 index 0000000..1f9e9e4 --- /dev/null +++ b/docs/database.md @@ -0,0 +1,165 @@ +# Aurora Core — Database + +PostgreSQL is authoritative (principle 4). Nothing important lives only in +Redis. See [erd.md](erd.md) for the full entity-relationship diagram. + +## Aurora IDs vs. primary keys + +Every table's primary key is a UUIDv7 (`app/models/base.py:UUIDPrimaryKeyMixin`), +generated in the application, not the database. UUIDv7 is time-ordered, so it +stays btree-friendly (unlike UUIDv4) and can double as a rough creation-order +cursor. + +The public "Aurora ID" strings clients see (`aur_movie_01m1q7p1c...`) are a +lossless, sortable Crockford-base32 encoding of that same UUID, produced by +`app/core/ids.py:aurora_id()`. Encoding (not a second stored column) keeps +the public ID and the primary key from ever drifting apart. See section 3 of +`docs/aurora-core-spec.md` for why Aurora never uses a provider's ID (TMDb, +IMDb, ...) as a primary key. + +## Reference data + +- `languages` (ISO 639-1), `countries` (ISO 3166-1 alpha-2), `certifications` + (country + rating system + rating, e.g. US/MPA/PG-13) — populated + incrementally as ingestion encounters new codes (`get_or_create_*` in + `app/repositories/reference_repository.py`), not preloaded. A curated + seed list is a reasonable follow-up (see Known limitations). +- `genres`, `keywords` — deduplicated by a slugified name. Two providers' + "Sci-Fi" and "Science Fiction" are **not** currently merged; that requires + a curated synonym table, deliberately deferred (section 33 lists dedup + *foundations*, not a finished thesaurus). + +## Movies / TV + +`movies` and `series` hold the record itself; per-locale and per-relationship +data lives in satellite tables (`movie_alternate_titles`, +`movie_spoken_languages`, `movie_production_countries`, `release_dates`, +`movie_recommendations`, and the `series_*` equivalents) rather than JSON +blobs, per section 26. `seasons` and `episodes` form a strict +`series -> season -> episode` hierarchy; `episodes.series_id` is denormalized +(in addition to `season_id`) purely so episode queries don't need to hop +through `seasons` to filter/join by series. + +`movies.status` / `series.status` are native Postgres enums, values `.value` +of `app/models/enums.py:MovieStatus`/`SeriesStatus` — see "Enum storage" +below. + +## Credits + +`credits` is the flexible cast/crew system from section 7: one row per +(person, media entity, department/job/character), not per-movie text blobs. +`department == "Acting"` rows carry `character`/`billing_order`; everything +else carries `job`. Series-level "Creator" credits and episode +`is_guest` (guest stars) reuse the same table rather than adding new ones. + +## Companies / Collections + +A single `companies` table serves production companies, studios, networks, +and distributors, distinguished by `company_type` (section 8) — a TV network +and a movie studio have the same shape (name, logo, country, parent +company), so a separate `Network` entity would just duplicate columns. +`collections` (franchises) are first-class, with their own artwork and +`collection_relations` for shared-universe links between collections +(section 9). + +## Artwork / Videos + +Never a `poster_url` column (section 10/11). `artwork` and `videos` are +their own tables carrying provider, provider-native asset ID, dimensions, +and (for artwork) `attribution`/`license` fields — Aurora doesn't assume +externally-sourced images can be freely redistributed. Aurora never +downloads video media; `videos.url` is a reference to the provider's +hosted copy (YouTube, Vimeo, ...). + +## Polymorphic associations + +`credits`, `artwork`, `videos`, `external_ids`, and `release_dates` attach to +more than one entity type via an `(entity_type, entity_id)` pair instead of a +typed foreign key, because a single column can't reference "movies.id or +series.id or people.id depending on a sibling column" in Postgres. The +tradeoff: **the schema does not enforce that `entity_id` actually points at +a real row of `entity_type`.** That integrity is enforced by the +repository/service layer (`app/repositories/polymorphic.py`, +`app/services/ingestion/*`), which is the only code path allowed to write +these tables. If Core grows enough that this becomes a real risk, the +standard fix is per-type junction tables (`movie_credits`, `series_credits`, +...) at the cost of losing the single shared query path — deliberately not +done for 0.1. + +## External IDs & provenance + +`external_ids` is the generalized external-identifier system (section 12): +`(provider, entity_type, external_id)` is unique (one Aurora entity per +provider-native ID), and `(entity_type, entity_id, provider)` is unique (one +mapping per provider per entity). + +`provider_records` is the full provenance ledger (section 14): one row per +`(provider, entity_type, external_id)` holding the raw payload +(`raw_data` JSONB), `match_confidence`, and which `ingestion_job` produced +it. Entity-level provenance is coarse — `movies.source_provider_id` / +`series.source_provider_id` point at "the provider that was authoritative at +creation time" for quick display; full field-level provenance (which +provider supplied *this specific* field) is not tracked in 0.1. See Known +limitations. + +`ingestion_jobs` tracks async work with a unique `idempotency_key` (e.g. +`tmdb:movie:603`) so re-triggering the same sync doesn't double-enqueue — +see `app/services/ingestion/runner.py`. + +## Enum storage + +Python enums in `app/models/enums.py` are `enum.StrEnum` (member value == +member name's meaning, e.g. `MovieStatus.RELEASED == "released"`). +SQLAlchemy's `Enum` type stores `.name` (`"RELEASED"`) by default, which +would silently diverge from the lowercase strings the API and ingestion code +use everywhere. `app/models/base.py:_enum_column()` overrides this via +`values_callable` so the database, the ORM, and the API all agree on the +same lowercase strings. + +## Migrations + +Alembic (section 27); the application **never** creates tables at startup — +see `app/core/database.py`. The initial migration +(`migrations/versions/0001_initial_schema.py`) is hand-written rather than +autogenerated (no throwaway Postgres was available in the environment this +was built in — see the root README's Known Limitations), but was written +directly from the same models this document describes and validated with +`alembic history`; treat a first `alembic upgrade head` against a scratch +database as the real acceptance test, and autogenerate future migrations +against it (`alembic revision --autogenerate`) rather than the empty +database this one was written from. + +```bash +alembic upgrade head # apply all migrations +alembic revision --autogenerate -m "add xyz" # generate the next one +alembic downgrade -1 # roll back one revision +``` + +Integration/API tests build schema directly from `Base.metadata.create_all` +(see `tests/conftest.py`) for speed and to catch model drift quickly — this +is a testing shortcut, not how the running application ever gets its schema. + +## Search indexes + +`pg_trgm` (trigram similarity, for typo-tolerant matching) plus functional +GIN indexes over `to_tsvector('simple', title || ' ' || original_title)` on +`movies`/`series`, `name` on `people`/`collections`/`companies`, and trigram +indexes on the alternate-title tables. See +`app/services/search/postgresql_provider.py` and section 16 of the spec. + +## Known limitations (tracked, not hidden) + +- No curated language/country seed data — rows are created on first + encounter during ingestion, so `english_name`/`name` are sometimes just + the code uppercased until backfilled. +- No fuzzy/candidate entity matching — only exact external-ID matches are + auto-linked (section 14: never silently merge ambiguous matches). A + review queue for AMBIGUOUS matches is the natural next step once Core has + an admin UI to act on it. +- Field-level provenance (which provider supplied *this* value) is not + tracked, only entity-level (`source_provider_id`) and per-provider raw + payloads (`provider_records.raw_data`). +- Satellite associations (credits, artwork, videos, genre/keyword/company + links) are replaced wholesale on each re-sync rather than diffed, so their + Aurora IDs are not stable across re-ingestion — see + `app/services/ingestion/movie_ingestion.py`. diff --git a/docs/erd.md b/docs/erd.md new file mode 100644 index 0000000..938f9df --- /dev/null +++ b/docs/erd.md @@ -0,0 +1,201 @@ +# Aurora Core — Entity-Relationship Diagram + +This is the Core 0.1 schema (36 tables, see [database.md](database.md) for +column-level detail). Association/junction tables are drawn as relationships +directly where Mermaid supports it; a few many-to-many tables with their own +attributes (`movie_companies.role`, `credits.*`) are drawn as entities. + +```mermaid +erDiagram + MOVIE ||--o{ MOVIE_ALTERNATE_TITLE : has + MOVIE ||--o{ MOVIE_SPOKEN_LANGUAGE : has + MOVIE ||--o{ MOVIE_PRODUCTION_COUNTRY : has + MOVIE ||--o{ RELEASE_DATE : has + MOVIE ||--o{ MOVIE_RECOMMENDATION : recommends + MOVIE }o--o{ GENRE : "movie_genres" + MOVIE }o--o{ KEYWORD : "movie_keywords" + MOVIE }o--o{ COMPANY : "movie_companies" + MOVIE }o--|| COLLECTION : "belongs to" + MOVIE ||--o{ CREDIT : "cast/crew" + MOVIE ||--o{ ARTWORK : "images" + MOVIE ||--o{ VIDEO : "trailers" + MOVIE ||--o{ EXTERNAL_ID : "provider refs" + + SERIES ||--o{ SEASON : has + SEASON ||--o{ EPISODE : has + SERIES ||--o{ SERIES_ALTERNATE_TITLE : has + SERIES ||--o{ SERIES_SPOKEN_LANGUAGE : has + SERIES ||--o{ SERIES_PRODUCTION_COUNTRY : has + SERIES }o--o{ GENRE : "series_genres" + SERIES }o--o{ KEYWORD : "series_keywords" + SERIES }o--o{ COMPANY : "series_companies (network/studio)" + SERIES ||--o{ CREDIT : "cast/crew/creators" + SERIES ||--o{ ARTWORK : "images" + SERIES ||--o{ VIDEO : "trailers" + SERIES ||--o{ EXTERNAL_ID : "provider refs" + EPISODE ||--o{ CREDIT : "cast/guest stars/crew" + EPISODE ||--o{ ARTWORK : "stills" + EPISODE ||--o{ EXTERNAL_ID : "provider refs" + SEASON ||--o{ ARTWORK : "posters" + SEASON ||--o{ EXTERNAL_ID : "provider refs" + + COLLECTION ||--o{ MOVIE : "collection_movies" + COLLECTION ||--o{ SERIES : "collection_series" + COLLECTION ||--o{ COLLECTION : "collection_relations" + COLLECTION ||--o{ ARTWORK : "images" + + COMPANY ||--o{ COMPANY : "parent_company" + COMPANY ||--o{ ARTWORK : "logos" + COMPANY ||--o{ EXTERNAL_ID : "provider refs" + COMPANY }o--o| COUNTRY : "based in" + + PERSON ||--o{ CREDIT : "credited on" + PERSON ||--o{ ARTWORK : "profile images" + PERSON ||--o{ EXTERNAL_ID : "provider refs" + + PROVIDER ||--o{ PROVIDER_RECORD : "provenance" + PROVIDER ||--o{ INGESTION_JOB : "runs" + PROVIDER ||--o{ EXTERNAL_ID : "issues" + PROVIDER ||--o{ ARTWORK : "sourced" + PROVIDER ||--o{ VIDEO : "sourced" + PROVIDER ||--o{ CREDIT : "sourced" + INGESTION_JOB ||--o{ PROVIDER_RECORD : produces + + COUNTRY ||--o{ CERTIFICATION : defines + RELEASE_DATE }o--o| CERTIFICATION : rated + + MOVIE { + uuid id PK + string title + string sort_title + enum status + date primary_release_date + int runtime_minutes + uuid collection_id FK + uuid source_provider_id FK + } + SERIES { + uuid id PK + string title + enum status + date first_air_date + int number_of_seasons + } + SEASON { + uuid id PK + uuid series_id FK + int season_number + } + EPISODE { + uuid id PK + uuid season_id FK + uuid series_id FK + int episode_number + } + PERSON { + uuid id PK + string name + date birthday + string known_for_department + } + CREDIT { + uuid id PK + uuid person_id FK + string entity_type + uuid entity_id + string department + string job + string character + } + COMPANY { + uuid id PK + string name + enum company_type + uuid parent_company_id FK + } + COLLECTION { + uuid id PK + string name + string slug + } + GENRE { + uuid id PK + string name + } + KEYWORD { + uuid id PK + string name + } + ARTWORK { + uuid id PK + string entity_type + uuid entity_id + enum artwork_type + uuid provider_id FK + string original_url + } + VIDEO { + uuid id PK + string entity_type + uuid entity_id + enum video_type + uuid provider_id FK + string url + } + EXTERNAL_ID { + uuid id PK + string entity_type + uuid entity_id + uuid provider_id FK + string external_id + } + PROVIDER { + uuid id PK + string slug + string name + } + PROVIDER_RECORD { + uuid id PK + uuid provider_id FK + string entity_type + string external_id + uuid aurora_entity_id + enum match_confidence + jsonb raw_data + } + INGESTION_JOB { + uuid id PK + uuid provider_id FK + string job_type + enum status + string idempotency_key + } + COUNTRY { + string code PK + string name + } + CERTIFICATION { + uuid id PK + string country_code FK + string system + string rating + } + RELEASE_DATE { + uuid id PK + uuid entity_id + string country_code FK + date release_date + uuid certification_id FK + } +``` + +## Notes on the polymorphic tables + +`CREDIT`, `ARTWORK`, `VIDEO`, `EXTERNAL_ID`, and `RELEASE_DATE` attach to more +than one entity type (movie, series, season, episode, person, company, +collection) via an `(entity_type, entity_id)` pair rather than a typed +foreign key. Postgres can't enforce referential integrity across a +polymorphic pair with a single `FOREIGN KEY`, so integrity there is enforced +by the repository/service layer (`app/repositories/polymorphic.py`), not the +schema. See [database.md](database.md#polymorphic-associations) for the +tradeoff. diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..c5f9c2c --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from app.core.config import get_settings +from app.models import Base # noqa: F401 (imports every model onto Base.metadata) + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + +# The application uses an async (asyncpg) URL for normal operation, but +# Alembic migrations run synchronously (psycopg2) — see DATABASE_URL_SYNC in +# app/core/config.py. +config.set_main_option("sqlalchemy.url", get_settings().DATABASE_URL_SYNC) + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..b1f8b89 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_schema.py b/migrations/versions/0001_initial_schema.py new file mode 100644 index 0000000..04886db --- /dev/null +++ b/migrations/versions/0001_initial_schema.py @@ -0,0 +1,813 @@ +"""initial schema + +Revision ID: 0001_initial_schema +Revises: +Create Date: 2026-09-04 + +Creates every Aurora Core 0.1 table (section 26/33 of the build spec): +reference data (languages/countries/certifications), providers and +provenance, genres/keywords, companies, collections, people, movies and +their satellite tables, series/seasons/episodes and their satellite tables, +and the polymorphic credit/artwork/video/external-id tables. Also enables +`pg_trgm` and creates the functional GIN indexes PostgreSQLSearchProvider +relies on (section 16). +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0001_initial_schema" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _timestamps() -> list[sa.Column]: + return [ + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ] + + +MOVIE_STATUS = postgresql.ENUM( + "rumored", + "planned", + "in_production", + "post_production", + "released", + "canceled", + name="moviestatus_enum", +) +SERIES_STATUS = postgresql.ENUM( + "planned", + "pilot", + "in_production", + "returning_series", + "ended", + "canceled", + name="seriesstatus_enum", +) +ARTWORK_TYPE = postgresql.ENUM( + "poster", "backdrop", "logo", "profile", "still", "banner", "thumbnail", + name="artworktype_enum", +) +VIDEO_TYPE = postgresql.ENUM( + "trailer", "teaser", "clip", "featurette", "behind_the_scenes", "interview", "other", + name="videotype_enum", +) +COMPANY_TYPE = postgresql.ENUM( + "production_company", "network", "studio", "distributor", name="companytype_enum" +) +INGESTION_JOB_STATUS = postgresql.ENUM( + "pending", "running", "succeeded", "failed", "retrying", name="ingestionjobstatus_enum" +) +MATCH_CONFIDENCE = postgresql.ENUM( + "exact", "high", "medium", "low", "ambiguous", "new", name="matchconfidence_enum" +) + + +def upgrade() -> None: + bind = op.get_bind() + op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + + for enum_type in ( + MOVIE_STATUS, + SERIES_STATUS, + ARTWORK_TYPE, + VIDEO_TYPE, + COMPANY_TYPE, + INGESTION_JOB_STATUS, + MATCH_CONFIDENCE, + ): + enum_type.create(bind, checkfirst=True) + + # --- Reference data ----------------------------------------------- + op.create_table( + "languages", + sa.Column("code", sa.String(10), primary_key=True), + sa.Column("english_name", sa.String(200), nullable=False), + sa.Column("native_name", sa.String(200), nullable=True), + ) + + op.create_table( + "countries", + sa.Column("code", sa.String(2), primary_key=True), + sa.Column("name", sa.String(200), nullable=False), + ) + + op.create_table( + "certifications", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=False), + sa.Column("system", sa.String(50), nullable=False), + sa.Column("rating", sa.String(20), nullable=False), + sa.Column("meaning", sa.Text(), nullable=True), + sa.UniqueConstraint("country_code", "system", "rating", name="uq_certification_scope"), + ) + + # --- Providers & provenance ----------------------------------------- + op.create_table( + "providers", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("slug", sa.String(50), nullable=False, unique=True), + sa.Column("name", sa.String(200), nullable=False), + sa.Column("base_url", sa.String(500), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column( + "config", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default="{}" + ), + ) + + op.create_table( + "ingestion_jobs", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("job_type", sa.String(50), nullable=False), + sa.Column("entity_type", sa.String(30), nullable=True), + sa.Column("external_id", sa.String(200), nullable=True), + sa.Column("idempotency_key", sa.String(300), nullable=False), + sa.Column("status", INGESTION_JOB_STATUS, nullable=False, server_default="pending"), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="5"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("result_aurora_id", sa.Uuid(), nullable=True), + sa.UniqueConstraint("idempotency_key", name="uq_ingestion_job_idempotency"), + ) + + op.create_table( + "provider_records", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("entity_type", sa.String(30), nullable=False), + sa.Column("external_id", sa.String(200), nullable=False), + sa.Column("aurora_entity_id", sa.Uuid(), nullable=True), + sa.Column( + "raw_data", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default="{}" + ), + sa.Column("source_version", sa.String(100), nullable=True), + sa.Column("match_confidence", MATCH_CONFIDENCE, nullable=False, server_default="new"), + sa.Column("match_notes", sa.Text(), nullable=True), + sa.Column( + "ingestion_job_id", + sa.Uuid(), + sa.ForeignKey("ingestion_jobs.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "last_synced_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.UniqueConstraint( + "provider_id", "entity_type", "external_id", name="uq_provider_record_source" + ), + ) + + # --- Genres / keywords ---------------------------------------------- + op.create_table( + "genres", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("name", sa.String(100), nullable=False, unique=True), + sa.Column("slug", sa.String(100), nullable=False, unique=True), + ) + + op.create_table( + "keywords", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("name", sa.String(150), nullable=False, unique=True), + sa.Column("slug", sa.String(150), nullable=False, unique=True), + ) + + # --- Companies -------------------------------------------------------- + op.create_table( + "companies", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("name", sa.String(300), nullable=False), + sa.Column( + "company_type", COMPANY_TYPE, nullable=False, server_default="production_company" + ), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=True), + sa.Column("headquarters", sa.String(300), nullable=True), + sa.Column("homepage", sa.String(500), nullable=True), + sa.Column( + "parent_company_id", + sa.Uuid(), + sa.ForeignKey("companies.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.create_index("ix_companies_name", "companies", ["name"]) + + # --- Collections -------------------------------------------------------- + op.create_table( + "collections", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("name", sa.String(300), nullable=False), + sa.Column("overview", sa.Text(), nullable=True), + sa.Column("slug", sa.String(350), nullable=False, unique=True), + ) + op.create_index("ix_collections_name", "collections", ["name"]) + + # --- People ----------------------------------------------------------- + op.create_table( + "people", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("name", sa.String(300), nullable=False), + sa.Column( + "aliases", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default="[]" + ), + sa.Column("biography", sa.Text(), nullable=True), + sa.Column("birthday", sa.Date(), nullable=True), + sa.Column("deathday", sa.Date(), nullable=True), + sa.Column("birthplace", sa.String(300), nullable=True), + sa.Column("gender", sa.String(30), nullable=True), + sa.Column("known_for_department", sa.String(100), nullable=True), + sa.Column("popularity", sa.Float(), nullable=True), + ) + op.create_index("ix_people_name", "people", ["name"]) + + # --- Movies ------------------------------------------------------------- + op.create_table( + "movies", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("title", sa.String(500), nullable=False), + sa.Column("original_title", sa.String(500), nullable=True), + sa.Column("sort_title", sa.String(500), nullable=False), + sa.Column("overview", sa.Text(), nullable=True), + sa.Column("tagline", sa.Text(), nullable=True), + sa.Column("status", MOVIE_STATUS, nullable=False, server_default="released"), + sa.Column( + "original_language_code", sa.String(10), sa.ForeignKey("languages.code"), nullable=True + ), + sa.Column("primary_release_date", sa.Date(), nullable=True), + sa.Column("runtime_minutes", sa.Integer(), nullable=True), + sa.Column("budget", sa.BigInteger(), nullable=True), + sa.Column("revenue", sa.BigInteger(), nullable=True), + sa.Column("popularity", sa.Float(), nullable=True), + sa.Column("vote_average", sa.Float(), nullable=True), + sa.Column("vote_count", sa.Integer(), nullable=True), + sa.Column("adult", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "collection_id", + sa.Uuid(), + sa.ForeignKey("collections.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "source_provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.create_index("ix_movies_title", "movies", ["title"]) + op.create_index("ix_movies_sort_title", "movies", ["sort_title"]) + op.create_index("ix_movies_primary_release_date", "movies", ["primary_release_date"]) + + op.create_table( + "movie_alternate_titles", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column( + "movie_id", sa.Uuid(), sa.ForeignKey("movies.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("title", sa.String(500), nullable=False), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=True), + sa.Column("type", sa.String(50), nullable=True), + ) + op.create_index( + "ix_movie_alternate_titles_movie_id", "movie_alternate_titles", ["movie_id"] + ) + + op.create_table( + "movie_spoken_languages", + sa.Column( + "movie_id", + sa.Uuid(), + sa.ForeignKey("movies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "language_code", + sa.String(10), + sa.ForeignKey("languages.code"), + primary_key=True, + ), + ) + + op.create_table( + "movie_production_countries", + sa.Column( + "movie_id", + sa.Uuid(), + sa.ForeignKey("movies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "country_code", + sa.String(2), + sa.ForeignKey("countries.code"), + primary_key=True, + ), + ) + + op.create_table( + "release_dates", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("entity_type", sa.String(30), nullable=False, server_default="movie"), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=False), + sa.Column("release_date", sa.Date(), nullable=False), + sa.Column("release_type", sa.String(50), nullable=True), + sa.Column( + "certification_id", + sa.Uuid(), + sa.ForeignKey("certifications.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("note", sa.Text(), nullable=True), + ) + op.create_index("ix_release_dates_entity_id", "release_dates", ["entity_id"]) + + op.create_table( + "movie_recommendations", + sa.Column( + "movie_id", + sa.Uuid(), + sa.ForeignKey("movies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "recommended_movie_id", + sa.Uuid(), + sa.ForeignKey("movies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("rank", sa.Integer(), nullable=True), + sa.Column("source", sa.String(50), nullable=True), + ) + + op.create_table( + "movie_genres", + sa.Column( + "movie_id", sa.Uuid(), sa.ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "genre_id", sa.Uuid(), sa.ForeignKey("genres.id", ondelete="CASCADE"), primary_key=True + ), + ) + + op.create_table( + "movie_keywords", + sa.Column( + "movie_id", sa.Uuid(), sa.ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "keyword_id", + sa.Uuid(), + sa.ForeignKey("keywords.id", ondelete="CASCADE"), + primary_key=True, + ), + ) + + op.create_table( + "movie_companies", + sa.Column( + "movie_id", sa.Uuid(), sa.ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "company_id", + sa.Uuid(), + sa.ForeignKey("companies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("role", sa.String(50), nullable=True), + ) + + # --- Series / Seasons / Episodes --------------------------------------- + op.create_table( + "series", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("title", sa.String(500), nullable=False), + sa.Column("original_title", sa.String(500), nullable=True), + sa.Column("sort_title", sa.String(500), nullable=False), + sa.Column("overview", sa.Text(), nullable=True), + sa.Column("tagline", sa.Text(), nullable=True), + sa.Column("status", SERIES_STATUS, nullable=False, server_default="returning_series"), + sa.Column( + "original_language_code", sa.String(10), sa.ForeignKey("languages.code"), nullable=True + ), + sa.Column("first_air_date", sa.Date(), nullable=True), + sa.Column("last_air_date", sa.Date(), nullable=True), + sa.Column("number_of_seasons", sa.Integer(), nullable=True), + sa.Column("number_of_episodes", sa.Integer(), nullable=True), + sa.Column("popularity", sa.Float(), nullable=True), + sa.Column("vote_average", sa.Float(), nullable=True), + sa.Column("vote_count", sa.Integer(), nullable=True), + sa.Column( + "source_provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.create_index("ix_series_title", "series", ["title"]) + op.create_index("ix_series_sort_title", "series", ["sort_title"]) + op.create_index("ix_series_first_air_date", "series", ["first_air_date"]) + + op.create_table( + "series_alternate_titles", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("title", sa.String(500), nullable=False), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=True), + sa.Column("type", sa.String(50), nullable=True), + ) + op.create_index( + "ix_series_alternate_titles_series_id", "series_alternate_titles", ["series_id"] + ) + + op.create_table( + "series_spoken_languages", + sa.Column( + "series_id", + sa.Uuid(), + sa.ForeignKey("series.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "language_code", + sa.String(10), + sa.ForeignKey("languages.code"), + primary_key=True, + ), + ) + + op.create_table( + "series_production_countries", + sa.Column( + "series_id", + sa.Uuid(), + sa.ForeignKey("series.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "country_code", + sa.String(2), + sa.ForeignKey("countries.code"), + primary_key=True, + ), + ) + + op.create_table( + "seasons", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("season_number", sa.Integer(), nullable=False), + sa.Column("title", sa.String(500), nullable=True), + sa.Column("overview", sa.Text(), nullable=True), + sa.Column("air_date", sa.Date(), nullable=True), + sa.Column("episode_count", sa.Integer(), nullable=True), + sa.UniqueConstraint("series_id", "season_number", name="uq_season_number"), + ) + op.create_index("ix_seasons_series_id", "seasons", ["series_id"]) + + op.create_table( + "episodes", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column( + "season_id", sa.Uuid(), sa.ForeignKey("seasons.id", ondelete="CASCADE"), nullable=False + ), + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("season_number", sa.Integer(), nullable=False), + sa.Column("episode_number", sa.Integer(), nullable=False), + sa.Column("title", sa.String(500), nullable=True), + sa.Column("overview", sa.Text(), nullable=True), + sa.Column("air_date", sa.Date(), nullable=True), + sa.Column("runtime_minutes", sa.Integer(), nullable=True), + sa.UniqueConstraint("season_id", "episode_number", name="uq_episode_number"), + ) + op.create_index("ix_episodes_season_id", "episodes", ["season_id"]) + op.create_index("ix_episodes_series_id", "episodes", ["series_id"]) + + op.create_table( + "series_genres", + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "genre_id", sa.Uuid(), sa.ForeignKey("genres.id", ondelete="CASCADE"), primary_key=True + ), + ) + + op.create_table( + "series_keywords", + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "keyword_id", + sa.Uuid(), + sa.ForeignKey("keywords.id", ondelete="CASCADE"), + primary_key=True, + ), + ) + + op.create_table( + "series_companies", + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column( + "company_id", + sa.Uuid(), + sa.ForeignKey("companies.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("role", sa.String(50), nullable=True), + ) + + # --- Collection memberships --------------------------------------------- + op.create_table( + "collection_movies", + sa.Column( + "collection_id", + sa.Uuid(), + sa.ForeignKey("collections.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "movie_id", sa.Uuid(), sa.ForeignKey("movies.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column("sort_order", sa.Integer(), nullable=True), + ) + + op.create_table( + "collection_series", + sa.Column( + "collection_id", + sa.Uuid(), + sa.ForeignKey("collections.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "series_id", sa.Uuid(), sa.ForeignKey("series.id", ondelete="CASCADE"), primary_key=True + ), + sa.Column("sort_order", sa.Integer(), nullable=True), + ) + + op.create_table( + "collection_relations", + sa.Column( + "collection_id", + sa.Uuid(), + sa.ForeignKey("collections.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "related_collection_id", + sa.Uuid(), + sa.ForeignKey("collections.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("relation_type", sa.String(50), nullable=False, server_default="related"), + ) + + # --- Polymorphic satellite tables --------------------------------------- + op.create_table( + "credits", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column( + "person_id", sa.Uuid(), sa.ForeignKey("people.id", ondelete="CASCADE"), nullable=False + ), + sa.Column("entity_type", sa.String(30), nullable=False), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column("department", sa.String(100), nullable=False), + sa.Column("job", sa.String(150), nullable=True), + sa.Column("character", sa.String(300), nullable=True), + sa.Column("billing_order", sa.Integer(), nullable=True), + sa.Column("is_guest", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("episode_count", sa.Integer(), nullable=True), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.create_index("ix_credits_entity", "credits", ["entity_type", "entity_id"]) + op.create_index("ix_credits_person_id", "credits", ["person_id"]) + + op.create_table( + "artwork", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("entity_type", sa.String(30), nullable=False), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column("artwork_type", ARTWORK_TYPE, nullable=False), + sa.Column("language_code", sa.String(10), sa.ForeignKey("languages.code"), nullable=True), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("aspect_ratio", sa.Float(), nullable=True), + sa.Column("vote_average", sa.Float(), nullable=True), + sa.Column("is_primary", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("provider_asset_id", sa.String(200), nullable=True), + sa.Column("original_url", sa.String(1000), nullable=False), + sa.Column("local_path", sa.String(1000), nullable=True), + sa.Column("checksum", sa.String(128), nullable=True), + sa.Column("attribution", sa.Text(), nullable=True), + sa.Column("license", sa.Text(), nullable=True), + ) + op.create_index( + "ix_artwork_entity", "artwork", ["entity_type", "entity_id", "artwork_type"] + ) + + op.create_table( + "videos", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("entity_type", sa.String(30), nullable=False), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column("video_type", VIDEO_TYPE, nullable=False), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("provider_video_id", sa.String(200), nullable=False), + sa.Column("site", sa.String(100), nullable=True), + sa.Column("url", sa.String(1000), nullable=False), + sa.Column("language_code", sa.String(10), sa.ForeignKey("languages.code"), nullable=True), + sa.Column("country_code", sa.String(2), sa.ForeignKey("countries.code"), nullable=True), + sa.Column("is_official", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("published_at", sa.Date(), nullable=True), + sa.UniqueConstraint( + "provider_id", "provider_video_id", name="uq_video_provider_source" + ), + ) + op.create_index("ix_videos_entity", "videos", ["entity_type", "entity_id"]) + + op.create_table( + "external_ids", + sa.Column("id", sa.Uuid(), primary_key=True), + *_timestamps(), + sa.Column("entity_type", sa.String(30), nullable=False), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column( + "provider_id", + sa.Uuid(), + sa.ForeignKey("providers.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("external_id", sa.String(200), nullable=False), + sa.Column("url", sa.String(500), nullable=True), + sa.UniqueConstraint( + "provider_id", "entity_type", "external_id", name="uq_external_id_source" + ), + sa.UniqueConstraint( + "entity_type", + "entity_id", + "provider_id", + name="uq_external_id_entity_provider", + ), + ) + + # --- Search indexes (section 16) ---------------------------------------- + op.execute( + "CREATE INDEX ix_movies_title_trgm ON movies USING gin (title gin_trgm_ops)" + ) + op.execute( + "CREATE INDEX ix_movies_title_fts ON movies " + "USING gin (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(original_title, '')))" + ) + op.execute( + "CREATE INDEX ix_movie_alt_titles_trgm ON movie_alternate_titles " + "USING gin (title gin_trgm_ops)" + ) + op.execute( + "CREATE INDEX ix_series_title_trgm ON series USING gin (title gin_trgm_ops)" + ) + op.execute( + "CREATE INDEX ix_series_title_fts ON series " + "USING gin (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(original_title, '')))" + ) + op.execute( + "CREATE INDEX ix_series_alt_titles_trgm ON series_alternate_titles " + "USING gin (title gin_trgm_ops)" + ) + op.execute("CREATE INDEX ix_people_name_trgm ON people USING gin (name gin_trgm_ops)") + op.execute( + "CREATE INDEX ix_people_name_fts ON people USING gin (to_tsvector('simple', name))" + ) + op.execute( + "CREATE INDEX ix_collections_name_trgm ON collections USING gin (name gin_trgm_ops)" + ) + op.execute( + "CREATE INDEX ix_companies_name_trgm ON companies USING gin (name gin_trgm_ops)" + ) + + +def downgrade() -> None: + op.drop_table("external_ids") + op.drop_table("videos") + op.drop_table("artwork") + op.drop_table("credits") + op.drop_table("collection_relations") + op.drop_table("collection_series") + op.drop_table("collection_movies") + op.drop_table("series_companies") + op.drop_table("series_keywords") + op.drop_table("series_genres") + op.drop_table("episodes") + op.drop_table("seasons") + op.drop_table("series_production_countries") + op.drop_table("series_spoken_languages") + op.drop_table("series_alternate_titles") + op.drop_table("series") + op.drop_table("movie_companies") + op.drop_table("movie_keywords") + op.drop_table("movie_genres") + op.drop_table("movie_recommendations") + op.drop_table("release_dates") + op.drop_table("movie_production_countries") + op.drop_table("movie_spoken_languages") + op.drop_table("movie_alternate_titles") + op.drop_table("movies") + op.drop_table("people") + op.drop_table("collections") + op.drop_table("companies") + op.drop_table("keywords") + op.drop_table("genres") + op.drop_table("provider_records") + op.drop_table("ingestion_jobs") + op.drop_table("providers") + op.drop_table("certifications") + op.drop_table("countries") + op.drop_table("languages") + + bind = op.get_bind() + for enum_type in ( + MATCH_CONFIDENCE, + INGESTION_JOB_STATUS, + COMPANY_TYPE, + VIDEO_TYPE, + ARTWORK_TYPE, + SERIES_STATUS, + MOVIE_STATUS, + ): + enum_type.drop(bind, checkfirst=True) + + op.execute("DROP EXTENSION IF EXISTS pg_trgm") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f5cf38c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,75 @@ +[project] +name = "aurora-core" +version = "0.1.0" +description = "Aurora Core — production-grade media metadata, discovery, and intelligence backend for the Aurora ecosystem." +readme = "README.md" +requires-python = ">=3.12" +license = { text = "Proprietary" } +authors = [{ name = "Aurora Dragon Studio" }] + +dependencies = [ + "fastapi>=0.115.0", + "uvicorn[standard]>=0.30.0", + "sqlalchemy>=2.0.35", + "alembic>=1.13.2", + "asyncpg>=0.29.0", + "psycopg2-binary>=2.9.9", + "pydantic>=2.9.0", + "pydantic-settings>=2.5.0", + "redis>=5.0.8", + "arq>=0.26.0", + "httpx>=0.27.0", + "structlog>=24.4.0", + "orjson>=3.10.0", + "uuid6>=2024.7.10", + "python-dotenv>=1.0.1", + "tenacity>=9.0.0", + "prometheus-client>=0.20.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.0", + "pytest-asyncio>=0.24.0", + "pytest-cov>=5.0.0", + "ruff>=0.6.9", + "mypy>=1.11.2", + "aiosqlite>=0.20.0", + "faker>=28.4.1", + "types-redis", +] +load = [ + "locust>=2.31.0", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["app*"] + +[tool.ruff] +line-length = 100 +target-version = "py312" +exclude = ["migrations"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM", "N"] +# E501: line length handled by formatter, not worth failing CI over. +# B027: MetadataProvider.aclose() is an intentional optional-override hook +# (concrete no-op default), not an accidentally-unimplemented abstract method. +# B008: FastAPI's `Depends(...)` dependency-injection idiom is a call in an +# argument default by design; this is the standard FastAPI pattern. +ignore = ["E501", "B027", "B008"] + +[tool.mypy] +python_version = "3.12" +plugins = ["pydantic.mypy"] +ignore_missing_imports = true +disallow_untyped_defs = false +exclude = ["migrations", "tests"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f97cc3c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,87 @@ +"""Shared pytest fixtures. + +Integration/API tests need a real PostgreSQL instance (JSONB, trigram/FTS +indexes, and native enums are Postgres-specific — see docs/database.md) and +Redis. Point `TEST_DATABASE_URL` / `TEST_REDIS_URL` at throwaway instances, +e.g. via `docker compose up -d postgres redis` and a separate test database: + + createdb -h localhost -U aurora aurora_core_test + +Unit tests (tests/unit/) never touch either and always run offline. +""" + +from __future__ import annotations + +import os +from collections.abc import AsyncGenerator + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +TEST_DATABASE_URL = os.environ.get( + "TEST_DATABASE_URL", "postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core_test" +) +TEST_REDIS_URL = os.environ.get("TEST_REDIS_URL", "redis://localhost:6379/15") + +os.environ.setdefault("DATABASE_URL", TEST_DATABASE_URL) +os.environ.setdefault("REDIS_URL", TEST_REDIS_URL) +os.environ.setdefault("AURORA_ENV", "test") +os.environ.setdefault("AURORA_SECRET_KEY", "test-secret") +os.environ.setdefault("AURORA_ADMIN_API_KEYS", "test-admin-key") + + +def _requires_postgres() -> bool: + return "--no-db" not in os.environ.get("PYTEST_ADDOPTS", "") + + +@pytest_asyncio.fixture(scope="session") +async def engine(): + from sqlalchemy import text + + from app.models import Base + + eng = create_async_engine(TEST_DATABASE_URL) + try: + async with eng.begin() as conn: + # Tests build schema straight from ORM metadata for speed and + # to catch model/migration drift; docs/database.md notes this + # is deliberately *not* how the running application migrates. + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm")) + await conn.run_sync(Base.metadata.create_all) + except Exception as exc: # pragma: no cover - environment guard + pytest.skip(f"PostgreSQL not reachable at {TEST_DATABASE_URL}: {exc}") + yield eng + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + await eng.dispose() + + +@pytest_asyncio.fixture +async def db_session(engine) -> AsyncGenerator[AsyncSession, None]: + connection = await engine.connect() + transaction = await connection.begin() + session_factory = async_sessionmaker(bind=connection, expire_on_commit=False) + session = session_factory() + + yield session + + await session.close() + await transaction.rollback() + await connection.close() + + +@pytest_asyncio.fixture +async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]: + from app.core.database import get_db + from app.main import app + + async def _override_get_db(): + yield db_session + + app.dependency_overrides[get_db] = _override_get_db + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as ac: + yield ac + app.dependency_overrides.clear() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/factories.py b/tests/integration/factories.py new file mode 100644 index 0000000..0138d15 --- /dev/null +++ b/tests/integration/factories.py @@ -0,0 +1,81 @@ +"""Minimal data builders for integration tests — inserts ORM rows directly +so tests don't depend on network calls to a real provider.""" + +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.ids import aurora_id +from app.models.artwork import Artwork +from app.models.credit import Credit +from app.models.enums import ArtworkType, MovieStatus +from app.models.external_id import ExternalID +from app.models.genre import Genre, MovieGenre +from app.models.movie import Movie +from app.models.person import Person +from app.models.provider import Provider + + +async def make_provider(session: AsyncSession, slug: str = "tmdb") -> Provider: + provider = Provider(slug=slug, name=slug.upper()) + session.add(provider) + await session.flush() + return provider + + +async def make_movie( + session: AsyncSession, *, title: str = "The Matrix", provider: Provider | None = None +) -> Movie: + movie = Movie( + title=title, + sort_title=title, + overview="A hacker discovers reality is a simulation.", + status=MovieStatus.RELEASED, + runtime_minutes=136, + ) + session.add(movie) + await session.flush() + + genre = Genre(name="Science Fiction", slug="science-fiction") + session.add(genre) + await session.flush() + session.add(MovieGenre(movie_id=movie.id, genre_id=genre.id)) + + person = Person(name="Keanu Reeves") + session.add(person) + await session.flush() + session.add( + Credit( + person_id=person.id, + entity_type="movie", + entity_id=movie.id, + department="Acting", + character="Neo", + billing_order=0, + ) + ) + + session.add( + Artwork( + entity_type="movie", + entity_id=movie.id, + artwork_type=ArtworkType.POSTER, + provider_id=(provider or await make_provider(session)).id, + original_url="https://image.tmdb.org/t/p/original/poster.jpg", + is_primary=True, + ) + ) + + if provider is not None: + session.add( + ExternalID( + entity_type="movie", entity_id=movie.id, provider_id=provider.id, external_id="603" + ) + ) + + await session.flush() + return movie + + +def movie_aurora_id(movie: Movie) -> str: + return aurora_id("movie", str(movie.id)) diff --git a/tests/integration/test_health.py b/tests/integration/test_health.py new file mode 100644 index 0000000..8c980d0 --- /dev/null +++ b/tests/integration/test_health.py @@ -0,0 +1,26 @@ +"""Integration tests for liveness/readiness (section 24). Requires +PostgreSQL and Redis — see tests/conftest.py.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient + +pytestmark = pytest.mark.usefixtures("db_session") + + +@pytest.mark.asyncio +async def test_health_is_always_ok(client: AsyncClient): + response = await client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.asyncio +async def test_ready_reports_postgresql_dependency(client: AsyncClient): + response = await client.get("/ready") + assert response.status_code == 200 + body = response.json() + names = {dep["name"] for dep in body["dependencies"]} + assert "postgresql" in names + assert "redis" in names diff --git a/tests/integration/test_movie_ingestion.py b/tests/integration/test_movie_ingestion.py new file mode 100644 index 0000000..20910ce --- /dev/null +++ b/tests/integration/test_movie_ingestion.py @@ -0,0 +1,105 @@ +"""Integration tests for the movie ingestion pipeline (section 15): a fake +provider stands in for TMDb so these run without network access, while +still exercising the real normalize -> match -> upsert -> provenance path +against PostgreSQL.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.credit import Credit +from app.models.genre import Genre +from app.models.movie import Movie +from app.models.provider import ProviderRecord +from app.providers.base import MetadataProvider +from app.services.ingestion.movie_ingestion import ingest_movie +from tests.unit.test_normalizer import FAKE_MOVIE + + +class FakeTMDbProvider(MetadataProvider): + slug = "tmdb" + name = "The Movie Database (TMDb)" + + def __init__(self, payload: dict[str, Any]) -> None: + self._payload = payload + + async def get_movie(self, external_id: str) -> dict[str, Any]: + return self._payload + + async def get_series(self, external_id: str) -> dict[str, Any]: + raise NotImplementedError + + async def get_season(self, series_external_id: str, season_number: int) -> dict[str, Any]: + raise NotImplementedError + + async def get_person(self, external_id: str) -> dict[str, Any]: + raise NotImplementedError + + async def search_movie(self, query: str) -> list[dict[str, Any]]: + raise NotImplementedError + + +@pytest.mark.asyncio +async def test_ingest_movie_creates_normalized_record(db_session: AsyncSession): + provider = FakeTMDbProvider(FAKE_MOVIE) + + movie_id = await ingest_movie(db_session, provider, "603") + await db_session.commit() + + movie = await db_session.get(Movie, movie_id) + assert movie is not None + assert movie.title == "The Matrix" + assert movie.runtime_minutes == 136 + + genres = (await db_session.execute(select(Genre.name))).scalars().all() + assert "Action" in genres + + credits_ = ( + (await db_session.execute(select(Credit).where(Credit.entity_id == movie_id))) + .scalars() + .all() + ) + assert any(c.character == "Neo" for c in credits_) + + record = ( + await db_session.execute( + select(ProviderRecord).where(ProviderRecord.external_id == "603") + ) + ).scalar_one() + assert record.aurora_entity_id == movie_id + assert record.match_confidence.value == "new" + + +@pytest.mark.asyncio +async def test_ingest_movie_is_idempotent(db_session: AsyncSession): + """Re-running ingestion for the same provider + external_id must update + the same Aurora movie, never create a duplicate (section 2: zero + duplicate ingestion records).""" + + provider = FakeTMDbProvider(FAKE_MOVIE) + + first_id = await ingest_movie(db_session, provider, "603") + await db_session.commit() + + updated_payload = {**FAKE_MOVIE, "title": "The Matrix (Updated Title)"} + second_provider = FakeTMDbProvider(updated_payload) + second_id = await ingest_movie(db_session, second_provider, "603") + await db_session.commit() + + assert first_id == second_id + + all_movies = (await db_session.execute(select(Movie))).scalars().all() + assert len(all_movies) == 1 + assert all_movies[0].title == "The Matrix (Updated Title)" + + credits_ = ( + (await db_session.execute(select(Credit).where(Credit.entity_id == first_id))) + .scalars() + .all() + ) + # Credits are replaced wholesale on re-sync, not duplicated. + assert len(credits_) == 2 # 1 cast + 1 crew from FAKE_MOVIE diff --git a/tests/integration/test_movies_api.py b/tests/integration/test_movies_api.py new file mode 100644 index 0000000..415a101 --- /dev/null +++ b/tests/integration/test_movies_api.py @@ -0,0 +1,91 @@ +"""Integration tests for the Movie API (section 17/18). Requires +PostgreSQL and Redis — see tests/conftest.py.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.integration.factories import make_movie, make_provider, movie_aurora_id + + +@pytest.mark.asyncio +async def test_get_movie_returns_normalized_record(client: AsyncClient, db_session: AsyncSession): + provider = await make_provider(db_session) + movie = await make_movie(db_session, provider=provider) + await db_session.commit() + + response = await client.get(f"/api/v1/movies/{movie_aurora_id(movie)}") + assert response.status_code == 200 + body = response.json() + + assert body["id"] == movie_aurora_id(movie) + assert body["type"] == "movie" + assert body["title"] == "The Matrix" + assert body["runtime"] == 136 + assert {g["name"] for g in body["genres"]} == {"Science Fiction"} + assert len(body["cast"]) == 1 + assert body["cast"][0]["character"] == "Neo" + assert body["images"]["posters"][0]["is_primary"] is True + assert body["external_ids"]["tmdb"] == "603" + + +@pytest.mark.asyncio +async def test_get_movie_404_for_unknown_id(client: AsyncClient): + from app.core.ids import aurora_id, new_uuid7 + + unknown_id = aurora_id("movie", new_uuid7()) + response = await client.get(f"/api/v1/movies/{unknown_id}") + assert response.status_code == 404 + assert response.json()["error"]["code"] == "movie_not_found" + + +@pytest.mark.asyncio +async def test_get_movie_rejects_malformed_id(client: AsyncClient): + response = await client.get("/api/v1/movies/not-a-real-id") + assert response.status_code == 404 + + +@pytest.mark.asyncio +async def test_get_movie_rejects_wrong_entity_type_id( + client: AsyncClient, db_session: AsyncSession +): + # A well-formed person ID must not resolve as a movie. + from app.core.ids import aurora_id, new_uuid7 + + person_shaped_id = aurora_id("person", new_uuid7()) + response = await client.get(f"/api/v1/movies/{person_shaped_id}") + assert response.status_code == 404 + + +@pytest.mark.asyncio +async def test_get_movie_credits(client: AsyncClient, db_session: AsyncSession): + movie = await make_movie(db_session) + await db_session.commit() + + response = await client.get(f"/api/v1/movies/{movie_aurora_id(movie)}/credits") + assert response.status_code == 200 + body = response.json() + assert len(body["cast"]) == 1 + assert body["cast"][0]["person"]["name"] == "Keanu Reeves" + + +@pytest.mark.asyncio +async def test_get_movie_images(client: AsyncClient, db_session: AsyncSession): + movie = await make_movie(db_session) + await db_session.commit() + + response = await client.get(f"/api/v1/movies/{movie_aurora_id(movie)}/images") + assert response.status_code == 200 + assert len(response.json()["images"]["posters"]) == 1 + + +@pytest.mark.asyncio +async def test_get_movie_videos_empty(client: AsyncClient, db_session: AsyncSession): + movie = await make_movie(db_session) + await db_session.commit() + + response = await client.get(f"/api/v1/movies/{movie_aurora_id(movie)}/videos") + assert response.status_code == 200 + assert response.json()["videos"] == [] diff --git a/tests/load/locustfile.py b/tests/load/locustfile.py new file mode 100644 index 0000000..29a56b5 --- /dev/null +++ b/tests/load/locustfile.py @@ -0,0 +1,103 @@ +"""Load test for Aurora Core 0.1 (section 30/2). + +Simulates the read-heavy interactive traffic Core 0.1 targets: searching, +opening movie/series/person detail pages, and paging through search +results. Run against a seeded environment (ingest a reasonable movie/series/ +person catalog first — an empty database will pass load tests trivially but +tell you nothing about real query cost). + +Usage: + + pip install locust + locust -f tests/load/locustfile.py --host http://localhost:8000 + +Then open http://localhost:8089 and start a run with up to 500 users to +verify the section 2 targets (search p95 < 500ms, other endpoints p95 < +250ms, 5xx rate < 0.1%). Locust reports p50/p95/p99 latency and failure +rate directly; compare them against docs/architecture.md's targets. + +See docs/architecture.md for how to seed sample data before running this. +""" + +from __future__ import annotations + +import random + +from locust import HttpUser, between, task + +SEARCH_TERMS = [ + "matrix", + "star wars", + "alien", + "the office", + "breaking bad", + "keanu reeves", + "christopher nolan", + "marvel", + "batman", + "inception", +] + + +class AuroraCoreUser(HttpUser): + """A typical interactive client: search, then drill into a result.""" + + wait_time = between(1, 3) + + def on_start(self) -> None: + self._known_movie_ids: list[str] = [] + self._known_series_ids: list[str] = [] + self._known_person_ids: list[str] = [] + + @task(5) + def search(self) -> None: + query = random.choice(SEARCH_TERMS) + limit = random.choice([5, 10, 20]) + with self.client.get( + f"/api/v1/search?q={query}&limit={limit}", + name="/api/v1/search", + catch_response=True, + ) as response: + if response.status_code != 200: + response.failure(f"search failed: {response.status_code}") + return + body = response.json() + self._known_movie_ids.extend(m["id"] for m in body.get("movies", [])) + self._known_series_ids.extend(s["id"] for s in body.get("series", [])) + self._known_person_ids.extend(p["id"] for p in body.get("people", [])) + # Keep the pools bounded so memory doesn't grow unbounded over a + # long run. + self._known_movie_ids = self._known_movie_ids[-200:] + self._known_series_ids = self._known_series_ids[-200:] + self._known_person_ids = self._known_person_ids[-200:] + + @task(4) + def open_movie(self) -> None: + if not self._known_movie_ids: + return + movie_id = random.choice(self._known_movie_ids) + self.client.get(f"/api/v1/movies/{movie_id}", name="/api/v1/movies/[id]") + self.client.get( + f"/api/v1/movies/{movie_id}/credits", name="/api/v1/movies/[id]/credits" + ) + + @task(2) + def open_series(self) -> None: + if not self._known_series_ids: + return + series_id = random.choice(self._known_series_ids) + self.client.get(f"/api/v1/series/{series_id}", name="/api/v1/series/[id]") + self.client.get( + f"/api/v1/series/{series_id}/seasons", name="/api/v1/series/[id]/seasons" + ) + + @task(2) + def open_person(self) -> None: + if not self._known_person_ids: + return + person_id = random.choice(self._known_person_ids) + self.client.get(f"/api/v1/people/{person_id}", name="/api/v1/people/[id]") + + @task(1) + def health_probe(self) -> None: + self.client.get("/health", name="/health") diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_ids.py b/tests/unit/test_ids.py new file mode 100644 index 0000000..c58043d --- /dev/null +++ b/tests/unit/test_ids.py @@ -0,0 +1,73 @@ +"""Unit tests for the Aurora ID system (app/core/ids.py) — section 3.""" + +from __future__ import annotations + +import pytest + +from app.core.ids import ( + ENTITY_PREFIXES, + aurora_id, + is_valid_aurora_id, + new_uuid7, + parse_aurora_id, +) + + +def test_new_uuid7_is_unique_and_well_formed(): + ids = {new_uuid7() for _ in range(1000)} + assert len(ids) == 1000 + for value in list(ids)[:5]: + assert len(value) == 36 # canonical hyphenated form + + +def test_aurora_id_roundtrips_through_parse(): + u = new_uuid7() + aid = aurora_id("movie", u) + assert aid.startswith("aur_movie_") + assert parse_aurora_id(aid, expected_type="movie") == u + + +def test_aurora_id_never_leaks_provider_identity(): + # Aurora IDs must not be derived from or embed provider IDs (section 3). + u = new_uuid7() + aid = aurora_id("movie", u) + assert "550" not in aid # a plausible TMDb id, just shouldn't appear + assert "tmdb" not in aid + + +def test_every_entity_prefix_produces_a_parseable_id(): + for entity_type in ENTITY_PREFIXES: + aid = aurora_id(entity_type, new_uuid7()) + assert is_valid_aurora_id(aid, expected_type=entity_type) + + +def test_parse_rejects_wrong_entity_type(): + aid = aurora_id("movie", new_uuid7()) + with pytest.raises(ValueError): + parse_aurora_id(aid, expected_type="person") + assert is_valid_aurora_id(aid, expected_type="person") is False + + +@pytest.mark.parametrize( + "malformed", + [ + "", + "not-an-aurora-id", + "aur_movie_", + "aur_movie_tooshort", + "aur_unknownprefix_" + "0" * 26, + "aur_movie_" + "!" * 26, + ], +) +def test_parse_rejects_malformed_ids(malformed: str): + with pytest.raises(ValueError): + parse_aurora_id(malformed) + assert is_valid_aurora_id(malformed) is False + + +def test_aurora_ids_sort_in_creation_order(): + # UUIDv7 is time-ordered, and the Crockford base32 encoding preserves + # byte order, so IDs generated later should sort later. + first = aurora_id("movie", new_uuid7()) + second = aurora_id("movie", new_uuid7()) + assert first < second diff --git a/tests/unit/test_normalizer.py b/tests/unit/test_normalizer.py new file mode 100644 index 0000000..6ec7f20 --- /dev/null +++ b/tests/unit/test_normalizer.py @@ -0,0 +1,184 @@ +"""Unit tests for the TMDb normalizer (app/services/metadata/normalizer.py) +— section 13. Uses hand-built payload fragments shaped like real TMDb +responses rather than calling the network.""" + +from __future__ import annotations + +from datetime import date + +from app.services.metadata.normalizer import normalize_movie, normalize_person + +FAKE_MOVIE = { + "id": 603, + "title": "The Matrix", + "original_title": "The Matrix", + "overview": "A computer hacker learns...", + "tagline": "Welcome to the Real World", + "status": "Released", + "original_language": "en", + "release_date": "1999-03-30", + "runtime": 136, + "budget": 63000000, + "revenue": 463517383, + "popularity": 45.5, + "vote_average": 8.2, + "vote_count": 24000, + "adult": False, + "imdb_id": "tt0133093", + "genres": [{"id": 28, "name": "Action"}, {"id": 878, "name": "Science Fiction"}], + "production_companies": [ + {"id": 79, "name": "Village Roadshow Pictures", "origin_country": "US"} + ], + "poster_path": "/poster.jpg", + "backdrop_path": "/backdrop.jpg", + "belongs_to_collection": {"id": 2344, "name": "The Matrix Collection"}, + "credits": { + "cast": [ + { + "id": 6384, + "name": "Keanu Reeves", + "character": "Neo", + "order": 0, + "profile_path": "/keanu.jpg", + "gender": 2, + } + ], + "crew": [ + { + "id": 9339, + "name": "Lana Wachowski", + "department": "Directing", + "job": "Director", + "gender": 1, + } + ], + }, + "images": { + "posters": [ + {"file_path": "/poster.jpg", "width": 2000, "height": 3000, "aspect_ratio": 0.667, "iso_639_1": "en", "vote_average": 8.1} + ], + "backdrops": [], + "logos": [], + }, + "videos": { + "results": [ + { + "id": "abc123", + "type": "Trailer", + "site": "YouTube", + "key": "vKQi3bBA1y8", + "iso_639_1": "en", + "iso_3166_1": "US", + "official": True, + "published_at": "1999-03-01T00:00:00.000Z", + } + ] + }, + "keywords": {"keywords": [{"id": 1, "name": "dystopia"}]}, + "alternative_titles": {"titles": [{"title": "Matrix", "iso_3166_1": "US"}]}, + "external_ids": {"imdb_id": "tt0133093"}, + "recommendations": {"results": [{"id": 604}]}, +} + +FAKE_PERSON = { + "id": 6384, + "name": "Keanu Reeves", + "biography": "Keanu Charles Reeves is a Canadian actor.", + "birthday": "1964-09-02", + "deathday": None, + "place_of_birth": "Beirut, Lebanon", + "gender": 2, + "known_for_department": "Acting", + "popularity": 30.2, + "also_known_as": ["Keanu Charles Reeves"], + "profile_path": "/keanu.jpg", + "images": {"profiles": [{"file_path": "/keanu.jpg", "width": 1000, "height": 1500, "aspect_ratio": 0.667, "iso_639_1": None, "vote_average": 5.0}]}, + "external_ids": {"imdb_id": "nm0000206"}, +} + + +def test_normalize_movie_maps_core_fields(): + movie = normalize_movie(FAKE_MOVIE) + assert movie.external_id == "603" + assert movie.title == "The Matrix" + assert movie.status == "released" + assert movie.release_date == date(1999, 3, 30) + assert movie.runtime_minutes == 136 + assert movie.budget == 63000000 + assert movie.adult is False + + +def test_normalize_movie_maps_genres_and_keywords(): + movie = normalize_movie(FAKE_MOVIE) + assert movie.genres == ["Action", "Science Fiction"] + assert movie.keywords == ["dystopia"] + assert movie.alternate_titles == ["Matrix"] + + +def test_normalize_movie_maps_cast_and_crew(): + movie = normalize_movie(FAKE_MOVIE) + assert len(movie.cast) == 1 + assert movie.cast[0].external_id == "6384" + assert movie.cast[0].department == "Acting" + assert movie.cast[0].character == "Neo" + assert movie.cast[0].gender == "male" + + assert len(movie.crew) == 1 + assert movie.crew[0].department == "Directing" + assert movie.crew[0].job == "Director" + assert movie.crew[0].gender == "female" + + +def test_normalize_movie_maps_artwork_and_videos(): + movie = normalize_movie(FAKE_MOVIE) + poster_types = [a.artwork_type for a in movie.artwork] + assert poster_types.count("poster") >= 1 + assert all(a.url.startswith("https://image.tmdb.org/t/p/original/") for a in movie.artwork) + + assert len(movie.videos) == 1 + assert movie.videos[0].url == "https://www.youtube.com/watch?v=vKQi3bBA1y8" + assert movie.videos[0].is_official is True + + +def test_normalize_movie_external_ids_never_include_tmdb_itself(): + # TMDb's own id is handled separately by the ingestion pipeline via + # `normalized.external_id`, not folded into `external_ids` (section 3: + # Aurora doesn't conflate its own identity with provider identity). + movie = normalize_movie(FAKE_MOVIE) + assert "tmdb" not in movie.external_ids + assert movie.external_ids["imdb"] == "tt0133093" + + +def test_normalize_movie_maps_collection_and_recommendations(): + movie = normalize_movie(FAKE_MOVIE) + assert movie.collection == ("2344", "The Matrix Collection") + assert movie.recommendation_external_ids == ["604"] + + +def test_normalize_movie_handles_missing_optional_sections(): + minimal = {"id": 1, "title": "Minimal Movie"} + movie = normalize_movie(minimal) + assert movie.title == "Minimal Movie" + assert movie.genres == [] + assert movie.cast == [] + assert movie.artwork == [] + assert movie.collection is None + + +def test_normalize_person_maps_core_fields(): + person = normalize_person(FAKE_PERSON) + assert person.external_id == "6384" + assert person.name == "Keanu Reeves" + assert person.birthday == date(1964, 9, 2) + assert person.deathday is None + assert person.gender == "male" + assert person.external_ids["imdb"] == "nm0000206" + assert "Keanu Charles Reeves" in person.aliases + + +def test_normalize_person_profile_artwork_deduplicated(): + person = normalize_person(FAKE_PERSON) + # profile_path duplicates an entry already in images.profiles; it must + # not be inserted twice. + paths = [a.provider_asset_id for a in person.artwork] + assert paths.count("/keanu.jpg") == 1