Initial Aurora Core 0.1 implementation

Production-grade media metadata backend per docs/aurora-core-spec.md:
Aurora-owned UUIDv7 identity system, normalized movie/TV/people/credits
schema, provider abstraction with a TMDb adapter, an idempotent async
ingestion pipeline (normalize -> match -> upsert -> provenance), a
Postgres FTS/trigram search layer, a stateless FastAPI v1 API with
Redis caching/rate-limiting, arq workers, dev/prod Docker Compose
topologies, unit + integration + load tests, CI, and architecture/
database/API/ERD documentation.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/core/config.py

@@ -0,0 +1,103 @@
+"""Environment-based application configuration.
+
+All runtime configuration flows through this module. Nothing else in the
+codebase should read `os.environ` directly, so that configuration stays
+auditable and testable in one place.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # --- Runtime ---
+    AURORA_ENV: str = Field(default="development")
+    AURORA_LOG_LEVEL: str = Field(default="INFO")
+    AURORA_API_DOCS_ENABLED: bool = Field(default=True)
+
+    # --- Database ---
+    DATABASE_URL: str = Field(
+        default="postgresql+asyncpg://aurora:aurora@localhost:5432/aurora_core"
+    )
+    DATABASE_URL_SYNC: str = Field(
+        default="postgresql+psycopg2://aurora:aurora@localhost:5432/aurora_core"
+    )
+    DATABASE_POOL_SIZE: int = Field(default=10)
+    DATABASE_MAX_OVERFLOW: int = Field(default=20)
+    # Set true when DATABASE_URL points at PgBouncer in transaction pooling
+    # mode (docker-compose.prod.yml): asyncpg's server-side prepared
+    # statements don't survive connections being handed to different clients
+    # between transactions, so they must be disabled.
+    DATABASE_BEHIND_TRANSACTION_POOLER: bool = Field(default=False)
+
+    # --- Redis ---
+    REDIS_URL: str = Field(default="redis://localhost:6379/0")
+    REDIS_REQUIRED: bool = Field(default=False)
+
+    # --- Security ---
+    AURORA_SECRET_KEY: str = Field(default="changeme-generate-a-real-secret")
+    AURORA_ADMIN_API_KEYS: str = Field(default="")
+
+    # --- CORS ---
+    CORS_ORIGINS: str = Field(default="http://localhost:3000")
+
+    # --- Providers ---
+    TMDB_API_KEY: str = Field(default="")
+
+    # --- Workers ---
+    WORKER_CONCURRENCY: int = Field(default=4)
+
+    # --- Caching ---
+    CACHE_DEFAULT_TTL: int = Field(default=300)
+
+    # --- Rate limiting ---
+    RATE_LIMIT_ENABLED: bool = Field(default=True)
+    RATE_LIMIT_PER_MINUTE: int = Field(default=120)
+
+    @field_validator("AURORA_ENV")
+    @classmethod
+    def _validate_env(cls, v: str) -> str:
+        allowed = {"development", "test", "staging", "production"}
+        if v not in allowed:
+            raise ValueError(f"AURORA_ENV must be one of {allowed}, got {v!r}")
+        return v
+
+    @property
+    def cors_origins_list(self) -> list[str]:
+        return [origin.strip() for origin in self.CORS_ORIGINS.split(",") if origin.strip()]
+
+    @property
+    def admin_api_keys_set(self) -> set[str]:
+        return {key.strip() for key in self.AURORA_ADMIN_API_KEYS.split(",") if key.strip()}
+
+    @property
+    def is_production(self) -> bool:
+        return self.AURORA_ENV == "production"
+
+    @property
+    def docs_url(self) -> str | None:
+        if not self.AURORA_API_DOCS_ENABLED:
+            return None
+        return "/docs"
+
+    @property
+    def openapi_url(self) -> str | None:
+        if not self.AURORA_API_DOCS_ENABLED:
+            return None
+        return "/openapi.json"
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

app/core/database.py

@@ -0,0 +1,88 @@
+"""Async SQLAlchemy engine/session management.
+
+The API is stateless: every request gets its own AsyncSession from a shared
+connection pool, and nothing is held across requests. This module never
+creates tables — schema changes always go through Alembic (see
+`migrations/`).
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from app.core.config import get_settings
+
+settings = get_settings()
+
+_connect_args: dict = {}
+if settings.DATABASE_BEHIND_TRANSACTION_POOLER:
+    _connect_args["statement_cache_size"] = 0
+
+_engine: AsyncEngine = create_async_engine(
+    settings.DATABASE_URL,
+    pool_size=settings.DATABASE_POOL_SIZE,
+    max_overflow=settings.DATABASE_MAX_OVERFLOW,
+    pool_pre_ping=True,
+    pool_recycle=1800,
+    connect_args=_connect_args,
+    echo=False,
+)
+
+async_session_factory = async_sessionmaker(
+    bind=_engine,
+    class_=AsyncSession,
+    expire_on_commit=False,
+    autoflush=False,
+)
+
+
+def get_engine() -> AsyncEngine:
+    return _engine
+
+
+async def get_db() -> AsyncGenerator[AsyncSession, None]:
+    """FastAPI dependency yielding a request-scoped session."""
+
+    async with async_session_factory() as session:
+        yield session
+
+
+@asynccontextmanager
+async def session_scope() -> AsyncGenerator[AsyncSession, None]:
+    """Context manager for use outside request handlers (workers, scripts).
+
+    Commits on success, rolls back on exception.
+    """
+
+    async with async_session_factory() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise
+
+
+async def database_is_ready() -> bool:
+    """Cheap connectivity check used by the /ready endpoint."""
+
+    from sqlalchemy import text
+
+    try:
+        async with _engine.connect() as conn:
+            await conn.execute(text("SELECT 1"))
+        return True
+    except Exception:
+        return False
+
+
+async def dispose_engine() -> None:
+    await _engine.dispose()

app/core/ids.py

@@ -0,0 +1,135 @@
+"""Aurora's own identity system.
+
+Aurora Core owns its entity identities. Provider identifiers (TMDb, IMDb,
+TVDb, ...) are stored as external references, never as primary keys — see
+`app.models.external_id`.
+
+Every Aurora entity has:
+
+- a UUIDv7 database primary key (time-ordered, efficient as a btree index,
+  generated with :func:`new_uuid7`)
+- a public "Aurora ID" string derived from that same UUID, e.g.
+  ``aur_movie_1jz8f2n3k4m5p6q7r8s9t0v1w2``, produced by :func:`aurora_id`
+
+Because the public ID is a direct encoding of the UUIDv7 primary key, the two
+never drift apart, and Aurora IDs sort lexicographically in creation order —
+useful for cursors and debugging without leaking a sequential integer.
+"""
+
+from __future__ import annotations
+
+import re
+
+from uuid6 import uuid7
+
+# Crockford base32: excludes I, L, O, U to avoid visual ambiguity in IDs.
+_CROCKFORD_ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
+_CROCKFORD_INDEX = {ch: i for i, ch in enumerate(_CROCKFORD_ALPHABET)}
+
+# Aurora entity type -> ID prefix. Keep stable once shipped; these are
+# effectively part of the public API contract.
+ENTITY_PREFIXES: dict[str, str] = {
+    "movie": "movie",
+    "series": "series",
+    "season": "season",
+    "episode": "episode",
+    "person": "person",
+    "credit": "credit",
+    "company": "company",
+    "collection": "collection",
+    "genre": "genre",
+    "keyword": "keyword",
+    "artwork": "art",
+    "video": "vid",
+    "external_id": "ext",
+    "provider": "provider",
+    "provider_record": "prec",
+    "ingestion_job": "job",
+    "certification": "cert",
+    "language": "lang",
+    "country": "country",
+    "alternate_title": "altt",
+    "release": "rel",
+}
+
+
+def new_uuid7() -> str:
+    """Return a fresh UUIDv7 as its canonical hyphenated string form."""
+
+    return str(uuid7())
+
+
+def _encode_uuid(uuid_str: str) -> str:
+    """Encode a UUID's 128 bits as lowercase Crockford base32 (26 chars).
+
+    Byte-order-preserving: since the UUID is time-ordered (v7) and this
+    encoding processes bytes most-significant-first, the resulting strings
+    sort lexicographically the same way the UUIDs (and thus creation time)
+    do.
+    """
+
+    raw = bytes.fromhex(uuid_str.replace("-", ""))
+    value = int.from_bytes(raw, "big")
+    chars: list[str] = []
+    # 128 bits / 5 bits-per-symbol = 25.6 -> 26 symbols, top symbol uses 3 bits.
+    for shift in range(125, -5, -5):
+        chars.append(_CROCKFORD_ALPHABET[(value >> shift) & 0b11111])
+    return "".join(chars)
+
+
+def _decode_to_uuid(encoded: str) -> str:
+    if len(encoded) != 26:
+        raise ValueError(f"invalid Aurora ID segment length: {encoded!r}")
+    value = 0
+    for ch in encoded:
+        try:
+            value = (value << 5) | _CROCKFORD_INDEX[ch]
+        except KeyError as exc:
+            raise ValueError(f"invalid Aurora ID character: {ch!r}") from exc
+    try:
+        raw = value.to_bytes(16, "big")
+    except OverflowError as exc:
+        raise ValueError(f"invalid Aurora ID segment: {encoded!r}") from exc
+    hex_str = raw.hex()
+    return f"{hex_str[0:8]}-{hex_str[8:12]}-{hex_str[12:16]}-{hex_str[16:20]}-{hex_str[20:32]}"
+
+
+def aurora_id(entity_type: str, uuid_str: str) -> str:
+    """Build a public Aurora ID string from an entity type and UUIDv7 PK."""
+
+    prefix = ENTITY_PREFIXES.get(entity_type)
+    if prefix is None:
+        raise ValueError(f"unknown Aurora entity type: {entity_type!r}")
+    return f"aur_{prefix}_{_encode_uuid(uuid_str)}"
+
+
+_AURORA_ID_RE = re.compile(r"^aur_(?P<prefix>[a-z_]+)_(?P<encoded>[0-9a-hjkmnp-tv-z]{26})$")
+
+_PREFIX_TO_ENTITY: dict[str, str] = {v: k for k, v in ENTITY_PREFIXES.items()}
+
+
+def parse_aurora_id(aid: str, expected_type: str | None = None) -> str:
+    """Validate an Aurora ID and return the underlying UUID string.
+
+    Raises ValueError if the ID is malformed or (when `expected_type` is
+    given) does not belong to that entity type.
+    """
+
+    match = _AURORA_ID_RE.match(aid)
+    if not match:
+        raise ValueError(f"malformed Aurora ID: {aid!r}")
+    prefix = match.group("prefix")
+    entity_type = _PREFIX_TO_ENTITY.get(prefix)
+    if entity_type is None:
+        raise ValueError(f"unknown Aurora ID prefix: {prefix!r}")
+    if expected_type is not None and entity_type != expected_type:
+        raise ValueError(f"expected Aurora ID of type {expected_type!r}, got {entity_type!r}")
+    return _decode_to_uuid(match.group("encoded"))
+
+
+def is_valid_aurora_id(aid: str, expected_type: str | None = None) -> bool:
+    try:
+        parse_aurora_id(aid, expected_type=expected_type)
+    except ValueError:
+        return False
+    return True

app/core/logging.py

@@ -0,0 +1,62 @@
+"""Structured, JSON-compatible logging with per-request correlation IDs.
+
+Every request gets a request ID (from the `X-Request-ID` header if present,
+otherwise generated) bound into the log context for the lifetime of that
+request, via `RequestIDMiddleware` in `app.main`.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from contextvars import ContextVar
+
+import structlog
+
+from app.core.config import get_settings
+
+request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def _add_request_id(_logger: object, _method_name: str, event_dict: dict) -> dict:
+    request_id = request_id_ctx.get()
+    if request_id is not None:
+        event_dict["request_id"] = request_id
+    return event_dict
+
+
+def configure_logging() -> None:
+    settings = get_settings()
+    level = getattr(logging, settings.AURORA_LOG_LEVEL.upper(), logging.INFO)
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=level,
+    )
+
+    shared_processors: list = [
+        structlog.contextvars.merge_contextvars,
+        _add_request_id,
+        structlog.processors.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+    ]
+
+    renderer: structlog.types.Processor
+    if settings.is_production or settings.AURORA_ENV != "development":
+        renderer = structlog.processors.JSONRenderer()
+    else:
+        renderer = structlog.dev.ConsoleRenderer()
+
+    structlog.configure(
+        processors=[*shared_processors, structlog.processors.format_exc_info, renderer],
+        wrapper_class=structlog.make_filtering_bound_logger(level),
+        context_class=dict,
+        logger_factory=structlog.PrintLoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    return structlog.get_logger(name)

app/core/queue.py

@@ -0,0 +1,35 @@
+"""arq job queue connection (Redis-backed), shared by the API (enqueuing)
+and the worker process (consuming) — see app/workers/worker.py."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+
+from arq import ArqRedis, create_pool
+from arq.connections import RedisSettings
+
+from app.core.config import get_settings
+
+_pool: ArqRedis | None = None
+
+
+def redis_settings() -> RedisSettings:
+    return RedisSettings.from_dsn(get_settings().REDIS_URL)
+
+
+async def get_arq_pool() -> ArqRedis:
+    global _pool
+    if _pool is None:
+        _pool = await create_pool(redis_settings())
+    return _pool
+
+
+async def get_queue() -> AsyncGenerator[ArqRedis, None]:
+    yield await get_arq_pool()
+
+
+async def close_arq_pool() -> None:
+    global _pool
+    if _pool is not None:
+        await _pool.aclose()  # type: ignore[attr-defined]  # present at runtime; stubs lag redis-py 5.x
+        _pool = None

app/core/redis.py

@@ -0,0 +1,50 @@
+"""Redis connection management.
+
+Redis is disposable acceleration/coordination — caching, rate limiting,
+distributed locks, and the arq job queue. It is never the system of record.
+If `REDIS_REQUIRED` is false, the application degrades gracefully (caching
+becomes a no-op, readiness does not depend on Redis) instead of failing when
+Redis is unavailable.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+from functools import lru_cache
+
+import redis.asyncio as redis_asyncio
+from redis.asyncio import Redis
+
+from app.core.config import get_settings
+
+settings = get_settings()
+
+
+@lru_cache
+def get_redis_pool() -> Redis:
+    return redis_asyncio.from_url(
+        settings.REDIS_URL,
+        encoding="utf-8",
+        decode_responses=True,
+        socket_connect_timeout=2,
+        socket_timeout=2,
+    )
+
+
+async def get_redis() -> AsyncGenerator[Redis, None]:
+    """FastAPI dependency yielding the shared Redis client."""
+
+    yield get_redis_pool()
+
+
+async def redis_is_ready() -> bool:
+    try:
+        client = get_redis_pool()
+        return bool(await client.ping())
+    except Exception:
+        return False
+
+
+async def close_redis() -> None:
+    client = get_redis_pool()
+    await client.aclose()  # type: ignore[attr-defined]  # present at runtime; stubs lag redis-py 5.x

app/core/security.py

@@ -0,0 +1,76 @@
+"""Administrative auth and Redis-backed rate limiting.
+
+Public read endpoints (`/api/v1/movies`, `/search`, ...) are unauthenticated
+but rate-limited. Administrative/provider endpoints (ingestion triggers,
+provider config) require an API key via the `X-API-Key` header.
+"""
+
+from __future__ import annotations
+
+import time
+
+from fastapi import Depends, HTTPException, Request, status
+from fastapi.security import APIKeyHeader
+from redis.asyncio import Redis
+
+from app.core.config import get_settings
+from app.core.redis import get_redis
+
+_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+
+async def require_admin_api_key(
+    api_key: str | None = Depends(_api_key_header),
+) -> str:
+    settings = get_settings()
+    allowed = settings.admin_api_keys_set
+    if not allowed:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Administrative API is not configured (AURORA_ADMIN_API_KEYS unset).",
+        )
+    if not api_key or api_key not in allowed:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing or invalid X-API-Key.",
+        )
+    return api_key
+
+
+class RedisRateLimiter:
+    """Fixed-window rate limiter backed by Redis `INCR` + `EXPIRE`.
+
+    Fails open: if Redis is unreachable, requests are allowed through rather
+    than rejected, since Redis is disposable infrastructure and must never
+    become a hard dependency for serving reads.
+    """
+
+    def __init__(self, requests_per_minute: int) -> None:
+        self.requests_per_minute = requests_per_minute
+
+    async def check(self, redis: Redis, key: str) -> tuple[bool, int]:
+        window = int(time.time() // 60)
+        redis_key = f"ratelimit:{key}:{window}"
+        try:
+            count = await redis.incr(redis_key)
+            if count == 1:
+                await redis.expire(redis_key, 60)
+            remaining = max(0, self.requests_per_minute - count)
+            return count <= self.requests_per_minute, remaining
+        except Exception:
+            return True, self.requests_per_minute
+
+
+async def enforce_rate_limit(request: Request, redis: Redis = Depends(get_redis)) -> None:
+    settings = get_settings()
+    if not settings.RATE_LIMIT_ENABLED:
+        return
+    limiter = RedisRateLimiter(settings.RATE_LIMIT_PER_MINUTE)
+    client_id = request.client.host if request.client else "unknown"
+    allowed, remaining = await limiter.check(redis, client_id)
+    if not allowed:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail="Rate limit exceeded.",
+            headers={"Retry-After": "60"},
+        )