NixReaperandClaude Sonnet 5
c20df1c503
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Production-grade media metadata backend per docs/aurora-core-spec.md: Aurora-owned UUIDv7 identity system, normalized movie/TV/people/credits schema, provider abstraction with a TMDb adapter, an idempotent async ingestion pipeline (normalize -> match -> upsert -> provenance), a Postgres FTS/trigram search layer, a stateless FastAPI v1 API with Redis caching/rate-limiting, arq workers, dev/prod Docker Compose topologies, unit + integration + load tests, CI, and architecture/ database/API/ERD documentation. Co-Authored-By: Claude Sonnet 5 <[email protected]>
109 lines
4.7 KiB
Python
109 lines
4.7 KiB
Python
"""Provider abstraction and provenance tables.
|
|
|
|
`Provider` identifies a metadata source (TMDb, IMDb dataset export, ...).
|
|
`ProviderRecord` is the provenance ledger: one row per (provider, entity
|
|
type, provider-native ID) linking to the Aurora entity it was matched to,
|
|
retaining the raw payload and match confidence so ambiguous matches stay
|
|
reviewable instead of being silently merged (see docs/architecture.md).
|
|
`IngestionJob` tracks async ingestion work so jobs are observable, retryable,
|
|
and protected against duplicate processing via `idempotency_key`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
|
|
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
|
from app.models.enums import IngestionJobStatus, MatchConfidence
|
|
|
|
JSONVariant = JSONB().with_variant(JSON(), "sqlite")
|
|
|
|
|
|
class Provider(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
|
"""A metadata data source. Providers are replaceable adapters, never
|
|
Aurora's identity system — see app/providers/base.py."""
|
|
|
|
__tablename__ = "providers"
|
|
|
|
slug: Mapped[str] = mapped_column(String(50), unique=True, nullable=False) # "tmdb", "imdb"
|
|
name: Mapped[str] = mapped_column(String(200), nullable=False)
|
|
base_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
|
is_active: Mapped[bool] = mapped_column(default=True, nullable=False)
|
|
config: Mapped[dict] = mapped_column(JSONVariant, default=dict)
|
|
|
|
|
|
class ProviderRecord(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
|
"""Provenance ledger row: this provider's native record, and what Aurora
|
|
entity (if any) it has been matched to."""
|
|
|
|
__tablename__ = "provider_records"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"provider_id", "entity_type", "external_id", name="uq_provider_record_source"
|
|
),
|
|
)
|
|
|
|
provider_id: Mapped[uuid.UUID] = mapped_column(
|
|
ForeignKey("providers.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
entity_type: Mapped[str] = mapped_column(String(30), nullable=False)
|
|
external_id: Mapped[str] = mapped_column(String(200), nullable=False)
|
|
|
|
# Polymorphic reference to the Aurora entity this record resolved to.
|
|
# No DB-level FK (the target table varies by entity_type); integrity is
|
|
# enforced by the entity matcher / repositories, not the schema.
|
|
aurora_entity_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True)
|
|
|
|
raw_data: Mapped[dict] = mapped_column(JSONVariant, default=dict)
|
|
source_version: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
|
match_confidence: Mapped[MatchConfidence] = mapped_column(
|
|
default=MatchConfidence.NEW, nullable=False
|
|
)
|
|
match_notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
|
|
ingestion_job_id: Mapped[uuid.UUID | None] = mapped_column(
|
|
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"), nullable=True
|
|
)
|
|
last_synced_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
|
|
provider: Mapped[Provider] = relationship()
|
|
|
|
|
|
class IngestionJob(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
|
"""A unit of asynchronous ingestion work processed by workers."""
|
|
|
|
__tablename__ = "ingestion_jobs"
|
|
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_ingestion_job_idempotency"),)
|
|
|
|
provider_id: Mapped[uuid.UUID] = mapped_column(
|
|
ForeignKey("providers.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
job_type: Mapped[str] = mapped_column(String(50), nullable=False) # "movie_sync", "full_sync"
|
|
entity_type: Mapped[str | None] = mapped_column(String(30), nullable=True)
|
|
external_id: Mapped[str | None] = mapped_column(String(200), nullable=True)
|
|
|
|
idempotency_key: Mapped[str] = mapped_column(String(300), nullable=False)
|
|
status: Mapped[IngestionJobStatus] = mapped_column(
|
|
default=IngestionJobStatus.PENDING, nullable=False
|
|
)
|
|
attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
max_attempts: Mapped[int] = mapped_column(Integer, default=5, nullable=False)
|
|
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
result_aurora_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True)
|
|
|
|
provider: Mapped[Provider] = relationship()
|
|
|
|
|
|
__all__ = ["Provider", "ProviderRecord", "IngestionJob"]
|