Initial Aurora Core 0.1 implementation
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / lint (push) Canceled after 0s
CI / typecheck (push) Canceled after 0s
CI / unit-tests (push) Canceled after 0s
CI / integration-tests (push) Canceled after 0s
CI / container-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
Production-grade media metadata backend per docs/aurora-core-spec.md: Aurora-owned UUIDv7 identity system, normalized movie/TV/people/credits schema, provider abstraction with a TMDb adapter, an idempotent async ingestion pipeline (normalize -> match -> upsert -> provenance), a Postgres FTS/trigram search layer, a stateless FastAPI v1 API with Redis caching/rate-limiting, arq workers, dev/prod Docker Compose topologies, unit + integration + load tests, CI, and architecture/ database/API/ERD documentation. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
"""Provider abstraction and provenance tables.
|
||||
|
||||
`Provider` identifies a metadata source (TMDb, IMDb dataset export, ...).
|
||||
`ProviderRecord` is the provenance ledger: one row per (provider, entity
|
||||
type, provider-native ID) linking to the Aurora entity it was matched to,
|
||||
retaining the raw payload and match confidence so ambiguous matches stay
|
||||
reviewable instead of being silently merged (see docs/architecture.md).
|
||||
`IngestionJob` tracks async ingestion work so jobs are observable, retryable,
|
||||
and protected against duplicate processing via `idempotency_key`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.enums import IngestionJobStatus, MatchConfidence
|
||||
|
||||
JSONVariant = JSONB().with_variant(JSON(), "sqlite")
|
||||
|
||||
|
||||
class Provider(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
||||
"""A metadata data source. Providers are replaceable adapters, never
|
||||
Aurora's identity system — see app/providers/base.py."""
|
||||
|
||||
__tablename__ = "providers"
|
||||
|
||||
slug: Mapped[str] = mapped_column(String(50), unique=True, nullable=False) # "tmdb", "imdb"
|
||||
name: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
base_url: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
is_active: Mapped[bool] = mapped_column(default=True, nullable=False)
|
||||
config: Mapped[dict] = mapped_column(JSONVariant, default=dict)
|
||||
|
||||
|
||||
class ProviderRecord(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
||||
"""Provenance ledger row: this provider's native record, and what Aurora
|
||||
entity (if any) it has been matched to."""
|
||||
|
||||
__tablename__ = "provider_records"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"provider_id", "entity_type", "external_id", name="uq_provider_record_source"
|
||||
),
|
||||
)
|
||||
|
||||
provider_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("providers.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
entity_type: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
external_id: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
|
||||
# Polymorphic reference to the Aurora entity this record resolved to.
|
||||
# No DB-level FK (the target table varies by entity_type); integrity is
|
||||
# enforced by the entity matcher / repositories, not the schema.
|
||||
aurora_entity_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True)
|
||||
|
||||
raw_data: Mapped[dict] = mapped_column(JSONVariant, default=dict)
|
||||
source_version: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||
match_confidence: Mapped[MatchConfidence] = mapped_column(
|
||||
default=MatchConfidence.NEW, nullable=False
|
||||
)
|
||||
match_notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
ingestion_job_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
last_synced_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
provider: Mapped[Provider] = relationship()
|
||||
|
||||
|
||||
class IngestionJob(Base, UUIDPrimaryKeyMixin, TimestampMixin):
|
||||
"""A unit of asynchronous ingestion work processed by workers."""
|
||||
|
||||
__tablename__ = "ingestion_jobs"
|
||||
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_ingestion_job_idempotency"),)
|
||||
|
||||
provider_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("providers.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
job_type: Mapped[str] = mapped_column(String(50), nullable=False) # "movie_sync", "full_sync"
|
||||
entity_type: Mapped[str | None] = mapped_column(String(30), nullable=True)
|
||||
external_id: Mapped[str | None] = mapped_column(String(200), nullable=True)
|
||||
|
||||
idempotency_key: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||
status: Mapped[IngestionJobStatus] = mapped_column(
|
||||
default=IngestionJobStatus.PENDING, nullable=False
|
||||
)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, default=5, nullable=False)
|
||||
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
result_aurora_id: Mapped[uuid.UUID | None] = mapped_column(nullable=True)
|
||||
|
||||
provider: Mapped[Provider] = relationship()
|
||||
|
||||
|
||||
__all__ = ["Provider", "ProviderRecord", "IngestionJob"]
|
||||
Reference in New Issue
Block a user