admin / Syanpse-Vanguard
public
Syanpse-Vanguard / synapse-vanguard-v3 / sv / storage / base.py
2428 B · main
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Storage Abstraction Layer. The pipeline, query API, auth store, and agent registry depend ONLY on this interface. DuckDB (embedded, zero-config) vs ClickHouse (scaled) is chosen at startup by build_engine() reading SV_STORAGE_BACKEND. Adding a backend == implementing this class; no caller changes.""" from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime from typing import Any, AsyncIterator, Sequence import pyarrow as pa from sv.models.event import EventModel @dataclass class Query: """A parameterized query. Callers never build SQL by string concatenation; the query builder / NL->SQL translator produce these, and params are bound by the driver — this is the SQL-injection boundary.""" text: str params: list[Any] = field(default_factory=list) class StorageEngine(ABC): @abstractmethod async def init_schema(self) -> None: ... @abstractmethod async def write_batch(self, events: Sequence[EventModel]) -> int: ... @abstractmethod async def execute(self, query: Query) -> pa.Table: ... @abstractmethod async def stream(self, query: Query, batch_rows: int = 10_000 ) -> AsyncIterator[pa.RecordBatch]: ... @abstractmethod async def retention_sweep(self, older_than: datetime) -> int: ... # --- control-plane helpers (users, agents) shared by every backend ------ @abstractmethod async def fetchone(self, query: Query) -> dict | None: ... @abstractmethod async def fetchall(self, query: Query) -> list[dict]: ... @abstractmethod async def run(self, query: Query) -> None: """Execute a write statement (INSERT/UPDATE/DELETE) with bound params.""" @abstractmethod async def close(self) -> None: ... def build_engine() -> StorageEngine: from sv.config import settings if settings.storage_backend == "duckdb": from sv.storage.duckdb_engine import DuckDBEngine return DuckDBEngine(path=settings.duckdb_path, cold_dir=settings.cold_dir) if settings.storage_backend == "clickhouse": # Stub target for milestone-scale-out; implements the same interface. from sv.storage.clickhouse_engine import ClickHouseEngine return ClickHouseEngine(dsn=settings.clickhouse_dsn) raise ValueError(f"unknown SV_STORAGE_BACKEND={settings.storage_backend!r}") |