Catalyst / admin/Syanpse-Vanguard 14.8 GB / 57.8 GB 40.0 GB free
Help Sign in

admin / Syanpse-Vanguard

public
Code Issues Pull requests Pipelines Packages Security Insights Wiki Settings
Syanpse-Vanguard / synapse-vanguard-v3 / agents / common / buffer.py 2358 B · main
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""Local disk buffer (SQLite ring) so no logs are lost during a server/network
outage. Each event gets a monotonic autoincrement id; the sender uses the max id
in a batch as the bundle seq and trims rows <= the server-acked last_seq. Replay
after reconnect is therefore idempotent server-side."""
from __future__ import annotations

import json
import os
import sqlite3
import threading
import time


class DiskBuffer:
    def __init__(self, path: str, max_rows: int = 500_000):
        os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
        self._db = sqlite3.connect(path, check_same_thread=False)
        self._db.execute("PRAGMA journal_mode=WAL")
        self._db.execute(
            "CREATE TABLE IF NOT EXISTS buf ("
            "id INTEGER PRIMARY KEY AUTOINCREMENT, ts REAL, line TEXT)"
        )
        self._db.commit()
        self._max_rows = max_rows
        self._lock = threading.Lock()

    def append(self, event: dict) -> None:
        line = json.dumps(event, default=str)
        with self._lock:
            self._db.execute("INSERT INTO buf (ts, line) VALUES (?, ?)",
                             (time.time(), line))
            self._db.commit()
            self._enforce_cap()

    def _enforce_cap(self) -> None:
        row = self._db.execute("SELECT COUNT(*) FROM buf").fetchone()
        if row and row[0] > self._max_rows:
            # Drop the oldest overflow — bounded disk use; newest data wins.
            excess = row[0] - self._max_rows
            self._db.execute(
                "DELETE FROM buf WHERE id IN "
                "(SELECT id FROM buf ORDER BY id ASC LIMIT ?)", (excess,))
            self._db.commit()

    def next_batch(self, limit: int) -> tuple[int, list[str]]:
        """Return (max_id, lines). max_id becomes the bundle seq."""
        with self._lock:
            rows = self._db.execute(
                "SELECT id, line FROM buf ORDER BY id ASC LIMIT ?", (limit,)
            ).fetchall()
        if not rows:
            return 0, []
        return rows[-1][0], [r[1] for r in rows]

    def trim(self, last_id: int) -> None:
        with self._lock:
            self._db.execute("DELETE FROM buf WHERE id <= ?", (last_id,))
            self._db.commit()

    def pending(self) -> int:
        return self._db.execute("SELECT COUNT(*) FROM buf").fetchone()[0]