feat: per-sink delivery, journal forwarding, and operational API
CI / lint (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / release (push) Canceled after 0s

Persistence:
- one delivery state per sink, with claim, crash recovery, and index replacement
- retention for delivered and dead records
- LMDB writes moved off the event loop

Pipeline:
- one attempt per dispatch, with the next try scheduled by next_retry_at
- backpressure through max_queue_size
- reload re-reads YAML; SIGTERM drains and SIGHUP reloads

Plugins:
- journald source, enricher and filter processors, Loki sink
- built-ins register when entry points are missing

API and CLI:
- health, metrics, per-sink queue, and dead-letter replay
- startup prints the pipeline on one line

Tests:
- dispatch, plugins, metrics, CLI, and persistence coverage
This commit is contained in:
ahp
2026-10-14 01:58:07 +03:30
parent 5e081144d5
commit 06d233054b
38 changed files with 3282 additions and 557 deletions
+30 -4
View File
@@ -8,18 +8,20 @@ import pytest
from daena.api.grpc_server import DaenaGRPCServer
from daena.api.pb2 import daena_pb2 as pb2
from daena.api.pb2 import daena_pb2_grpc as pb2_grpc
from daena.config import DaenaConfig, SinkConfig, SourceConfig
from daena.config import DaenaConfig, PipelineConfig, SinkConfig, SourceConfig
from daena.core.runtime import DaenaRuntime
from daena.persistence.models import RecordState
@pytest.fixture
async def grpc_env():
async def grpc_env(tmp_path):
config = DaenaConfig()
config.sources = [
SourceConfig(name="dummy", type="dummy", options={"interval": 0.2, "message": "test"})
]
config.sinks = [SinkConfig(name="log", type="dummy")]
config.persistence.lmdb.path = "/tmp/daena_grpc_test"
config.persistence.lmdb.path = str(tmp_path / "lmdb")
config.pipeline = PipelineConfig(batch_size=100, flush_interval=5.0, max_queue_size=10_000)
config.api.grpc.port = 18642
runtime = DaenaRuntime(config)
@@ -76,7 +78,7 @@ async def test_get_queue_state(grpc_env) -> None:
await asyncio.sleep(1)
resp = await stub.GetQueueState(pb2.Empty(), timeout=5)
assert resp.total > 0
assert resp.pending > 0
assert resp.pending + resp.delivered > 0
@pytest.mark.asyncio
@@ -98,6 +100,30 @@ async def test_list_sinks(grpc_env) -> None:
assert resp.sinks[0].name == "log"
@pytest.mark.asyncio
async def test_replay_dead(grpc_env) -> None:
runtime, stub, _ = grpc_env
await asyncio.sleep(1)
assert runtime.backend is not None
stored = None
for state in (RecordState.pending, RecordState.delivered, RecordState.failed):
found = runtime.backend.list_by_state(state, limit=1)
if found:
stored = found[0]
break
assert stored is not None
for sink in stored.deliveries:
runtime.backend.dead(stored.record_id, "grpc replay", sink=sink)
resp = await stub.ReplayDead(
pb2.ReplayRequest(record_id=stored.record_id, limit=10),
timeout=5,
)
assert resp.replayed == 1
again = runtime.backend.get(stored.record_id)
assert again is not None
assert again.deliveries["log"].state == RecordState.pending
@pytest.mark.asyncio
async def test_reload_config(grpc_env) -> None:
_, stub, _ = grpc_env