feat: per-sink delivery, journal forwarding, and operational API
Persistence: - one delivery state per sink, with claim, crash recovery, and index replacement - retention for delivered and dead records - LMDB writes moved off the event loop Pipeline: - one attempt per dispatch, with the next try scheduled by next_retry_at - backpressure through max_queue_size - reload re-reads YAML; SIGTERM drains and SIGHUP reloads Plugins: - journald source, enricher and filter processors, Loki sink - built-ins register when entry points are missing API and CLI: - health, metrics, per-sink queue, and dead-letter replay - startup prints the pipeline on one line Tests: - dispatch, plugins, metrics, CLI, and persistence coverage
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from daena.config import (
|
||||
DaenaConfig,
|
||||
MetricsConfig,
|
||||
ProcessorConfig,
|
||||
SinkConfig,
|
||||
SourceConfig,
|
||||
)
|
||||
from daena.core.layout import preview_message, render_layout
|
||||
|
||||
|
||||
def _config() -> DaenaConfig:
|
||||
config = DaenaConfig()
|
||||
config.persistence.lmdb.path = "/tmp/daena/data"
|
||||
config.sources = [
|
||||
SourceConfig(name="dummy", type="dummy", options={"interval": 2.0}),
|
||||
SourceConfig(
|
||||
name="system",
|
||||
type="journald",
|
||||
enabled=False,
|
||||
options={"units": ["ssh.service"]},
|
||||
),
|
||||
]
|
||||
config.processors = [
|
||||
ProcessorConfig(name="host", type="enricher", options={"static": {"env": "local"}}),
|
||||
ProcessorConfig(name="keep", type="filter", options={"drop": {"body.priority": "7"}}),
|
||||
]
|
||||
config.sinks = [
|
||||
SinkConfig(name="log", type="dummy"),
|
||||
SinkConfig(
|
||||
name="loki",
|
||||
type="loki",
|
||||
options={"url": "http://127.0.0.1:3100/loki/api/v1/push"},
|
||||
),
|
||||
]
|
||||
config.logging.verbose = True
|
||||
return config
|
||||
|
||||
|
||||
def test_layout_is_a_single_pipeline_line() -> None:
|
||||
text = render_layout(_config(), "./daena.yaml")
|
||||
assert "┌" not in text
|
||||
assert "▼" not in text
|
||||
flow = text.splitlines()[1]
|
||||
assert flow.index("dummy") < flow.index("host")
|
||||
assert flow.index("host") < flow.index("lmdb")
|
||||
assert flow.index("lmdb") < flow.index("log")
|
||||
assert "daena 0.1.0 ./daena.yaml" in text
|
||||
assert "/tmp/daena/data" in text
|
||||
assert "127.0.0.1:8642 127.0.0.1:8643" in text
|
||||
|
||||
|
||||
def test_layout_names_each_plugin_once() -> None:
|
||||
text = render_layout(_config(), None)
|
||||
flow = text.splitlines()[1]
|
||||
assert flow == " → ".join(
|
||||
[
|
||||
"dummy (2s), system (journald, off)",
|
||||
"host (enricher), keep (filter)",
|
||||
"lmdb",
|
||||
"log (dummy), loki (127.0.0.1:3100)",
|
||||
]
|
||||
)
|
||||
assert "daena 0.1.0 defaults" in text
|
||||
|
||||
|
||||
def test_layout_skips_empty_stages_and_disabled_metrics() -> None:
|
||||
config = DaenaConfig()
|
||||
config.api.metrics = MetricsConfig(enabled=False)
|
||||
text = render_layout(config, None)
|
||||
assert "→ lmdb" not in text
|
||||
assert text.splitlines()[1] == "lmdb"
|
||||
assert text.splitlines()[-1] == "127.0.0.1:8642"
|
||||
|
||||
|
||||
def test_preview_message_uses_message_then_clips() -> None:
|
||||
assert preview_message({"message": "daena heartbeat #1"}) == "daena heartbeat #1"
|
||||
assert preview_message({"message": "a\nb"}) == "a b"
|
||||
assert preview_message({"counter": 1}) == '{"counter":1}'
|
||||
clipped = preview_message({"message": "x" * 130}, limit=10)
|
||||
assert clipped.endswith("…")
|
||||
assert len(clipped) == 10
|
||||
Reference in New Issue
Block a user