feat: per-sink delivery, journal forwarding, and operational API
CI / lint (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / release (push) Canceled after 0s

Persistence:
- one delivery state per sink, with claim, crash recovery, and index replacement
- retention for delivered and dead records
- LMDB writes moved off the event loop

Pipeline:
- one attempt per dispatch, with the next try scheduled by next_retry_at
- backpressure through max_queue_size
- reload re-reads YAML; SIGTERM drains and SIGHUP reloads

Plugins:
- journald source, enricher and filter processors, Loki sink
- built-ins register when entry points are missing

API and CLI:
- health, metrics, per-sink queue, and dead-letter replay
- startup prints the pipeline on one line

Tests:
- dispatch, plugins, metrics, CLI, and persistence coverage
This commit is contained in:
ahp
2026-10-14 01:58:07 +03:30
parent 5e081144d5
commit 06d233054b
38 changed files with 3282 additions and 557 deletions
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from daena.pipeline.processors.enricher import EnricherProcessor
from daena.pipeline.processors.filter import FilterProcessor
from daena.pipeline.record import Record
from daena.pipeline.sinks.loki import LokiSink
from daena.pipeline.sources.journald import (
JournaldSource,
build_journalctl_command,
entry_to_record,
)
def test_journalctl_command_tails_new_entries_without_cursor() -> None:
command = build_journalctl_command(cursor=None, units=["ssh.service"], matches=["_PID=1"])
assert command[:4] == ["journalctl", "--follow", "--output=json", "--no-pager"]
assert "--lines=0" in command
assert command[command.index("--unit") + 1] == "ssh.service"
assert "_PID=1" in command
def test_journalctl_command_uses_cursor() -> None:
command = build_journalctl_command(cursor="abc", units=[], matches=[])
assert "--after-cursor=abc" in command
assert "--lines=0" not in command
def test_entry_to_record_uses_journal_timestamp() -> None:
record = entry_to_record(
"system",
{
"MESSAGE": "hello",
"SYSLOG_IDENTIFIER": "sshd",
"_SYSTEMD_UNIT": "ssh.service",
"__CURSOR": "cur",
"__REALTIME_TIMESTAMP": "1600000000000000",
},
)
assert record.source == "system"
assert record.record_type == "journald"
assert record.body["message"] == "hello"
assert record.body["syslog_identifier"] == "sshd"
assert record.metadata["unit"] == "ssh.service"
assert record.timestamp.startswith("2020-")
class _ListReader:
def __init__(self, entries: list[dict[str, Any]]) -> None:
self._entries = entries
self.stopped = False
async def run(self, on_entry) -> None: # type: ignore[no-untyped-def]
for entry in self._entries:
await on_entry(entry)
async def stop(self) -> None:
self.stopped = True
@pytest.mark.asyncio
async def test_journald_source_emits_and_stores_cursor(tmp_path: Path) -> None:
cursor_path = tmp_path / "cursor"
reader = _ListReader(
[
{"MESSAGE": "one", "__CURSOR": "c1"},
{"MESSAGE": "two", "__CURSOR": "c2"},
]
)
source = JournaldSource(
"journal",
config={"cursor_path": str(cursor_path)},
reader=reader,
)
seen: list[Record] = []
async def capture(record: Record) -> None:
seen.append(record)
source.set_callback(capture)
await source.start()
await asyncio.sleep(0.05)
await source.stop()
assert [record.body["message"] for record in seen] == ["one", "two"]
assert cursor_path.read_text(encoding="utf-8").strip() == "c2"
assert reader.stopped is True
@pytest.mark.asyncio
async def test_enricher_sets_hostname_and_static_fields() -> None:
processor = EnricherProcessor(
"host",
config={"hostname": True, "static": {"env": "prod"}, "target": "metadata"},
)
record = Record(source="s", record_type="t", body={"message": "m"})
result = await processor.process(record)
assert result is not None
assert result.metadata["hostname"]
assert result.metadata["env"] == "prod"
@pytest.mark.asyncio
async def test_filter_drop_and_keep() -> None:
dropper = FilterProcessor("drop", config={"drop": {"body.level": "debug"}})
debug = Record(source="s", record_type="t", body={"level": "debug"})
info = Record(source="s", record_type="t", body={"level": "info"})
assert await dropper.process(debug) is None
assert await dropper.process(info) is not None
keeper = FilterProcessor("keep", config={"keep": {"source": "journal"}})
assert await keeper.process(Record(source="other", record_type="t", body={})) is None
kept = await keeper.process(Record(source="journal", record_type="t", body={}))
assert kept is not None
@pytest.mark.asyncio
async def test_loki_sink_posts_stream() -> None:
response = MagicMock()
response.is_success = True
response.status_code = 204
client = AsyncMock()
client.post = AsyncMock(return_value=response)
client.aclose = AsyncMock()
with patch("daena.pipeline.sinks.loki.httpx.AsyncClient", return_value=client):
sink = LokiSink("loki", config={"url": "http://loki/push", "labels": {"job": "daena"}})
await sink.start()
record = Record(
source="system",
record_type="journald",
body={"message": "hello"},
timestamp="2020-09-13T12:26:40+00:00",
)
result = await sink.deliver(record)
await sink.stop()
assert result.success is True
payload = client.post.await_args.kwargs["json"]
stream = payload["streams"][0]
assert stream["stream"]["job"] == "daena"
assert stream["stream"]["source"] == "system"
assert stream["values"][0][1] == "hello"
assert stream["values"][0][0].isdigit()
json.dumps(payload)