feat: per-sink delivery, journal forwarding, and operational API
Persistence: - one delivery state per sink, with claim, crash recovery, and index replacement - retention for delivered and dead records - LMDB writes moved off the event loop Pipeline: - one attempt per dispatch, with the next try scheduled by next_retry_at - backpressure through max_queue_size - reload re-reads YAML; SIGTERM drains and SIGHUP reloads Plugins: - journald source, enricher and filter processors, Loki sink - built-ins register when entry points are missing API and CLI: - health, metrics, per-sink queue, and dead-letter replay - startup prints the pipeline on one line Tests: - dispatch, plugins, metrics, CLI, and persistence coverage
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from daena.pipeline.processors.enricher import EnricherProcessor
|
||||
from daena.pipeline.processors.filter import FilterProcessor
|
||||
from daena.pipeline.record import Record
|
||||
from daena.pipeline.sinks.loki import LokiSink
|
||||
from daena.pipeline.sources.journald import (
|
||||
JournaldSource,
|
||||
build_journalctl_command,
|
||||
entry_to_record,
|
||||
)
|
||||
|
||||
|
||||
def test_journalctl_command_tails_new_entries_without_cursor() -> None:
|
||||
command = build_journalctl_command(cursor=None, units=["ssh.service"], matches=["_PID=1"])
|
||||
assert command[:4] == ["journalctl", "--follow", "--output=json", "--no-pager"]
|
||||
assert "--lines=0" in command
|
||||
assert command[command.index("--unit") + 1] == "ssh.service"
|
||||
assert "_PID=1" in command
|
||||
|
||||
|
||||
def test_journalctl_command_uses_cursor() -> None:
|
||||
command = build_journalctl_command(cursor="abc", units=[], matches=[])
|
||||
assert "--after-cursor=abc" in command
|
||||
assert "--lines=0" not in command
|
||||
|
||||
|
||||
def test_entry_to_record_uses_journal_timestamp() -> None:
|
||||
record = entry_to_record(
|
||||
"system",
|
||||
{
|
||||
"MESSAGE": "hello",
|
||||
"SYSLOG_IDENTIFIER": "sshd",
|
||||
"_SYSTEMD_UNIT": "ssh.service",
|
||||
"__CURSOR": "cur",
|
||||
"__REALTIME_TIMESTAMP": "1600000000000000",
|
||||
},
|
||||
)
|
||||
assert record.source == "system"
|
||||
assert record.record_type == "journald"
|
||||
assert record.body["message"] == "hello"
|
||||
assert record.body["syslog_identifier"] == "sshd"
|
||||
assert record.metadata["unit"] == "ssh.service"
|
||||
assert record.timestamp.startswith("2020-")
|
||||
|
||||
|
||||
class _ListReader:
|
||||
def __init__(self, entries: list[dict[str, Any]]) -> None:
|
||||
self._entries = entries
|
||||
self.stopped = False
|
||||
|
||||
async def run(self, on_entry) -> None: # type: ignore[no-untyped-def]
|
||||
for entry in self._entries:
|
||||
await on_entry(entry)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self.stopped = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_journald_source_emits_and_stores_cursor(tmp_path: Path) -> None:
|
||||
cursor_path = tmp_path / "cursor"
|
||||
reader = _ListReader(
|
||||
[
|
||||
{"MESSAGE": "one", "__CURSOR": "c1"},
|
||||
{"MESSAGE": "two", "__CURSOR": "c2"},
|
||||
]
|
||||
)
|
||||
source = JournaldSource(
|
||||
"journal",
|
||||
config={"cursor_path": str(cursor_path)},
|
||||
reader=reader,
|
||||
)
|
||||
seen: list[Record] = []
|
||||
|
||||
async def capture(record: Record) -> None:
|
||||
seen.append(record)
|
||||
|
||||
source.set_callback(capture)
|
||||
await source.start()
|
||||
await asyncio.sleep(0.05)
|
||||
await source.stop()
|
||||
assert [record.body["message"] for record in seen] == ["one", "two"]
|
||||
assert cursor_path.read_text(encoding="utf-8").strip() == "c2"
|
||||
assert reader.stopped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enricher_sets_hostname_and_static_fields() -> None:
|
||||
processor = EnricherProcessor(
|
||||
"host",
|
||||
config={"hostname": True, "static": {"env": "prod"}, "target": "metadata"},
|
||||
)
|
||||
record = Record(source="s", record_type="t", body={"message": "m"})
|
||||
result = await processor.process(record)
|
||||
assert result is not None
|
||||
assert result.metadata["hostname"]
|
||||
assert result.metadata["env"] == "prod"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_drop_and_keep() -> None:
|
||||
dropper = FilterProcessor("drop", config={"drop": {"body.level": "debug"}})
|
||||
debug = Record(source="s", record_type="t", body={"level": "debug"})
|
||||
info = Record(source="s", record_type="t", body={"level": "info"})
|
||||
assert await dropper.process(debug) is None
|
||||
assert await dropper.process(info) is not None
|
||||
|
||||
keeper = FilterProcessor("keep", config={"keep": {"source": "journal"}})
|
||||
assert await keeper.process(Record(source="other", record_type="t", body={})) is None
|
||||
kept = await keeper.process(Record(source="journal", record_type="t", body={}))
|
||||
assert kept is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loki_sink_posts_stream() -> None:
|
||||
response = MagicMock()
|
||||
response.is_success = True
|
||||
response.status_code = 204
|
||||
client = AsyncMock()
|
||||
client.post = AsyncMock(return_value=response)
|
||||
client.aclose = AsyncMock()
|
||||
with patch("daena.pipeline.sinks.loki.httpx.AsyncClient", return_value=client):
|
||||
sink = LokiSink("loki", config={"url": "http://loki/push", "labels": {"job": "daena"}})
|
||||
await sink.start()
|
||||
record = Record(
|
||||
source="system",
|
||||
record_type="journald",
|
||||
body={"message": "hello"},
|
||||
timestamp="2020-09-13T12:26:40+00:00",
|
||||
)
|
||||
result = await sink.deliver(record)
|
||||
await sink.stop()
|
||||
assert result.success is True
|
||||
payload = client.post.await_args.kwargs["json"]
|
||||
stream = payload["streams"][0]
|
||||
assert stream["stream"]["job"] == "daena"
|
||||
assert stream["stream"]["source"] == "system"
|
||||
assert stream["values"][0][1] == "hello"
|
||||
assert stream["values"][0][0].isdigit()
|
||||
json.dumps(payload)
|
||||
Reference in New Issue
Block a user