diff --git a/.gitignore b/.gitignore index f558782..80e7755 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ data/ logs/ *.pid *.sock +/daena.yaml +/deploy/graylog/ # Env .env diff --git a/Dockerfile b/Dockerfile index c68daba..8d7d666 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ COPY proto/ /app/proto/ ENV PATH="/app/.venv/bin:$PATH" \ DAENA_CONFIG_PATH=/etc/daena/daena.yaml -EXPOSE 8642 +EXPOSE 8642 8643 USER daena ENTRYPOINT ["daena-serve"] diff --git a/README.md b/README.md index ab344de..96e54c3 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ Daena is a modular, extensible Linux service that: ## Architecture ``` -Sources → [Processors] → Pipeline → Persistence (LMDB) → Sink Dispatcher → Remote - ↻ retry queue +Sources → [Processors] → LMDB (one delivery per sink) → dispatchers + ↻ per-sink retry, no head-of-line sleep ``` Key design decisions: @@ -67,8 +67,9 @@ asyncio.run(main()) # Run tests uv run pytest tests/ -v -# Start gRPC server +# Start gRPC server. Startup prints the pipeline. --verbose logs each record. uv run daena-serve +uv run daena-serve --verbose # Use the CLI (point at a running daena gRPC API) uv run daena status @@ -160,14 +161,15 @@ The gRPC API runs on `127.0.0.1:8642` with the following RPCs: | RPC | Description | |-----|-------------| | `GetStatus` | Service status and version | -| `GetHealth` | Health check (backend, plugins, runtime) | +| `GetHealth` | Health, including oldest inflight age (`ok`, `degraded`, `down`) | | `GetConfig` | Loaded configuration as JSON | | `ListPlugins` | Registered sources, processors, and sinks | -| `GetQueueState` | Queue state counts (pending, delivered, failed, dead) | +| `GetQueueState` | Delivery counts, including a per-sink breakdown | | `ListPendingRecords` | Pending records with pagination | | `ListDeadRecords` | Dead-letter records with pagination | | `ListSinks` | Configured sinks | -| `ReloadConfig` | Reload service configuration at runtime | +| `ReloadConfig` | Re-read the config file and restart the pipeline | +| `ReplayDead` | Requeue dead-letter deliveries | ### Reflect the gRPC service @@ -182,9 +184,12 @@ grpcurl -plaintext 127.0.0.1:8642 describe daena.v1.Daena Daena provides a CLI that connects to the gRPC server: ```bash -daena status # Show service status -daena health # Check health -daena queue # Inspect queue state +daena --target 127.0.0.1:8642 status +daena health # ok / degraded / down +daena queue # Inspect queue state, per sink +daena records pending +daena records dead +daena replay # Requeue dead-letter deliveries daena sinks # List configured sinks daena plugins # List registered plugins daena config # Show loaded configuration @@ -250,6 +255,21 @@ my_source = "my_sources:MySource" Same pattern applies for `Processor` and `Sink` base classes. +Built-in plugins: + +| Kind | Name | Role | +|------|------|------| +| source | `dummy` | Interval test events (`count` stops after N) | +| source | `journald` | `journalctl --follow` with an optional cursor file | +| processor | `passthrough` | No-op | +| processor | `enricher` | Hostname and static fields | +| processor | `filter` | Drop or keep by field equality | +| sink | `dummy` | Accept locally | +| sink | `http` | JSON POST | +| sink | `loki` | Loki push API | + +`SIGHUP` re-reads the config file (systemd `ExecReload`). `SIGTERM` drains and closes LMDB. Prometheus text is served at `http://127.0.0.1:8643/metrics` when metrics are enabled. Each sink has its own delivery state, and a failed attempt is scheduled with `next_retry_at` instead of blocking the dispatcher. + ## Branch workflow - `main` — stable integration branch diff --git a/config/daena.yaml b/config/daena.yaml index 9b82129..1d3145d 100644 --- a/config/daena.yaml +++ b/config/daena.yaml @@ -8,18 +8,59 @@ pid_file: /run/daena/daena.pid logging: level: info format: json + verbose: false api: grpc: host: 127.0.0.1 port: 8642 max_workers: 10 + metrics: + enabled: true + host: 127.0.0.1 + port: 8643 pipeline: batch_size: 100 flush_interval: 5.0 max_queue_size: 10000 +retention: + delivered_ttl_seconds: 86400 + dead_ttl_seconds: 604800 + interval_seconds: 60 + +health: + max_pending_age_seconds: 300 + +# journald source, enricher, filter, and loki sink are registered plugins. +# Example: +# sources: +# - name: system +# type: journald +# options: +# units: ["ssh.service"] +# cursor_path: /var/lib/daena/journal.cursor +# processors: +# - name: host +# type: enricher +# options: +# hostname: true +# static: +# env: prod +# - name: drop_debug +# type: filter +# options: +# drop: +# body.priority: "7" +# sinks: +# - name: loki +# type: loki +# options: +# url: http://127.0.0.1:3100/loki/api/v1/push +# labels: +# job: daena + persistence: backend: lmdb lmdb: diff --git a/proto/daena/v1/daena.proto b/proto/daena/v1/daena.proto index 7091adb..a8154c2 100644 --- a/proto/daena/v1/daena.proto +++ b/proto/daena/v1/daena.proto @@ -12,6 +12,7 @@ service Daena { rpc ListDeadRecords(Pagination) returns (RecordList); rpc ListSinks(Empty) returns (SinkList); rpc ReloadConfig(Empty) returns (ReloadResponse); + rpc ReplayDead(ReplayRequest) returns (ReplayResponse); } message Empty {} @@ -27,6 +28,9 @@ message HealthResponse { bool running = 2; bool backend = 3; bool plugins_loaded = 4; + string status = 5; + int64 oldest_pending_age_seconds = 6; + string detail = 7; } message ConfigResponse { @@ -48,12 +52,23 @@ message PluginListResponse { PluginCategory sinks = 3; } +message SinkQueueState { + string sink = 1; + int64 pending = 2; + int64 delivering = 3; + int64 delivered = 4; + int64 failed = 5; + int64 dead = 6; +} + message QueueStateResponse { int64 pending = 1; int64 delivered = 2; int64 failed = 3; int64 dead = 4; int64 total = 5; + repeated SinkQueueState sinks = 6; + int64 delivering = 7; } message Pagination { @@ -61,6 +76,15 @@ message Pagination { int32 offset = 2; } +message DeliveryData { + string sink = 1; + string state = 2; + int32 retry_count = 3; + string next_retry_at = 4; + string last_error = 5; + string updated_at = 6; +} + message RecordData { string record_id = 1; string source = 2; @@ -74,6 +98,7 @@ message RecordData { string next_retry_at = 10; string last_error = 11; string destination = 12; + repeated DeliveryData deliveries = 13; } message RecordList { @@ -94,3 +119,13 @@ message ReloadResponse { bool success = 1; string message = 2; } + +message ReplayRequest { + string record_id = 1; + string sink = 2; + int32 limit = 3; +} + +message ReplayResponse { + int32 replayed = 1; +} diff --git a/pyproject.toml b/pyproject.toml index e10bd11..5eff909 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,13 +50,17 @@ daena-serve = "daena.app:serve" [project.entry-points."daena.sources"] dummy = "daena.pipeline.sources.dummy:DummySource" +journald = "daena.pipeline.sources.journald:JournaldSource" [project.entry-points."daena.sinks"] dummy = "daena.pipeline.sinks.dummy:DummySink" http = "daena.pipeline.sinks.http:HTTPSink" +loki = "daena.pipeline.sinks.loki:LokiSink" [project.entry-points."daena.processors"] passthrough = "daena.pipeline.processors.passthrough:PassthroughProcessor" +enricher = "daena.pipeline.processors.enricher:EnricherProcessor" +filter = "daena.pipeline.processors.filter:FilterProcessor" [tool.pyproject-fmt] max_blank_lines = 1 diff --git a/src/daena/api/grpc_server.py b/src/daena/api/grpc_server.py index 0ef2c6a..cf33545 100644 --- a/src/daena/api/grpc_server.py +++ b/src/daena/api/grpc_server.py @@ -1,33 +1,47 @@ from __future__ import annotations +import asyncio import json import grpc import grpc.aio +from daena._version import __version__ from daena.api.pb2 import daena_pb2 as pb2 from daena.api.pb2 import daena_pb2_grpc as pb2_grpc from daena.core.runtime import DaenaRuntime from daena.logutil import get_logger -from daena.persistence.models import RecordState +from daena.persistence.models import RecordState, StoredRecord log = get_logger("daena.grpc") -def _record_to_proto(record) -> pb2.RecordData: +def _record_to_proto(record: StoredRecord) -> pb2.RecordData: + deliveries = [ + pb2.DeliveryData( + sink=name, + state=item.state.value, + retry_count=item.retry_count, + next_retry_at=item.next_retry_at or "", + last_error=item.last_error or "", + updated_at=item.updated_at or "", + ) + for name, item in sorted(record.deliveries.items()) + ] return pb2.RecordData( record_id=record.record_id, source=record.source, record_type=record.record_type, body=json.dumps(record.body), metadata=json.dumps(record.metadata), - state=record.state.value if hasattr(record.state, "value") else record.state, + state=record.state.value, created_at=record.created_at or "", updated_at=record.updated_at or "", retry_count=record.retry_count, next_retry_at=record.next_retry_at or "", last_error=record.last_error or "", destination=record.destination or "", + deliveries=deliveries, ) @@ -39,16 +53,20 @@ class DaenaServicer(pb2_grpc.DaenaServicer): async def GetStatus(self, request, context): return pb2.StatusResponse( service="daena", - version="0.1.0", - running=self._runtime._running, + version=__version__, + running=self._runtime.running, ) async def GetHealth(self, request, context): + report = await asyncio.to_thread(self._runtime.health) return pb2.HealthResponse( - healthy=self._runtime._running, - running=self._runtime._running, - backend=self._runtime.backend is not None, - plugins_loaded=bool(self._runtime.registry.list_sources()), + healthy=report.healthy, + running=report.running, + backend=report.backend, + plugins_loaded=report.plugins_loaded, + status=report.status, + oldest_pending_age_seconds=report.oldest_pending_age_seconds, + detail=report.detail, ) async def GetConfig(self, request, context): @@ -56,12 +74,21 @@ class DaenaServicer(pb2_grpc.DaenaServicer): raw = { "data_dir": cfg.data_dir, "log_dir": cfg.log_dir, - "api": {"grpc": {"host": cfg.api.grpc.host, "port": cfg.api.grpc.port}}, + "api": { + "grpc": { + "host": cfg.api.grpc.host, + "port": cfg.api.grpc.port, + "max_workers": cfg.api.grpc.max_workers, + }, + "metrics": cfg.api.metrics.model_dump(), + }, "pipeline": cfg.pipeline.model_dump(), "persistence": cfg.persistence.model_dump(), - "sources": [s.model_dump() for s in cfg.sources], - "processors": [p.model_dump() for p in cfg.processors], - "sinks": [s.model_dump() for s in cfg.sinks], + "retention": cfg.retention.model_dump(), + "health": cfg.health.model_dump(), + "sources": [item.model_dump() for item in cfg.sources], + "processors": [item.model_dump() for item in cfg.processors], + "sinks": [item.model_dump() for item in cfg.sinks], } return pb2.ConfigResponse(config_json=json.dumps(raw)) @@ -69,11 +96,12 @@ class DaenaServicer(pb2_grpc.DaenaServicer): registry = self._runtime.registry all_plugins = registry.all_plugins() - def _cat(key: str): + def _cat(key: str) -> pb2.PluginCategory: items = all_plugins.get(key, {}) - return pb2.PluginCategory( - items=[pb2.PluginInfo(name=n, class_name=cls.__name__) for n, cls in items.items()] - ) + infos = [ + pb2.PluginInfo(name=name, class_name=cls.__name__) for name, cls in items.items() + ] + return pb2.PluginCategory(items=infos) return pb2.PluginListResponse( sources=_cat("sources"), @@ -85,69 +113,103 @@ class DaenaServicer(pb2_grpc.DaenaServicer): backend = self._runtime.backend if backend is None: return pb2.QueueStateResponse() + pending, delivering, delivered, failed, dead, total, sinks = await asyncio.to_thread( + self._queue_counts + ) return pb2.QueueStateResponse( - pending=backend.count(RecordState.pending), - delivered=backend.count(RecordState.delivered), - failed=backend.count(RecordState.failed), - dead=backend.count(RecordState.dead), - total=backend.count(), + pending=pending, + delivering=delivering, + delivered=delivered, + failed=failed, + dead=dead, + total=total, + sinks=[ + pb2.SinkQueueState( + sink=name, + pending=counts.get("pending", 0), + delivering=counts.get("delivering", 0), + delivered=counts.get("delivered", 0), + failed=counts.get("failed", 0), + dead=counts.get("dead", 0), + ) + for name, counts in sorted(sinks.items()) + ], + ) + + def _queue_counts( + self, + ) -> tuple[int, int, int, int, int, int, dict[str, dict[str, int]]]: + backend = self._runtime.backend + if backend is None: + return (0, 0, 0, 0, 0, 0, {}) + return ( + backend.count(RecordState.pending), + backend.count(RecordState.delivering), + backend.count(RecordState.delivered), + backend.count(RecordState.failed), + backend.count(RecordState.dead), + backend.count(), + backend.sink_counts(), ) async def ListPendingRecords(self, request, context): - backend = self._runtime.backend - if backend is None: - return pb2.RecordList(records=[], count=0) - records = backend.list_by_state( - RecordState.pending, - limit=request.limit or 100, - offset=request.offset or 0, - ) - return pb2.RecordList( - records=[_record_to_proto(r) for r in records], - count=len(records), - ) + return await self._list(RecordState.pending, request.limit or 100, request.offset or 0) async def ListDeadRecords(self, request, context): + return await self._list(RecordState.dead, request.limit or 100, request.offset or 0) + + async def _list(self, state: RecordState, limit: int, offset: int) -> pb2.RecordList: backend = self._runtime.backend if backend is None: return pb2.RecordList(records=[], count=0) - records = backend.list_by_state( - RecordState.dead, - limit=request.limit or 100, - offset=request.offset or 0, - ) - return pb2.RecordList( - records=[_record_to_proto(r) for r in records], - count=len(records), - ) + records = await asyncio.to_thread(backend.list_by_state, state, limit, offset) + payload = [_record_to_proto(record) for record in records] + return pb2.RecordList(records=payload, count=len(payload)) async def ListSinks(self, request, context): pipeline = self._runtime.pipeline if pipeline is None: return pb2.SinkList(sinks=[]) - return pb2.SinkList( - sinks=[pb2.SinkInfo(name=s.name, type=type(s).__name__) for s in pipeline.sinks] - ) + types = pipeline.sink_types() + infos = [ + pb2.SinkInfo(name=sink.name, type=types.get(sink.name, "")) for sink in pipeline.sinks + ] + return pb2.SinkList(sinks=infos) async def ReloadConfig(self, request, context): try: await self._runtime.reload() - return pb2.ReloadResponse(success=True, message="Configuration reloaded") except Exception as exc: return pb2.ReloadResponse(success=False, message=str(exc)) + return pb2.ReloadResponse(success=True, message="Configuration reloaded") + + async def ReplayDead(self, request, context): + replayed = await self._runtime.replay_dead( + record_id=request.record_id or None, + sink=request.sink or None, + limit=request.limit or 100, + ) + return pb2.ReplayResponse(replayed=replayed) class DaenaGRPCServer: """Manages the gRPC server lifecycle.""" - def __init__(self, runtime: DaenaRuntime, host: str = "127.0.0.1", port: int = 8642) -> None: + def __init__( + self, + runtime: DaenaRuntime, + host: str = "127.0.0.1", + port: int = 8642, + maximum_concurrent_rpcs: int | None = None, + ) -> None: self._runtime = runtime self._host = host self._port = port + self._maximum_concurrent_rpcs = maximum_concurrent_rpcs self._server: grpc.aio.Server | None = None async def start(self) -> None: - self._server = grpc.aio.server() + self._server = grpc.aio.server(maximum_concurrent_rpcs=self._maximum_concurrent_rpcs) servicer = DaenaServicer(self._runtime) pb2_grpc.add_DaenaServicer_to_server(servicer, self._server) address = f"{self._host}:{self._port}" diff --git a/src/daena/api/metrics.py b/src/daena/api/metrics.py new file mode 100644 index 0000000..9cad729 --- /dev/null +++ b/src/daena/api/metrics.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MetricsSnapshot: + running: bool + stored_records: int + oldest_inflight_seconds: int + deliveries: dict[tuple[str, str], int] + sink_results: dict[str, dict[str, int]] + + +def render_prometheus(snapshot: MetricsSnapshot) -> str: + lines = [ + "# HELP daena_up 1 when the runtime is running", + "# TYPE daena_up gauge", + f"daena_up {1 if snapshot.running else 0}", + "# HELP daena_records Stored records", + "# TYPE daena_records gauge", + f"daena_records {snapshot.stored_records}", + "# HELP daena_oldest_inflight_seconds Age of the oldest undelivered record", + "# TYPE daena_oldest_inflight_seconds gauge", + f"daena_oldest_inflight_seconds {snapshot.oldest_inflight_seconds}", + "# HELP daena_delivery_records Deliveries by sink and state", + "# TYPE daena_delivery_records gauge", + ] + for (sink, state), value in sorted(snapshot.deliveries.items()): + lines.append( + f'daena_delivery_records{{sink="{_escape(sink)}",state="{_escape(state)}"}} {value}' + ) + lines.extend( + [ + "# HELP daena_sink_deliveries_total Delivery attempts since process start", + "# TYPE daena_sink_deliveries_total counter", + ] + ) + for sink, results in sorted(snapshot.sink_results.items()): + for result, value in sorted(results.items()): + lines.append( + "daena_sink_deliveries_total" + f'{{sink="{_escape(sink)}",result="{_escape(result)}"}} {value}' + ) + return "\n".join(lines) + "\n" + + +def _escape(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +class MetricsServer: + """Tiny HTTP server for Prometheus scrapes and a JSON health body.""" + + def __init__( + self, + host: str, + port: int, + snapshot: Callable[[], MetricsSnapshot], + health_body: Callable[[], dict[str, object]], + ) -> None: + self._host = host + self._port = port + self._snapshot = snapshot + self._health_body = health_body + self._server: asyncio.AbstractServer | None = None + + async def start(self) -> None: + self._server = await asyncio.start_server(self._handle, self._host, self._port) + + async def stop(self) -> None: + if self._server is None: + return + self._server.close() + await self._server.wait_closed() + self._server = None + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + request = await reader.readuntil(b"\r\n\r\n") + except (asyncio.IncompleteReadError, asyncio.LimitOverrunError): + writer.close() + return + line = request.split(b"\r\n", 1)[0].decode(errors="replace") + parts = line.split(" ") + path = parts[1] if len(parts) >= 2 else "/" + if path == "/metrics": + body = render_prometheus(await asyncio.to_thread(self._snapshot)).encode() + content_type = "text/plain; version=0.0.4" + status = "200 OK" + elif path == "/health": + body = json.dumps(await asyncio.to_thread(self._health_body)).encode() + content_type = "application/json" + status = "200 OK" + else: + body = b"not found" + content_type = "text/plain" + status = "404 Not Found" + writer.write( + ( + f"HTTP/1.1 {status}\r\n" + f"Content-Type: {content_type}\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode() + + body + ) + await writer.drain() + writer.close() diff --git a/src/daena/api/pb2/daena_pb2.py b/src/daena/api/pb2/daena_pb2.py index 19ff006..8c9b43b 100644 --- a/src/daena/api/pb2/daena_pb2.py +++ b/src/daena/api/pb2/daena_pb2.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # ruff: noqa: I001, E501, UP009 +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: daena/v1/daena.proto -# Protobuf Python Version: 7.35.0 +# Protobuf Python Version: 7.35.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor @@ -13,7 +13,7 @@ from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, 7, 35, 0, "", "daena/v1/daena.proto" + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "daena/v1/daena.proto" ) # @@protoc_insertion_point(imports) @@ -21,7 +21,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x14\x64\x61\x65na/v1/daena.proto\x12\x08\x64\x61\x65na.v1"\x07\n\x05\x45mpty"C\n\x0eStatusResponse\x12\x0f\n\x07service\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0f\n\x07running\x18\x03 \x01(\x08"[\n\x0eHealthResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07running\x18\x02 \x01(\x08\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\x08\x12\x16\n\x0eplugins_loaded\x18\x04 \x01(\x08"%\n\x0e\x43onfigResponse\x12\x13\n\x0b\x63onfig_json\x18\x01 \x01(\t".\n\nPluginInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nclass_name\x18\x02 \x01(\t"5\n\x0ePluginCategory\x12#\n\x05items\x18\x01 \x03(\x0b\x32\x14.daena.v1.PluginInfo"\x96\x01\n\x12PluginListResponse\x12)\n\x07sources\x18\x01 \x01(\x0b\x32\x18.daena.v1.PluginCategory\x12,\n\nprocessors\x18\x02 \x01(\x0b\x32\x18.daena.v1.PluginCategory\x12\'\n\x05sinks\x18\x03 \x01(\x0b\x32\x18.daena.v1.PluginCategory"e\n\x12QueueStateResponse\x12\x0f\n\x07pending\x18\x01 \x01(\x03\x12\x11\n\tdelivered\x18\x02 \x01(\x03\x12\x0e\n\x06\x66\x61iled\x18\x03 \x01(\x03\x12\x0c\n\x04\x64\x65\x61\x64\x18\x04 \x01(\x03\x12\r\n\x05total\x18\x05 \x01(\x03"+\n\nPagination\x12\r\n\x05limit\x18\x01 \x01(\x05\x12\x0e\n\x06offset\x18\x02 \x01(\x05"\xf0\x01\n\nRecordData\x12\x11\n\trecord_id\x18\x01 \x01(\t\x12\x0e\n\x06source\x18\x02 \x01(\t\x12\x13\n\x0brecord_type\x18\x03 \x01(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x10\n\x08metadata\x18\x05 \x01(\t\x12\r\n\x05state\x18\x06 \x01(\t\x12\x12\n\ncreated_at\x18\x07 \x01(\t\x12\x12\n\nupdated_at\x18\x08 \x01(\t\x12\x13\n\x0bretry_count\x18\t \x01(\x05\x12\x15\n\rnext_retry_at\x18\n \x01(\t\x12\x12\n\nlast_error\x18\x0b \x01(\t\x12\x13\n\x0b\x64\x65stination\x18\x0c \x01(\t"B\n\nRecordList\x12%\n\x07records\x18\x01 \x03(\x0b\x32\x14.daena.v1.RecordData\x12\r\n\x05\x63ount\x18\x02 \x01(\x05"&\n\x08SinkInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t"-\n\x08SinkList\x12!\n\x05sinks\x18\x01 \x03(\x0b\x32\x12.daena.v1.SinkInfo"2\n\x0eReloadResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\x9b\x04\n\x05\x44\x61\x65na\x12\x36\n\tGetStatus\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.StatusResponse\x12\x36\n\tGetHealth\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.HealthResponse\x12\x36\n\tGetConfig\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.ConfigResponse\x12<\n\x0bListPlugins\x12\x0f.daena.v1.Empty\x1a\x1c.daena.v1.PluginListResponse\x12>\n\rGetQueueState\x12\x0f.daena.v1.Empty\x1a\x1c.daena.v1.QueueStateResponse\x12@\n\x12ListPendingRecords\x12\x14.daena.v1.Pagination\x1a\x14.daena.v1.RecordList\x12=\n\x0fListDeadRecords\x12\x14.daena.v1.Pagination\x1a\x14.daena.v1.RecordList\x12\x30\n\tListSinks\x12\x0f.daena.v1.Empty\x1a\x12.daena.v1.SinkList\x12\x39\n\x0cReloadConfig\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.ReloadResponseb\x06proto3' + b'\n\x14\x64\x61\x65na/v1/daena.proto\x12\x08\x64\x61\x65na.v1"\x07\n\x05\x45mpty"C\n\x0eStatusResponse\x12\x0f\n\x07service\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0f\n\x07running\x18\x03 \x01(\x08"\x9f\x01\n\x0eHealthResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07running\x18\x02 \x01(\x08\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\x08\x12\x16\n\x0eplugins_loaded\x18\x04 \x01(\x08\x12\x0e\n\x06status\x18\x05 \x01(\t\x12"\n\x1aoldest_pending_age_seconds\x18\x06 \x01(\x03\x12\x0e\n\x06\x64\x65tail\x18\x07 \x01(\t"%\n\x0e\x43onfigResponse\x12\x13\n\x0b\x63onfig_json\x18\x01 \x01(\t".\n\nPluginInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nclass_name\x18\x02 \x01(\t"5\n\x0ePluginCategory\x12#\n\x05items\x18\x01 \x03(\x0b\x32\x14.daena.v1.PluginInfo"\x96\x01\n\x12PluginListResponse\x12)\n\x07sources\x18\x01 \x01(\x0b\x32\x18.daena.v1.PluginCategory\x12,\n\nprocessors\x18\x02 \x01(\x0b\x32\x18.daena.v1.PluginCategory\x12\'\n\x05sinks\x18\x03 \x01(\x0b\x32\x18.daena.v1.PluginCategory"t\n\x0eSinkQueueState\x12\x0c\n\x04sink\x18\x01 \x01(\t\x12\x0f\n\x07pending\x18\x02 \x01(\x03\x12\x12\n\ndelivering\x18\x03 \x01(\x03\x12\x11\n\tdelivered\x18\x04 \x01(\x03\x12\x0e\n\x06\x66\x61iled\x18\x05 \x01(\x03\x12\x0c\n\x04\x64\x65\x61\x64\x18\x06 \x01(\x03"\xa2\x01\n\x12QueueStateResponse\x12\x0f\n\x07pending\x18\x01 \x01(\x03\x12\x11\n\tdelivered\x18\x02 \x01(\x03\x12\x0e\n\x06\x66\x61iled\x18\x03 \x01(\x03\x12\x0c\n\x04\x64\x65\x61\x64\x18\x04 \x01(\x03\x12\r\n\x05total\x18\x05 \x01(\x03\x12\'\n\x05sinks\x18\x06 \x03(\x0b\x32\x18.daena.v1.SinkQueueState\x12\x12\n\ndelivering\x18\x07 \x01(\x03"+\n\nPagination\x12\r\n\x05limit\x18\x01 \x01(\x05\x12\x0e\n\x06offset\x18\x02 \x01(\x05"\x7f\n\x0c\x44\x65liveryData\x12\x0c\n\x04sink\x18\x01 \x01(\t\x12\r\n\x05state\x18\x02 \x01(\t\x12\x13\n\x0bretry_count\x18\x03 \x01(\x05\x12\x15\n\rnext_retry_at\x18\x04 \x01(\t\x12\x12\n\nlast_error\x18\x05 \x01(\t\x12\x12\n\nupdated_at\x18\x06 \x01(\t"\x9c\x02\n\nRecordData\x12\x11\n\trecord_id\x18\x01 \x01(\t\x12\x0e\n\x06source\x18\x02 \x01(\t\x12\x13\n\x0brecord_type\x18\x03 \x01(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x10\n\x08metadata\x18\x05 \x01(\t\x12\r\n\x05state\x18\x06 \x01(\t\x12\x12\n\ncreated_at\x18\x07 \x01(\t\x12\x12\n\nupdated_at\x18\x08 \x01(\t\x12\x13\n\x0bretry_count\x18\t \x01(\x05\x12\x15\n\rnext_retry_at\x18\n \x01(\t\x12\x12\n\nlast_error\x18\x0b \x01(\t\x12\x13\n\x0b\x64\x65stination\x18\x0c \x01(\t\x12*\n\ndeliveries\x18\r \x03(\x0b\x32\x16.daena.v1.DeliveryData"B\n\nRecordList\x12%\n\x07records\x18\x01 \x03(\x0b\x32\x14.daena.v1.RecordData\x12\r\n\x05\x63ount\x18\x02 \x01(\x05"&\n\x08SinkInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t"-\n\x08SinkList\x12!\n\x05sinks\x18\x01 \x03(\x0b\x32\x12.daena.v1.SinkInfo"2\n\x0eReloadResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t"?\n\rReplayRequest\x12\x11\n\trecord_id\x18\x01 \x01(\t\x12\x0c\n\x04sink\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05""\n\x0eReplayResponse\x12\x10\n\x08replayed\x18\x01 \x01(\x05\x32\xdc\x04\n\x05\x44\x61\x65na\x12\x36\n\tGetStatus\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.StatusResponse\x12\x36\n\tGetHealth\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.HealthResponse\x12\x36\n\tGetConfig\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.ConfigResponse\x12<\n\x0bListPlugins\x12\x0f.daena.v1.Empty\x1a\x1c.daena.v1.PluginListResponse\x12>\n\rGetQueueState\x12\x0f.daena.v1.Empty\x1a\x1c.daena.v1.QueueStateResponse\x12@\n\x12ListPendingRecords\x12\x14.daena.v1.Pagination\x1a\x14.daena.v1.RecordList\x12=\n\x0fListDeadRecords\x12\x14.daena.v1.Pagination\x1a\x14.daena.v1.RecordList\x12\x30\n\tListSinks\x12\x0f.daena.v1.Empty\x1a\x12.daena.v1.SinkList\x12\x39\n\x0cReloadConfig\x12\x0f.daena.v1.Empty\x1a\x18.daena.v1.ReloadResponse\x12?\n\nReplayDead\x12\x17.daena.v1.ReplayRequest\x1a\x18.daena.v1.ReplayResponseb\x06proto3' ) _globals = globals() @@ -33,30 +33,38 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals["_EMPTY"]._serialized_end = 41 _globals["_STATUSRESPONSE"]._serialized_start = 43 _globals["_STATUSRESPONSE"]._serialized_end = 110 - _globals["_HEALTHRESPONSE"]._serialized_start = 112 - _globals["_HEALTHRESPONSE"]._serialized_end = 203 - _globals["_CONFIGRESPONSE"]._serialized_start = 205 - _globals["_CONFIGRESPONSE"]._serialized_end = 242 - _globals["_PLUGININFO"]._serialized_start = 244 - _globals["_PLUGININFO"]._serialized_end = 290 - _globals["_PLUGINCATEGORY"]._serialized_start = 292 - _globals["_PLUGINCATEGORY"]._serialized_end = 345 - _globals["_PLUGINLISTRESPONSE"]._serialized_start = 348 - _globals["_PLUGINLISTRESPONSE"]._serialized_end = 498 - _globals["_QUEUESTATERESPONSE"]._serialized_start = 500 - _globals["_QUEUESTATERESPONSE"]._serialized_end = 601 - _globals["_PAGINATION"]._serialized_start = 603 - _globals["_PAGINATION"]._serialized_end = 646 - _globals["_RECORDDATA"]._serialized_start = 649 - _globals["_RECORDDATA"]._serialized_end = 889 - _globals["_RECORDLIST"]._serialized_start = 891 - _globals["_RECORDLIST"]._serialized_end = 957 - _globals["_SINKINFO"]._serialized_start = 959 - _globals["_SINKINFO"]._serialized_end = 997 - _globals["_SINKLIST"]._serialized_start = 999 - _globals["_SINKLIST"]._serialized_end = 1044 - _globals["_RELOADRESPONSE"]._serialized_start = 1046 - _globals["_RELOADRESPONSE"]._serialized_end = 1096 - _globals["_DAENA"]._serialized_start = 1099 - _globals["_DAENA"]._serialized_end = 1638 + _globals["_HEALTHRESPONSE"]._serialized_start = 113 + _globals["_HEALTHRESPONSE"]._serialized_end = 272 + _globals["_CONFIGRESPONSE"]._serialized_start = 274 + _globals["_CONFIGRESPONSE"]._serialized_end = 311 + _globals["_PLUGININFO"]._serialized_start = 313 + _globals["_PLUGININFO"]._serialized_end = 359 + _globals["_PLUGINCATEGORY"]._serialized_start = 361 + _globals["_PLUGINCATEGORY"]._serialized_end = 414 + _globals["_PLUGINLISTRESPONSE"]._serialized_start = 417 + _globals["_PLUGINLISTRESPONSE"]._serialized_end = 567 + _globals["_SINKQUEUESTATE"]._serialized_start = 569 + _globals["_SINKQUEUESTATE"]._serialized_end = 685 + _globals["_QUEUESTATERESPONSE"]._serialized_start = 688 + _globals["_QUEUESTATERESPONSE"]._serialized_end = 850 + _globals["_PAGINATION"]._serialized_start = 852 + _globals["_PAGINATION"]._serialized_end = 895 + _globals["_DELIVERYDATA"]._serialized_start = 897 + _globals["_DELIVERYDATA"]._serialized_end = 1024 + _globals["_RECORDDATA"]._serialized_start = 1027 + _globals["_RECORDDATA"]._serialized_end = 1311 + _globals["_RECORDLIST"]._serialized_start = 1313 + _globals["_RECORDLIST"]._serialized_end = 1379 + _globals["_SINKINFO"]._serialized_start = 1381 + _globals["_SINKINFO"]._serialized_end = 1419 + _globals["_SINKLIST"]._serialized_start = 1421 + _globals["_SINKLIST"]._serialized_end = 1466 + _globals["_RELOADRESPONSE"]._serialized_start = 1468 + _globals["_RELOADRESPONSE"]._serialized_end = 1518 + _globals["_REPLAYREQUEST"]._serialized_start = 1520 + _globals["_REPLAYREQUEST"]._serialized_end = 1583 + _globals["_REPLAYRESPONSE"]._serialized_start = 1585 + _globals["_REPLAYRESPONSE"]._serialized_end = 1619 + _globals["_DAENA"]._serialized_start = 1622 + _globals["_DAENA"]._serialized_end = 2226 # @@protoc_insertion_point(module_scope) diff --git a/src/daena/api/pb2/daena_pb2_grpc.py b/src/daena/api/pb2/daena_pb2_grpc.py index 5238cc1..710808f 100644 --- a/src/daena/api/pb2/daena_pb2_grpc.py +++ b/src/daena/api/pb2/daena_pb2_grpc.py @@ -91,6 +91,12 @@ class DaenaStub: response_deserializer=daena_dot_v1_dot_daena__pb2.ReloadResponse.FromString, _registered_method=True, ) + self.ReplayDead = channel.unary_unary( + "/daena.v1.Daena/ReplayDead", + request_serializer=daena_dot_v1_dot_daena__pb2.ReplayRequest.SerializeToString, + response_deserializer=daena_dot_v1_dot_daena__pb2.ReplayResponse.FromString, + _registered_method=True, + ) class DaenaServicer: @@ -150,6 +156,12 @@ class DaenaServicer: context.set_details("Method not implemented!") raise NotImplementedError("Method not implemented!") + def ReplayDead(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + def add_DaenaServicer_to_server(servicer, server): rpc_method_handlers = { @@ -198,6 +210,11 @@ def add_DaenaServicer_to_server(servicer, server): request_deserializer=daena_dot_v1_dot_daena__pb2.Empty.FromString, response_serializer=daena_dot_v1_dot_daena__pb2.ReloadResponse.SerializeToString, ), + "ReplayDead": grpc.unary_unary_rpc_method_handler( + servicer.ReplayDead, + request_deserializer=daena_dot_v1_dot_daena__pb2.ReplayRequest.FromString, + response_serializer=daena_dot_v1_dot_daena__pb2.ReplayResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler("daena.v1.Daena", rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) @@ -477,3 +494,33 @@ class Daena: metadata, _registered_method=True, ) + + @staticmethod + def ReplayDead( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/daena.v1.Daena/ReplayDead", + daena_dot_v1_dot_daena__pb2.ReplayRequest.SerializeToString, + daena_dot_v1_dot_daena__pb2.ReplayResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) diff --git a/src/daena/app.py b/src/daena/app.py index 249cd20..ab00e44 100644 --- a/src/daena/app.py +++ b/src/daena/app.py @@ -1,60 +1,170 @@ from __future__ import annotations +import argparse import asyncio +import os +import signal +import sys from pathlib import Path from daena.api.grpc_server import DaenaGRPCServer -from daena.config import load_config +from daena.api.metrics import MetricsServer, MetricsSnapshot +from daena.config import DaenaConfig, load_config +from daena.core.layout import render_layout from daena.core.runtime import DaenaRuntime +from daena.exceptions import ConfigurationError from daena.logutil import get_logger, setup_logging log = get_logger("daena.app") def _resolve_config_path() -> str | None: - for candidate in ( - "/etc/daena/daena.yaml", - "./daena.yaml", - "./config/daena.yaml", - ): - p = Path(candidate) - if p.exists(): - return str(p) + env_path = os.environ.get("DAENA_CONFIG_PATH") + if env_path: + if not Path(env_path).exists(): + raise ConfigurationError(f"DAENA_CONFIG_PATH not found: {env_path}") + return env_path + for candidate in ("/etc/daena/daena.yaml", "./daena.yaml", "./config/daena.yaml"): + if Path(candidate).exists(): + return candidate return None +def _metrics_snapshot(runtime: DaenaRuntime) -> MetricsSnapshot: + backend = runtime.backend + deliveries: dict[tuple[str, str], int] = {} + stored = 0 + oldest = runtime.health().oldest_pending_age_seconds if backend is not None else 0 + if backend is not None: + stored = backend.count() + for sink, counts in backend.sink_counts().items(): + for state, value in counts.items(): + deliveries[(sink, state)] = value + results = { + name: {"delivered": item.delivered, "failed": item.failed, "dead": item.dead} + for name, item in runtime.sink_stats().items() + } + return MetricsSnapshot( + running=runtime.health().running, + stored_records=stored, + oldest_inflight_seconds=oldest, + deliveries=deliveries, + sink_results=results, + ) + + +def _health_body(runtime: DaenaRuntime) -> dict[str, object]: + report = runtime.health() + return { + "healthy": report.healthy, + "status": report.status, + "running": report.running, + "backend": report.backend, + "oldest_pending_age_seconds": report.oldest_pending_age_seconds, + "detail": report.detail, + } + + +async def run_until_stopped(runtime: DaenaRuntime, stop: asyncio.Event) -> None: + """Start the API and block until `stop` is set, then shut down.""" + config = runtime.config + server = DaenaGRPCServer( + runtime, + host=config.api.grpc.host, + port=config.api.grpc.port, + maximum_concurrent_rpcs=config.api.grpc.max_workers, + ) + metrics: MetricsServer | None = None + await runtime.start() + await server.start() + if config.api.metrics.enabled: + metrics = MetricsServer( + config.api.metrics.host, + config.api.metrics.port, + lambda: _metrics_snapshot(runtime), + lambda: _health_body(runtime), + ) + await metrics.start() + log.info( + "metrics_server_serving", + address=f"{config.api.metrics.host}:{config.api.metrics.port}", + ) + log.info( + "grpc_server_serving", + address=f"{config.api.grpc.host}:{config.api.grpc.port}", + ) + reload_tasks: set[asyncio.Task[None]] = set() + reload_lock = asyncio.Lock() + + async def _reload() -> None: + if stop.is_set(): + return + async with reload_lock: + try: + await runtime.reload() + except Exception: + log.exception("sighup_reload_failed") + return + _print_layout(runtime.config, runtime.config_path) + + def _on_hup() -> None: + task = asyncio.create_task(_reload()) + reload_tasks.add(task) + task.add_done_callback(reload_tasks.discard) + + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGINT, stop.set) + loop.add_signal_handler(signal.SIGTERM, stop.set) + loop.add_signal_handler(signal.SIGHUP, _on_hup) + try: + await stop.wait() + finally: + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + loop.remove_signal_handler(sig) + for task in list(reload_tasks): + task.cancel() + if metrics is not None: + await metrics.stop() + await server.stop() + await runtime.stop() + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="daena-serve", + description="Run daena: collect, buffer, and forward logs.", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="log each collected, dropped, and delivered record", + ) + return parser.parse_args(argv) + + +def _print_layout(config: DaenaConfig, config_path: str | Path | None) -> None: + path = os.fspath(config_path) if config_path is not None else None + print(render_layout(config, path), file=sys.stderr) + + def serve() -> None: """Entry point for `daena-serve` (systemd/gRPC mode).""" + args = _parse_args() config_path = _resolve_config_path() config = load_config(config_path) + if args.verbose: + config.logging.verbose = True setup_logging( level=config.logging.level, fmt=config.logging.format, log_file=config.logging.file, ) - - runtime = DaenaRuntime(config) - - server = DaenaGRPCServer( - runtime, - host=config.api.grpc.host, - port=config.api.grpc.port, - ) + _print_layout(config, config_path) + runtime = DaenaRuntime(config, config_path=config_path, force_verbose=args.verbose) async def run() -> None: - await runtime.start() - await server.start() - log.info( - "grpc_server_serving", - address=f"{config.api.grpc.host}:{config.api.grpc.port}", - ) - try: - await server.serve_forever() - except asyncio.CancelledError: - pass - finally: - await server.stop() - await runtime.stop() + await run_until_stopped(runtime, asyncio.Event()) asyncio.run(run()) diff --git a/src/daena/cli/main.py b/src/daena/cli/main.py index e73c3c2..44cc564 100644 --- a/src/daena/cli/main.py +++ b/src/daena/cli/main.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from typing import Annotated import grpc import typer @@ -18,11 +19,22 @@ app = typer.Typer( console = Console() DEFAULT_TARGET = "127.0.0.1:8642" +_state: dict[str, str] = {"target": DEFAULT_TARGET} + + +@app.callback() +def main( + target: Annotated[ + str, + typer.Option("--target", "-t", help="gRPC address of a running daena service"), + ] = DEFAULT_TARGET, +) -> None: + """Operational client for a running daena service.""" + _state["target"] = target def _stub(target: str | None = None) -> pb2_grpc.DaenaStub: - tgt = target or DEFAULT_TARGET - channel = grpc.insecure_channel(tgt) + channel = grpc.insecure_channel(target or _state["target"]) return pb2_grpc.DaenaStub(channel) @@ -53,10 +65,12 @@ def health() -> None: """Check service health.""" stub = _connect() resp = stub.GetHealth(pb2.Empty(), timeout=5.0) - if resp.healthy: - console.print("[green]Healthy[/green]") - else: - console.print("[red]Unhealthy[/red]") + label = "green" if resp.healthy else "red" + console.print(f"[{label}]{resp.status}[/{label}]") + console.print(f"Oldest inflight: {resp.oldest_pending_age_seconds}s") + if resp.detail: + console.print(resp.detail) + if not resp.healthy: raise typer.Exit(1) @@ -66,9 +80,68 @@ def queue() -> None: stub = _connect() resp = stub.GetQueueState(pb2.Empty(), timeout=5.0) table = Table("State", "Count") - for key in ("pending", "delivered", "failed", "dead", "total"): + for key in ("pending", "delivering", "delivered", "failed", "dead", "total"): table.add_row(key, str(getattr(resp, key, 0))) console.print(table) + if resp.sinks: + sinks = Table("Sink", "Pending", "Delivering", "Delivered", "Failed", "Dead") + for item in resp.sinks: + sinks.add_row( + item.sink, + str(item.pending), + str(item.delivering), + str(item.delivered), + str(item.failed), + str(item.dead), + ) + console.print(sinks) + + +@app.command() +def records( + kind: Annotated[str, typer.Argument(help="pending or dead")] = "pending", + limit: Annotated[int, typer.Option(help="Maximum records to show")] = 20, + offset: Annotated[int, typer.Option(help="Records to skip")] = 0, +) -> None: + """List pending or dead-letter records.""" + if kind not in ("pending", "dead"): + console.print("[red]kind must be pending or dead[/red]") + raise typer.Exit(2) + stub = _connect() + page = pb2.Pagination(limit=limit, offset=offset) + if kind == "pending": + resp = stub.ListPendingRecords(page, timeout=5.0) + else: + resp = stub.ListDeadRecords(page, timeout=5.0) + if not resp.records: + console.print(f"No {kind} records") + return + table = Table("ID", "Source", "State", "Deliveries", "Error") + for record in resp.records: + deliveries = ",".join(f"{item.sink}:{item.state}" for item in record.deliveries) + table.add_row( + record.record_id, + record.source, + record.state, + deliveries, + record.last_error, + ) + console.print(table) + + +@app.command() +def replay( + record_id: Annotated[str, typer.Option(help="One record id; empty replays a batch")] = "", + sink: Annotated[str, typer.Option(help="Limit replay to one sink")] = "", + limit: Annotated[int, typer.Option(help="Maximum dead records to requeue")] = 100, +) -> None: + """Requeue dead-letter deliveries as pending.""" + stub = _connect() + resp = stub.ReplayDead( + pb2.ReplayRequest(record_id=record_id, sink=sink, limit=limit), + timeout=10.0, + ) + console.print(f"Replayed {resp.replayed} record(s)") @app.command() @@ -81,8 +154,8 @@ def sinks() -> None: console.print("No sinks configured") return table = Table("Name", "Type") - for s in items: - table.add_row(s.name, s.type) + for item in items: + table.add_row(item.name, item.type) console.print(table) @@ -91,14 +164,14 @@ def plugins() -> None: """List loaded plugins.""" stub = _connect() resp = stub.ListPlugins(pb2.Empty(), timeout=5.0) - for label, cat in ( + for label, category in ( ("Sources", resp.sources), ("Processors", resp.processors), ("Sinks", resp.sinks), ): - if cat.items: + if category.items: table = Table(label) - for item in cat.items: + for item in category.items: table.add_row(f"{item.name} ({item.class_name})") console.print(table) console.print() diff --git a/src/daena/config.py b/src/daena/config.py index e4d4f8c..395f077 100644 --- a/src/daena/config.py +++ b/src/daena/config.py @@ -12,6 +12,7 @@ class LoggingConfig(BaseSettings): level: str = "info" format: str = "json" file: str | None = None + verbose: bool = False class GRPCConfig(BaseSettings): @@ -20,8 +21,25 @@ class GRPCConfig(BaseSettings): max_workers: int = 10 +class MetricsConfig(BaseSettings): + enabled: bool = True + host: str = "127.0.0.1" + port: int = 8643 + + class APIConfig(BaseSettings): grpc: GRPCConfig = GRPCConfig() + metrics: MetricsConfig = MetricsConfig() + + +class RetentionConfig(BaseSettings): + delivered_ttl_seconds: int = 86_400 + dead_ttl_seconds: int = 604_800 + interval_seconds: float = 60.0 + + +class HealthConfig(BaseSettings): + max_pending_age_seconds: float = 300.0 class SinkRetryConfig(BaseSettings): @@ -82,6 +100,8 @@ class DaenaConfig(BaseSettings): api: APIConfig = APIConfig() pipeline: PipelineConfig = PipelineConfig() persistence: PersistenceConfig = PersistenceConfig() + retention: RetentionConfig = RetentionConfig() + health: HealthConfig = HealthConfig() sources: list[SourceConfig] = Field(default_factory=list) processors: list[ProcessorConfig] = Field(default_factory=list) diff --git a/src/daena/core/layout.py b/src/daena/core/layout.py new file mode 100644 index 0000000..5fd9187 --- /dev/null +++ b/src/daena/core/layout.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +from typing import Any +from urllib.parse import urlparse + +from daena._version import __version__ +from daena.config import DaenaConfig, SinkConfig, SourceConfig + + +def preview_message(body: dict[str, Any], limit: int = 120) -> str: + """One line of record text for verbose logs.""" + raw = body.get("message") + if not isinstance(raw, str) or not raw.strip(): + raw = json.dumps(body, default=str, separators=(",", ":")) + text = " ".join(raw.split()) + if len(text) <= limit: + return text + return text[: limit - 1] + "…" + + +def render_layout(config: DaenaConfig, config_path: str | None = None) -> str: + """One-line pipeline and the addresses the process listens on.""" + stages = [ + ", ".join(_source_label(item) for item in config.sources), + ", ".join(_node(item.name, item.type, item.enabled) for item in config.processors), + "lmdb", + ", ".join(_sink_label(item) for item in config.sinks), + ] + flow = " → ".join(stage for stage in stages if stage) + where = config_path or "defaults" + lines = [ + f"daena {__version__} {where}", + flow, + config.persistence.lmdb.path, + _listen(config), + ] + return "\n".join(line for line in lines if line) + + +def _listen(config: DaenaConfig) -> str: + grpc = f"{config.api.grpc.host}:{config.api.grpc.port}" + if not config.api.metrics.enabled: + return grpc + metrics = f"{config.api.metrics.host}:{config.api.metrics.port}" + return f"{grpc} {metrics}" + + +def _source_label(source: SourceConfig) -> str: + return _node(source.name, source.type, source.enabled, _source_hint(source)) + + +def _sink_label(sink: SinkConfig) -> str: + return _node(sink.name, sink.type, sink.enabled, _sink_hint(sink)) + + +def _node(name: str, type_name: str, enabled: bool, hint: str = "") -> str: + notes: list[str] = [] + if name != type_name: + notes.append(type_name) + if enabled and hint: + notes.append(hint) + if not enabled: + notes.append("off") + if not notes: + return name + return f"{name} ({', '.join(notes)})" + + +def _source_hint(source: SourceConfig) -> str: + options = source.options + if source.type == "dummy" and "interval" in options: + return _seconds(options["interval"]) + if source.type == "journald": + units = options.get("units") or [] + if isinstance(units, list) and units: + return str(units[0]) if len(units) == 1 else f"{units[0]} +{len(units) - 1}" + return "" + + +def _sink_hint(sink: SinkConfig) -> str: + url = sink.options.get("url") + if sink.type in {"http", "loki"} and isinstance(url, str): + return _endpoint(url) + return "" + + +def _seconds(value: Any) -> str: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return "" + try: + seconds = float(value) + except ValueError: + return "" + if seconds.is_integer(): + return f"{int(seconds)}s" + return f"{seconds}s" + + +def _endpoint(url: str) -> str: + parsed = urlparse(url) + if not parsed.hostname: + return url + if parsed.port: + return f"{parsed.hostname}:{parsed.port}" + return parsed.hostname diff --git a/src/daena/core/pipeline.py b/src/daena/core/pipeline.py index f3a0e46..c73777f 100644 --- a/src/daena/core/pipeline.py +++ b/src/daena/core/pipeline.py @@ -3,10 +3,14 @@ from __future__ import annotations import asyncio import contextlib from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta from typing import Any -from daena.config import DaenaConfig +from daena.config import DaenaConfig, SinkRetryConfig +from daena.core.layout import preview_message from daena.logutil import get_logger +from daena.persistence.async_backend import AsyncBackend from daena.persistence.backend import Backend from daena.persistence.models import StoredRecord from daena.pipeline.base import DeliveryResult, Processor, Sink, Source @@ -16,42 +20,49 @@ from daena.plugins.registry import PluginRegistry log = get_logger("daena.pipeline") -class SourceRunner: - """Wraps a Source and handles record ingestion into the pipeline.""" +@dataclass +class SinkStats: + """Cumulative delivery outcomes for one sink since process start.""" - def __init__( - self, - source: Source, - on_record: Callable[[Record], Any], - ) -> None: - self._source = source - self._on_record = on_record + delivered: int = 0 + failed: int = 0 + dead: int = 0 + last_error: str = "" + + +@dataclass +class SourceRunner: + """Binds a source to the pipeline ingest callback.""" + + source: Source + on_record: Callable[[Record], Any] async def start(self) -> None: - if hasattr(self._source, "set_callback"): - self._source.set_callback(self._on_record) - log.info("source_starting", source=self._source.name) - await self._source.start() + self.source.set_callback(self.on_record) + log.info("source_starting", source=self.source.name) + await self.source.start() async def stop(self) -> None: - log.info("source_stopping", source=self._source.name) - await self._source.stop() + log.info("source_stopping", source=self.source.name) + await self.source.stop() class SinkDispatcher: - """Manages delivery of records to a sink with retry logic.""" + """Owns one sink instance and forwards a single record to it.""" - def __init__( - self, - sink: Sink, - backend: Backend, - config: DaenaConfig, - ) -> None: + def __init__(self, sink: Sink, plugin_type: str) -> None: self._sink = sink - self._backend = backend - self._config = config + self.plugin_type = plugin_type self._logger = log.bind(sink=sink.name) + @property + def name(self) -> str: + return self._sink.name + + @property + def sink(self) -> Sink: + return self._sink + async def start(self) -> None: self._logger.info("sink_starting") await self._sink.start() @@ -65,14 +76,11 @@ class SinkDispatcher: class Pipeline: - """Orchestrates the full collection → persist → forward data flow. + """Collection, durable buffer, and per-sink delivery. - Lifecycle: - 1. Sources emit Records - 2. Processors transform/filter Records - 3. Records are persisted to the backend - 4. SinkDispatcher reads pending records and delivers them - 5. Delivery results update record state (ack/nack/dead) + Sources block when the number of inflight records reaches `max_queue_size`. + Each delivery attempt is a single try; retries wait on `next_retry_at` + instead of sleeping inside the dispatcher. """ def __init__( @@ -83,77 +91,148 @@ class Pipeline: ) -> None: self._config = config self._backend = backend + self._store = AsyncBackend(backend) self._registry = registry self._sources: list[Source] = [] self._processors: list[Processor] = [] self._dispatchers: list[SinkDispatcher] = [] self._source_runners: list[SourceRunner] = [] self._dispatch_task: asyncio.Task[None] | None = None + self._retention_task: asyncio.Task[None] | None = None self._running = False - self._queue: asyncio.Queue[Record] = asyncio.Queue(maxsize=config.pipeline.max_queue_size) + self._inflight = 0 + self._capacity = asyncio.Condition() + self._stats: dict[str, SinkStats] = {} + self._sink_names: list[str] = [] async def start(self) -> None: self._running = True self._build_pipeline() + self._inflight = await self._store.count_inflight() for runner in self._source_runners: await runner.start() for dispatcher in self._dispatchers: await dispatcher.start() self._dispatch_task = asyncio.create_task(self._dispatch_loop()) + self._retention_task = asyncio.create_task(self._retention_loop()) log.info( "pipeline_started", sources=len(self._source_runners), sinks=len(self._dispatchers), processors=len(self._processors), + inflight=self._inflight, ) async def stop(self) -> None: self._running = False - if self._dispatch_task: - self._dispatch_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._dispatch_task + async with self._capacity: + self._capacity.notify_all() + await self._cancel(self._dispatch_task) + await self._cancel(self._retention_task) + self._dispatch_task = None + self._retention_task = None for dispatcher in self._dispatchers: await dispatcher.stop() for runner in reversed(self._source_runners): await runner.stop() + recovered = await self._store.recover_inflight() + if recovered: + log.info("delivering_requeued", count=recovered) log.info("pipeline_stopped") + async def note_replayed(self, records: int) -> None: + """Account for dead records that are inflight again.""" + if records <= 0: + return + async with self._capacity: + self._inflight += records + + def stats(self) -> dict[str, SinkStats]: + return {name: SinkStats(**vars(item)) for name, item in self._stats.items()} + + @property + def sources(self) -> list[Source]: + return list(self._sources) + + @property + def sinks(self) -> list[Sink]: + return [dispatcher.sink for dispatcher in self._dispatchers] + + def sink_types(self) -> dict[str, str]: + return {dispatcher.name: dispatcher.plugin_type for dispatcher in self._dispatchers} + + @property + def queue_size(self) -> int: + return self._inflight + def _build_pipeline(self) -> None: - for sc in self._config.sources: - cls = self._registry.get_source(sc.type) - if not cls: - log.warning("source_type_not_found", type=sc.type) + self._sources.clear() + self._processors.clear() + self._dispatchers.clear() + self._source_runners.clear() + self._stats.clear() + self._sink_names = [] + + for source_cfg in self._config.sources: + if not source_cfg.enabled: + log.info("source_disabled", source=source_cfg.name) continue - source = cls(name=sc.name, config=sc.options) + cls = self._registry.get_source(source_cfg.type) + if cls is None: + log.warning("source_type_not_found", type=source_cfg.type, name=source_cfg.name) + continue + source = cls(name=source_cfg.name, config=source_cfg.options) self._sources.append(source) - runner = SourceRunner(source, self._ingest) - self._source_runners.append(runner) + self._source_runners.append(SourceRunner(source, self._ingest)) - for pc in self._config.processors: - cls = self._registry.get_processor(pc.type) - if not cls: - log.warning("processor_type_not_found", type=pc.type) + for processor_cfg in self._config.processors: + if not processor_cfg.enabled: + log.info("processor_disabled", processor=processor_cfg.name) continue - proc = cls(name=pc.name, config=pc.options) - self._processors.append(proc) + cls = self._registry.get_processor(processor_cfg.type) + if cls is None: + log.warning( + "processor_type_not_found", + type=processor_cfg.type, + name=processor_cfg.name, + ) + continue + self._processors.append(cls(name=processor_cfg.name, config=processor_cfg.options)) - for sc in self._config.sinks: - cls = self._registry.get_sink(sc.type) - if not cls: - log.warning("sink_type_not_found", type=sc.type) + for sink_cfg in self._config.sinks: + if not sink_cfg.enabled: + log.info("sink_disabled", sink=sink_cfg.name) continue - sink = cls(name=sc.name, config=sc.options) - self._dispatchers.append(SinkDispatcher(sink, self._backend, self._config)) + cls = self._registry.get_sink(sink_cfg.type) + if cls is None: + log.warning("sink_type_not_found", type=sink_cfg.type, name=sink_cfg.name) + continue + sink = cls(name=sink_cfg.name, config=sink_cfg.options) + self._dispatchers.append(SinkDispatcher(sink, sink_cfg.type)) + self._stats[sink_cfg.name] = SinkStats() + self._sink_names.append(sink_cfg.name) async def _ingest(self, record: Record) -> None: - """Called by sources. Runs processors and persists.""" - for proc in self._processors: - result = await proc.process(record) + for processor in self._processors: + result = await processor.process(record) if result is None: + log.debug("record_dropped", record_id=record.record_id) + self._flow("dropped", record, processor=processor.name) return record = result + if not self._sink_names: + log.warning("record_dropped_no_sinks", record_id=record.record_id) + return + + limit = self._config.pipeline.max_queue_size + async with self._capacity: + while self._running and limit > 0 and self._inflight >= limit: + await self._capacity.wait() + if not self._running: + return + self._inflight += 1 + stored = StoredRecord( record_id=record.record_id, source=record.source, @@ -162,104 +241,185 @@ class Pipeline: metadata=record.metadata, destination=record.destination, created_at=record.timestamp, + sinks=list(self._sink_names), ) - self._backend.put(stored) - log.debug("record_persisted", record_id=record.record_id) + try: + await self._store.put(stored) + except Exception: + async with self._capacity: + self._inflight = max(0, self._inflight - 1) + self._capacity.notify_all() + raise + log.debug("record_persisted", record_id=record.record_id, sinks=self._sink_names) + self._flow("collected", record) async def _dispatch_loop(self) -> None: - """Background loop: pull pending records from backend and deliver.""" while self._running: try: - await self._dispatch_batch() + did_work = await self._dispatch_batch() except asyncio.CancelledError: break except Exception: log.exception("dispatch_loop_error") await asyncio.sleep(1) + continue + if not did_work and self._running: + await asyncio.sleep(await self._idle_delay()) - async def _dispatch_batch(self) -> None: - pending = self._backend.pending(limit=self._config.pipeline.batch_size) - retry = self._backend.ready_for_retry(limit=self._config.pipeline.batch_size) - records = pending + retry + async def _dispatch_batch(self) -> bool: + if not self._dispatchers: + return False - if not records: - await asyncio.sleep(self._config.pipeline.flush_interval) - return - - for stored in records: - if not self._running: - break - record = Record( - source=stored.source, - record_type=stored.record_type, - body=stored.body, - metadata=stored.metadata, - destination=stored.destination, - record_id=stored.record_id, - timestamp=stored.created_at, + async def drain(dispatcher: SinkDispatcher) -> int: + claimed = await self._store.claim( + dispatcher.name, + limit=self._config.pipeline.batch_size, ) - for dispatcher in self._dispatchers: - await self._deliver_with_retry(dispatcher, record) + for stored in claimed: + if not self._running: + break + await self._deliver_once(dispatcher, stored) + return len(claimed) - if len(records) < self._config.pipeline.batch_size: - await asyncio.sleep(self._config.pipeline.flush_interval) + counts = await asyncio.gather(*(drain(dispatcher) for dispatcher in self._dispatchers)) + return any(count > 0 for count in counts) - async def _deliver_with_retry(self, dispatcher: SinkDispatcher, record: Record) -> None: - retry_cfg = None - for sc in self._config.sinks: - if sc.name == dispatcher._sink.name: - retry_cfg = sc.retry - break + async def _deliver_once(self, dispatcher: SinkDispatcher, stored: StoredRecord) -> None: + record = Record( + source=stored.source, + record_type=stored.record_type, + body=stored.body, + metadata=stored.metadata, + destination=stored.destination, + record_id=stored.record_id, + timestamp=stored.created_at, + ) + delivery = stored.deliveries.get(dispatcher.name) + retry_count = delivery.retry_count if delivery else 0 + try: + result = await dispatcher.deliver(record) + except Exception as exc: + result = DeliveryResult(success=False, record_id=record.record_id, error=str(exc)) - max_attempts = retry_cfg.max_attempts if retry_cfg else 3 - min_delay = retry_cfg.min_delay if retry_cfg else 1.0 - max_delay = retry_cfg.max_delay if retry_cfg else 120.0 + retry_cfg = self._retry_for(dispatcher.name) + attempts_used = retry_count + 1 + if result.success: + await self._finish(record.record_id, dispatcher.name, "ack") + self._flow("delivered", record, sink=dispatcher.name) + return + error = result.error or "unknown" + if result.permanent or attempts_used >= retry_cfg.max_attempts: + await self._finish(record.record_id, dispatcher.name, "dead", error) + self._flow("dead", record, sink=dispatcher.name, error=error) + return + delay = min( + retry_cfg.min_delay * (retry_cfg.multiplier**retry_count), + retry_cfg.max_delay, + ) + await self._finish(record.record_id, dispatcher.name, "nack", error, delay) + self._flow("retry", record, sink=dispatcher.name, error=error) - attempt = 0 - while attempt < max_attempts: - attempt += 1 - try: - result = await dispatcher.deliver(record) - except Exception as exc: - result = DeliveryResult( - success=False, - record_id=record.record_id, - error=str(exc), - ) + async def _finish( + self, + record_id: str, + sink: str, + action: str, + error: str | None = None, + delay: float = 0, + ) -> None: + stats = self._stats.setdefault(sink, SinkStats()) + if action == "ack": + await self._store.ack(record_id, sink) + stats.delivered += 1 + stats.last_error = "" + elif action == "nack": + await self._store.nack(record_id, error or "unknown", retry_delay=delay, sink=sink) + stats.failed += 1 + stats.last_error = error or "" + log.warning("delivery_retry", sink=sink, record_id=record_id, error=error, delay=delay) + else: + await self._store.dead(record_id, error or "dead", sink) + stats.dead += 1 + stats.last_error = error or "" + log.error("delivery_dead", sink=sink, record_id=record_id, error=error) - if result.success: - self._backend.ack(record.record_id) - return + stored = await self._store.get(record_id) + if stored is not None and stored.is_terminal(): + async with self._capacity: + self._inflight = max(0, self._inflight - 1) + self._capacity.notify_all() - if result.permanent: - self._backend.dead(record.record_id, result.error or "permanent failure") - return + async def _idle_delay(self) -> float: + delay = self._config.pipeline.flush_interval + earliest = await self._store.earliest_retry_at() + if earliest is None: + return delay + try: + when = datetime.fromisoformat(earliest) + except ValueError: + return delay + if when.tzinfo is None: + when = when.replace(tzinfo=UTC) + remaining = (when - datetime.now(UTC)).total_seconds() + if remaining < delay: + return max(0.0, remaining) + return delay - backoff = min(min_delay * (2 ** (attempt - 1)), max_delay) - stored = self._backend.get(record.record_id) - if stored: - self._backend.nack( - record.record_id, - error=result.error or "unknown", - retry_delay=backoff, - ) - - if attempt < max_attempts: - await asyncio.sleep(backoff) - - self._backend.dead( - record.record_id, - f"max retries ({max_attempts}) exceeded: {result.error}", + def _flow(self, stage: str, record: Record, **fields: str) -> None: + if not self._config.logging.verbose: + return + log.info( + "flow", + stage=stage, + source=record.source, + record_id=record.record_id[:8], + message=preview_message(record.body), + **{key: value for key, value in fields.items() if value}, ) - @property - def sources(self) -> list[Source]: - return list(self._sources) + def _retry_for(self, sink_name: str) -> SinkRetryConfig: + for sink_cfg in self._config.sinks: + if sink_cfg.name == sink_name: + return sink_cfg.retry + return SinkRetryConfig() - @property - def sinks(self) -> list[Sink]: - return [d._sink for d in self._dispatchers] + async def _retention_loop(self) -> None: + interval = self._config.retention.interval_seconds + while self._running: + try: + await asyncio.sleep(interval) + if not self._running: + break + await self._retain() + except asyncio.CancelledError: + break + except Exception: + log.exception("retention_error") - @property - def queue_size(self) -> int: - return self._queue.qsize() + async def _retain(self) -> None: + retention = self._config.retention + now = datetime.now(UTC) + delivered_before = (now - timedelta(seconds=retention.delivered_ttl_seconds)).isoformat() + dead_before = (now - timedelta(seconds=retention.dead_ttl_seconds)).isoformat() + removed = await self._store.delete_expired(delivered_before, dead_before) + if removed: + log.info("retention_deleted", records=removed) + + async def _cancel(self, task: asyncio.Task[None] | None) -> None: + if task is None: + return + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +def sink_stats_map(stats: dict[str, SinkStats]) -> dict[str, dict[str, int]]: + """Plain counters for metrics rendering.""" + rendered: dict[str, dict[str, int]] = {} + for name, item in stats.items(): + rendered[name] = field_counts(item) + return rendered + + +def field_counts(item: SinkStats) -> dict[str, int]: + return {"delivered": item.delivered, "failed": item.failed, "dead": item.dead} diff --git a/src/daena/core/runtime.py b/src/daena/core/runtime.py index 98200ee..ef2ea73 100644 --- a/src/daena/core/runtime.py +++ b/src/daena/core/runtime.py @@ -1,7 +1,12 @@ from __future__ import annotations -from daena.config import DaenaConfig -from daena.core.pipeline import Pipeline +import asyncio +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from daena.config import DaenaConfig, load_config +from daena.core.pipeline import Pipeline, SinkStats from daena.logutil import get_logger from daena.persistence.backend import Backend from daena.persistence.lmdb import LMDBBackend @@ -11,15 +16,34 @@ from daena.plugins.registry import PluginRegistry, get_registry log = get_logger("daena.runtime") +@dataclass(frozen=True) +class HealthReport: + healthy: bool + running: bool + backend: bool + plugins_loaded: bool + status: str + oldest_pending_age_seconds: int + detail: str + + class DaenaRuntime: """Top-level service runtime. - Owns configuration, persistence backend, pipeline, and lifecycle. - Designed to run under asyncio with clean signal-based shutdown. + Owns configuration, the persistence backend, and the pipeline. + Reload re-reads the config file when one was provided, then restarts + the pipeline against the same data directory. """ - def __init__(self, config: DaenaConfig) -> None: + def __init__( + self, + config: DaenaConfig, + config_path: str | Path | None = None, + force_verbose: bool = False, + ) -> None: self._config = config + self._config_path = Path(config_path) if config_path is not None else None + self._force_verbose = force_verbose self._registry = get_registry() self._backend: Backend | None = None self._pipeline: Pipeline | None = None @@ -29,6 +53,10 @@ class DaenaRuntime: def config(self) -> DaenaConfig: return self._config + @property + def config_path(self) -> Path | None: + return self._config_path + @property def registry(self) -> PluginRegistry: return self._registry @@ -41,9 +69,19 @@ class DaenaRuntime: def pipeline(self) -> Pipeline | None: return self._pipeline + @property + def running(self) -> bool: + return self._running + async def start(self) -> None: log.info("runtime_starting") + try: + await self._start() + except Exception: + await self.stop() + raise + async def _start(self) -> None: discover_plugins() log.info( "plugins_discovered", @@ -53,16 +91,17 @@ class DaenaRuntime: ) persistence_cfg = self._config.persistence - if persistence_cfg.backend == "lmdb": - self._backend = LMDBBackend( - path=persistence_cfg.lmdb.path, - map_size=persistence_cfg.lmdb.map_size, - max_databases=persistence_cfg.lmdb.max_databases, - ) - else: + if persistence_cfg.backend != "lmdb": raise ValueError(f"Unknown backend: {persistence_cfg.backend}") - + self._backend = LMDBBackend( + path=persistence_cfg.lmdb.path, + map_size=persistence_cfg.lmdb.map_size, + max_databases=persistence_cfg.lmdb.max_databases, + ) self._backend.open() + recovered = self._backend.recover_inflight() + if recovered: + log.info("startup_requeued", count=recovered) log.info("backend_opened", backend=persistence_cfg.backend) self._pipeline = Pipeline(self._config, self._backend, self._registry) @@ -73,18 +112,104 @@ class DaenaRuntime: async def stop(self) -> None: log.info("runtime_stopping") self._running = False - if self._pipeline: await self._pipeline.stop() - + self._pipeline = None if self._backend: self._backend.close() + self._backend = None log.info("backend_closed") - log.info("runtime_stopped") async def reload(self) -> None: - log.info("runtime_reloading") - await self.stop() - await self.start() + """Restart the pipeline. Re-read YAML when a config path is known.""" + previous = self._config + if self._config_path is not None: + self._config = load_config(self._config_path) + if self._force_verbose: + self._config.logging.verbose = True + log.info("runtime_reloading", path=str(self._config_path) if self._config_path else None) + try: + await self.stop() + await self.start() + except Exception: + log.exception("runtime_reload_failed") + self._config = previous + if not self._running: + await self.start() + raise log.info("runtime_reloaded") + + async def replay_dead( + self, + record_id: str | None = None, + sink: str | None = None, + limit: int = 100, + ) -> int: + if self._backend is None: + return 0 + replayed = await asyncio.to_thread( + self._backend.replay, + record_id, + sink, + limit, + ) + if self._pipeline is not None: + await self._pipeline.note_replayed(replayed) + return replayed + + def health(self) -> HealthReport: + running = self._running + backend_open = self._backend is not None + plugins_loaded = bool(self._registry.list_sources()) + age = self._oldest_age_seconds() if backend_open else 0 + threshold = self._config.health.max_pending_age_seconds + degraded = backend_open and age > threshold + if not running or not backend_open: + return HealthReport( + healthy=False, + running=running, + backend=backend_open, + plugins_loaded=plugins_loaded, + status="down", + oldest_pending_age_seconds=age, + detail="runtime is not running" if not running else "backend is closed", + ) + if degraded: + return HealthReport( + healthy=False, + running=True, + backend=True, + plugins_loaded=plugins_loaded, + status="degraded", + oldest_pending_age_seconds=age, + detail=f"oldest inflight record is {age}s old", + ) + return HealthReport( + healthy=True, + running=True, + backend=True, + plugins_loaded=plugins_loaded, + status="ok", + oldest_pending_age_seconds=age, + detail="ok", + ) + + def sink_stats(self) -> dict[str, SinkStats]: + if self._pipeline is None: + return {} + return self._pipeline.stats() + + def _oldest_age_seconds(self) -> int: + if self._backend is None: + return 0 + oldest = self._backend.oldest_queued_at() + if oldest is None: + return 0 + try: + queued = datetime.fromisoformat(oldest) + except ValueError: + return 0 + if queued.tzinfo is None: + queued = queued.replace(tzinfo=UTC) + return max(0, int((datetime.now(UTC) - queued).total_seconds())) diff --git a/src/daena/persistence/async_backend.py b/src/daena/persistence/async_backend.py new file mode 100644 index 0000000..39560bc --- /dev/null +++ b/src/daena/persistence/async_backend.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Iterator + +from daena.persistence.backend import Backend +from daena.persistence.models import RecordState, StoredRecord + + +class AsyncBackend: + """Runs synchronous backend calls off the event loop.""" + + def __init__(self, backend: Backend) -> None: + self._backend = backend + + async def put(self, record: StoredRecord) -> None: + await asyncio.to_thread(self._backend.put, record) + + async def get(self, record_id: str) -> StoredRecord | None: + return await asyncio.to_thread(self._backend.get, record_id) + + async def claim( + self, + sink: str, + limit: int = 100, + now: str | None = None, + ) -> list[StoredRecord]: + return await asyncio.to_thread(self._backend.claim, sink, limit, now) + + async def ack(self, record_id: str, sink: str | None = None) -> None: + await asyncio.to_thread(self._backend.ack, record_id, sink) + + async def nack( + self, + record_id: str, + error: str, + retry_delay: float = 10.0, + sink: str | None = None, + ) -> None: + await asyncio.to_thread(self._backend.nack, record_id, error, retry_delay, sink) + + async def dead(self, record_id: str, error: str, sink: str | None = None) -> None: + await asyncio.to_thread(self._backend.dead, record_id, error, sink) + + async def recover_inflight(self) -> int: + return await asyncio.to_thread(self._backend.recover_inflight) + + async def replay( + self, + record_id: str | None = None, + sink: str | None = None, + limit: int = 100, + ) -> int: + return await asyncio.to_thread(self._backend.replay, record_id, sink, limit) + + async def count(self, state: RecordState | None = None, sink: str | None = None) -> int: + return await asyncio.to_thread(self._backend.count, state, sink) + + async def sink_counts(self) -> dict[str, dict[str, int]]: + return await asyncio.to_thread(self._backend.sink_counts) + + async def count_inflight(self) -> int: + return await asyncio.to_thread(self._backend.count_inflight) + + async def oldest_queued_at(self) -> str | None: + return await asyncio.to_thread(self._backend.oldest_queued_at) + + async def earliest_retry_at(self) -> str | None: + return await asyncio.to_thread(self._backend.earliest_retry_at) + + async def list_by_state( + self, + state: RecordState, + limit: int = 100, + offset: int = 0, + sink: str | None = None, + ) -> list[StoredRecord]: + return await asyncio.to_thread(self._backend.list_by_state, state, limit, offset, sink) + + async def delete_expired(self, delivered_before: str, dead_before: str) -> int: + return await asyncio.to_thread(self._backend.delete_expired, delivered_before, dead_before) + + def iter_pending(self) -> Iterator[StoredRecord]: + return self._backend.iter_pending() diff --git a/src/daena/persistence/backend.py b/src/daena/persistence/backend.py index b3cc60a..d41a8db 100644 --- a/src/daena/persistence/backend.py +++ b/src/daena/persistence/backend.py @@ -7,11 +7,10 @@ from daena.persistence.models import RecordState, StoredRecord class Backend(Protocol): - """Abstract interface for persistence backends. + """Storage contract for the durable record buffer. - This Protocol defines the contract that all storage backends must fulfill. - The MVP implementation uses LMDB; future implementations may target - NATS JetStream, filesystem-backed stores, or cloud queues. + Delivery state is tracked per sink. Index keys for a delivery are replaced + on every transition so scans stay exact. """ def open(self) -> None: @@ -21,7 +20,7 @@ class Backend(Protocol): """Close the backend and release resources.""" def put(self, record: StoredRecord) -> None: - """Store a new record.""" + """Store a new record. Existing ids are left unchanged.""" def get(self, record_id: str) -> StoredRecord | None: """Retrieve a record by ID.""" @@ -32,57 +31,94 @@ class Backend(Protocol): state: RecordState, last_error: str | None = None, next_retry_at: str | None = None, + sink: str | None = None, ) -> None: - """Update a record's delivery state.""" + """Set one sink's delivery state. Sink is inferred when unambiguous.""" def pending( self, limit: int = 100, older_than: str | None = None, + sink: str | None = None, ) -> list[StoredRecord]: - """Return pending records for delivery, oldest first.""" + """Return records with a pending delivery, oldest first.""" def ready_for_retry( self, limit: int = 100, now: str | None = None, + sink: str | None = None, ) -> list[StoredRecord]: - """Return failed records whose retry time has arrived.""" + """Return records whose failed delivery is due for another attempt.""" - def ack(self, record_id: str) -> None: - """Mark a record as successfully delivered.""" + def claim(self, sink: str, limit: int = 100, now: str | None = None) -> list[StoredRecord]: + """Mark due deliveries for one sink as delivering and return them.""" + + def ack(self, record_id: str, sink: str | None = None) -> None: + """Mark one sink's delivery as successful.""" def nack( self, record_id: str, error: str, retry_delay: float = 10.0, + sink: str | None = None, ) -> None: - """Mark delivery failed and schedule retry.""" + """Mark one sink's delivery failed and schedule a later attempt.""" - def dead(self, record_id: str, error: str) -> None: - """Move a record to the dead-letter queue.""" + def dead(self, record_id: str, error: str, sink: str | None = None) -> None: + """Move one sink's delivery to the dead-letter state.""" - def count(self, state: RecordState | None = None) -> int: - """Count records, optionally filtered by state.""" + def recover_inflight(self) -> int: + """Return deliveries left in `delivering` back to `pending`.""" + + def replay( + self, + record_id: str | None = None, + sink: str | None = None, + limit: int = 100, + ) -> int: + """Requeue dead deliveries. Returns how many records became inflight.""" + + def count(self, state: RecordState | None = None, sink: str | None = None) -> int: + """Count records, or deliveries when a state or sink is given.""" + + def sink_counts(self) -> dict[str, dict[str, int]]: + """Delivery counts keyed by sink then state.""" + + def count_inflight(self) -> int: + """Records that still need work from at least one sink.""" + + def oldest_queued_at(self) -> str | None: + """Earliest queue time among pending, delivering, and failed deliveries.""" + + def earliest_retry_at(self) -> str | None: + """Soonest scheduled retry timestamp, if any failed delivery is waiting.""" def list_by_state( self, state: RecordState, limit: int = 100, offset: int = 0, + sink: str | None = None, ) -> list[StoredRecord]: - """List records by state with pagination.""" + """List records that have a delivery in `state`.""" def iter_pending(self) -> Iterator[StoredRecord]: - """Iterate over all pending records (for recovery).""" + """Iterate records that still have a pending delivery.""" - def delete_old( + def delete_old(self, before: str, state: RecordState | None = None) -> int: + """Delete records whose deliveries are all in `state` and older than `before`.""" + + def delete_expired( self, - before: str, - state: RecordState | None = None, + delivered_before: str, + dead_before: str, ) -> int: - """Delete records older than a timestamp. Returns count.""" + """Delete fully terminal records once each delivery is past its own cutoff.""" + + def index_count(self) -> int: + """Number of secondary index keys. Used to detect leaked indexes.""" def cleanup(self) -> None: - """Perform backend-specific maintenance.""" + """Rebuild derived counters from stored records.""" diff --git a/src/daena/persistence/lmdb.py b/src/daena/persistence/lmdb.py index 0b15460..332b677 100644 --- a/src/daena/persistence/lmdb.py +++ b/src/daena/persistence/lmdb.py @@ -1,45 +1,90 @@ from __future__ import annotations +import base64 import json -from collections.abc import Iterator -from datetime import UTC, datetime +from collections.abc import Callable, Iterator +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import lmdb from daena.exceptions import BackendError -from daena.persistence.models import RecordState, StoredRecord +from daena.persistence.models import DeliveryStatus, RecordState, StoredRecord _REC_PREFIX = b"rec:" -_IDX_PENDING = b"idx:pending:" -_IDX_DELIVERED = b"idx:delivered:" -_IDX_FAILED = b"idx:failed:" -_IDX_DEAD = b"idx:dead:" -_IDX_RETRY = b"idx:retry:" +_IDX_PREFIX = b"idx\x00" +_RETRY_PREFIX = b"rty\x00" +_META_SCHEMA = b"meta:schema" +_META_RECORDS = b"meta:records" +_META_STATE = b"meta:state\x00" +_SCHEMA_VERSION = b"2" +_QUEUE_STATES = frozenset({RecordState.pending, RecordState.delivering, RecordState.failed}) +_LEGACY_INDEX_PREFIXES = ( + b"idx:pending:", + b"idx:delivered:", + b"idx:failed:", + b"idx:dead:", + b"idx:retry:", +) -_PREFIX_MAP: dict[RecordState, bytes] = { - RecordState.pending: _IDX_PENDING, - RecordState.delivered: _IDX_DELIVERED, - RecordState.failed: _IDX_FAILED, - RecordState.dead: _IDX_DEAD, -} + +def _check_sink(sink: str) -> None: + if not sink or "\x00" in sink: + raise BackendError(f"Invalid sink name: {sink!r}") + + +def _resolve_sink(record: StoredRecord, sink: str | None) -> str: + if sink is not None: + _check_sink(sink) + if sink not in record.deliveries: + raise BackendError(f"Unknown sink {sink} for record {record.record_id}") + return sink + if len(record.deliveries) == 1: + return next(iter(record.deliveries)) + raise BackendError( + f"Record {record.record_id} has {len(record.deliveries)} deliveries; sink is required" + ) + + +def _state_index(sink: str, state: RecordState, ts: str, record_id: str) -> bytes: + return b"\x00".join( + (b"idx", sink.encode(), state.value.encode(), ts.encode(), record_id.encode()) + ) + + +def _state_prefix(sink: str, state: RecordState) -> bytes: + return b"\x00".join((b"idx", sink.encode(), state.value.encode(), b"")) + + +def _retry_index(sink: str, ts: str, record_id: str) -> bytes: + return b"\x00".join((b"rty", sink.encode(), ts.encode(), record_id.encode())) + + +def _retry_prefix(sink: str) -> bytes: + return b"\x00".join((b"rty", sink.encode(), b"")) + + +def _meta_state(sink: str, state: RecordState) -> bytes: + return _META_STATE + sink.encode() + b"\x00" + state.value.encode() + + +def _encode_key(key: bytes) -> str: + return base64.b64encode(key).decode("ascii") + + +def _decode_key(value: str) -> bytes: + return base64.b64decode(value.encode("ascii")) + + +def _as_bytes(value: bytes | memoryview) -> bytes: + if isinstance(value, memoryview): + return value.tobytes() + return value class LMDBBackend: - """LMDB-based persistence backend using key-prefix partitioning. - - Uses a single LMDB database with key prefixes to emulate named - sub-databases. - - Key design: - - rec:{record_id} -> serialized record JSON - - idx:pending:{ts}\x00{id} -> "" (ordered FIFO queue) - - idx:delivered:{ts}\x00{id} -> "" - - idx:failed:{ts}\x00{id} -> "" - - idx:dead:{ts}\x00{id} -> "" - - idx:retry:{next}\x00{id} -> "" - """ + """LMDB buffer. One payload key per record, one live index key per delivery.""" def __init__( self, @@ -59,7 +104,13 @@ class LMDBBackend: def open(self) -> None: self._path.mkdir(parents=True, exist_ok=True) - self._env = lmdb.open(str(self._path), self._map_size, self._max_databases) + self._env = lmdb.open( + str(self._path), + map_size=self._map_size, + subdir=True, + lock=True, + ) + self._migrate() def close(self) -> None: if self._env is not None: @@ -69,40 +120,103 @@ class LMDBBackend: def _rec_key(self, record_id: str) -> bytes: return _REC_PREFIX + record_id.encode() - def _idx_key(self, prefix: bytes, record_id: str) -> bytes: - ts = datetime.now(UTC).isoformat() - return prefix + f"{ts}\x00{record_id}".encode() - - def _record_id_from_idx(self, key: bytes, prefix: bytes) -> str: - suffix = key[len(prefix) :].decode() - return suffix.split("\x00", 1)[1] - - def _read_record(self, txn: lmdb.Transaction, record_id: str) -> StoredRecord | None: + def _read(self, txn: lmdb.Transaction, record_id: str) -> StoredRecord | None: raw = txn.get(self._rec_key(record_id)) if raw is None: return None - return StoredRecord.from_dict(json.loads(raw)) + payload: dict[str, Any] = json.loads(_as_bytes(raw)) + return StoredRecord.from_dict(payload) + + def _save(self, txn: lmdb.Transaction, record: StoredRecord) -> None: + txn.put(self._rec_key(record.record_id), json.dumps(record.to_dict()).encode()) + + def _counter_get(self, txn: lmdb.Transaction, key: bytes) -> int: + raw = txn.get(key) + if raw is None: + return 0 + return int(_as_bytes(raw)) + + def _counter_add(self, txn: lmdb.Transaction, key: bytes, delta: int) -> None: + value = self._counter_get(txn, key) + delta + if value <= 0: + txn.delete(key) + return + txn.put(key, str(value).encode()) + + def _reindex( + self, + txn: lmdb.Transaction, + record: StoredRecord, + delivery: DeliveryStatus, + previous: RecordState | None, + ) -> None: + if delivery.index_key: + txn.delete(_decode_key(delivery.index_key)) + delivery.index_key = None + if delivery.retry_index_key: + txn.delete(_decode_key(delivery.retry_index_key)) + delivery.retry_index_key = None + + ts = delivery.queued_at if delivery.state in _QUEUE_STATES else delivery.updated_at + key = _state_index(delivery.sink, delivery.state, ts, record.record_id) + txn.put(key, b"") + delivery.index_key = _encode_key(key) + if delivery.state == RecordState.failed and delivery.next_retry_at: + retry_key = _retry_index(delivery.sink, delivery.next_retry_at, record.record_id) + txn.put(retry_key, b"") + delivery.retry_index_key = _encode_key(retry_key) + if previous != delivery.state: + if previous is not None: + self._counter_add(txn, _meta_state(delivery.sink, previous), -1) + self._counter_add(txn, _meta_state(delivery.sink, delivery.state), 1) + + def _delete_record(self, txn: lmdb.Transaction, record: StoredRecord) -> None: + for delivery in record.deliveries.values(): + if delivery.index_key: + txn.delete(_decode_key(delivery.index_key)) + if delivery.retry_index_key: + txn.delete(_decode_key(delivery.retry_index_key)) + self._counter_add(txn, _meta_state(delivery.sink, delivery.state), -1) + txn.delete(self._rec_key(record.record_id)) + self._counter_add(txn, _META_RECORDS, -1) + + def _mutate( + self, + record_id: str, + sink: str | None, + change: Callable[[DeliveryStatus], None], + ) -> StoredRecord | None: + env = self._ensure_env() + with env.begin(write=True) as txn: + record = self._read(txn, record_id) + if record is None: + return None + sink_name = _resolve_sink(record, sink) + delivery = record.deliveries[sink_name] + previous = delivery.state + change(delivery) + self._reindex(txn, record, delivery, previous) + self._save(txn, record) + return record def put(self, record: StoredRecord) -> None: + for name in record.deliveries: + _check_sink(name) env = self._ensure_env() - rec_key = self._rec_key(record.record_id) - value = json.dumps(record.to_dict()).encode() - idx_key = self._idx_key(_IDX_PENDING, record.record_id) - with env.begin(write=True) as txn: - if txn.get(rec_key) is not None: + if txn.get(self._rec_key(record.record_id)) is not None: return - txn.put(rec_key, value) - txn.put(idx_key, b"") + for delivery in record.deliveries.values(): + delivery.index_key = None + delivery.retry_index_key = None + self._reindex(txn, record, delivery, None) + self._save(txn, record) + self._counter_add(txn, _META_RECORDS, 1) def get(self, record_id: str) -> StoredRecord | None: env = self._ensure_env() - key = self._rec_key(record_id) with env.begin() as txn: - value = txn.get(key) - if value is None: - return None - return StoredRecord.from_dict(json.loads(value)) + return self._read(txn, record_id) def update_state( self, @@ -110,195 +224,484 @@ class LMDBBackend: state: RecordState, last_error: str | None = None, next_retry_at: str | None = None, + sink: str | None = None, ) -> None: - env = self._ensure_env() - rec_key = self._rec_key(record_id) now = datetime.now(UTC).isoformat() - with env.begin(write=True) as txn: - raw = txn.get(rec_key) - if raw is None: - return - record_dict: dict[str, Any] = json.loads(raw) - record_dict["state"] = state.value - record_dict["updated_at"] = now + def change(delivery: DeliveryStatus) -> None: + delivery.state = state + delivery.updated_at = now if last_error is not None: - record_dict["last_error"] = last_error + delivery.last_error = last_error if next_retry_at is not None: - record_dict["next_retry_at"] = next_retry_at - txn.put(rec_key, json.dumps(record_dict).encode()) + delivery.next_retry_at = next_retry_at + if state != RecordState.failed: + delivery.next_retry_at = None - target_prefix = _PREFIX_MAP.get(state) - if target_prefix and state != RecordState.pending: - new_idx = self._idx_key(target_prefix, record_id) - txn.put(new_idx, b"") + self._mutate(record_id, sink, change) - if state == RecordState.failed and next_retry_at: - retry_key = _IDX_RETRY + f"{next_retry_at}\x00{record_id}".encode() - txn.put(retry_key, b"") - - def pending(self, limit: int = 100, older_than: str | None = None) -> list[StoredRecord]: - _ = older_than - return self._list_by_state_prefix(_IDX_PENDING, RecordState.pending, limit) - - def ready_for_retry(self, limit: int = 100, now: str | None = None) -> list[StoredRecord]: - env = self._ensure_env() + def claim(self, sink: str, limit: int = 100, now: str | None = None) -> list[StoredRecord]: + _check_sink(sink) now_str = now or datetime.now(UTC).isoformat() + env = self._ensure_env() + results: list[StoredRecord] = [] + with env.begin(write=True) as txn: + record_ids = self._due_ids(txn, sink, limit, now_str) + for record_id in record_ids: + record = self._read(txn, record_id) + if record is None or sink not in record.deliveries: + continue + delivery = record.deliveries[sink] + if delivery.state not in (RecordState.pending, RecordState.failed): + continue + previous = delivery.state + delivery.state = RecordState.delivering + delivery.updated_at = now_str + delivery.next_retry_at = None + self._reindex(txn, record, delivery, previous) + self._save(txn, record) + results.append(record) + return results + + def _due_ids( + self, + txn: lmdb.Transaction, + sink: str, + limit: int, + now_str: str, + ) -> list[str]: + found: list[str] = [] + seen: set[str] = set() + self._collect_pending_ids(txn, sink, limit, seen, found) + if len(found) < limit: + self._collect_retry_ids(txn, sink, limit, now_str, seen, found) + return found + + def _collect_pending_ids( + self, + txn: lmdb.Transaction, + sink: str, + limit: int, + seen: set[str], + found: list[str], + ) -> None: + prefix = _state_prefix(sink, RecordState.pending) + for key in self._scan(txn, prefix): + record_id = key.split(b"\x00")[-1].decode() + if record_id in seen: + continue + record = self._read(txn, record_id) + delivery = record.deliveries.get(sink) if record else None + if delivery is None or delivery.state != RecordState.pending: + continue + seen.add(record_id) + found.append(record_id) + if len(found) >= limit: + return + + def _collect_retry_ids( + self, + txn: lmdb.Transaction, + sink: str, + limit: int, + now_str: str, + seen: set[str], + found: list[str], + ) -> None: + prefix = _retry_prefix(sink) + for key in self._scan(txn, prefix): + parts = key.split(b"\x00") + if len(parts) < 4: + continue + retry_at = parts[2].decode() + if retry_at > now_str: + break + record_id = parts[3].decode() + if record_id in seen: + continue + record = self._read(txn, record_id) + delivery = record.deliveries.get(sink) if record else None + if delivery is None or delivery.state != RecordState.failed: + continue + seen.add(record_id) + found.append(record_id) + if len(found) >= limit: + return + + def ack(self, record_id: str, sink: str | None = None) -> None: + now = datetime.now(UTC).isoformat() + + def change(delivery: DeliveryStatus) -> None: + delivery.state = RecordState.delivered + delivery.updated_at = now + delivery.last_error = None + delivery.next_retry_at = None + + self._mutate(record_id, sink, change) + + def nack( + self, + record_id: str, + error: str, + retry_delay: float = 10.0, + sink: str | None = None, + ) -> None: + now = datetime.now(UTC) + next_retry = (now + timedelta(seconds=retry_delay)).isoformat() + + def change(delivery: DeliveryStatus) -> None: + delivery.state = RecordState.failed + delivery.updated_at = now.isoformat() + delivery.last_error = error + delivery.retry_count += 1 + delivery.next_retry_at = next_retry + + self._mutate(record_id, sink, change) + + def dead(self, record_id: str, error: str, sink: str | None = None) -> None: + now = datetime.now(UTC).isoformat() + + def change(delivery: DeliveryStatus) -> None: + delivery.state = RecordState.dead + delivery.updated_at = now + delivery.last_error = error + delivery.next_retry_at = None + + self._mutate(record_id, sink, change) + + def recover_inflight(self) -> int: + env = self._ensure_env() + now = datetime.now(UTC).isoformat() + recovered = 0 + with env.begin(write=True) as txn: + records = list(self._iter_records(txn)) + for record in records: + changed = False + for delivery in record.deliveries.values(): + if delivery.state != RecordState.delivering: + continue + previous = delivery.state + delivery.state = RecordState.pending + delivery.updated_at = now + self._reindex(txn, record, delivery, previous) + changed = True + recovered += 1 + if changed: + self._save(txn, record) + return recovered + + def replay( + self, + record_id: str | None = None, + sink: str | None = None, + limit: int = 100, + ) -> int: + env = self._ensure_env() + now = datetime.now(UTC).isoformat() + transitioned = 0 + with env.begin(write=True) as txn: + if record_id is not None: + records = [] + record = self._read(txn, record_id) + if record is not None: + records.append(record) + else: + records = self._records_in_state(txn, RecordState.dead, sink, limit) + for record in records: + if transitioned >= limit and record_id is None: + break + was_terminal = record.is_terminal() + changed = False + for name, delivery in record.deliveries.items(): + if sink is not None and name != sink: + continue + if delivery.state != RecordState.dead: + continue + previous = delivery.state + delivery.state = RecordState.pending + delivery.updated_at = now + delivery.queued_at = now + delivery.retry_count = 0 + delivery.last_error = None + delivery.next_retry_at = None + self._reindex(txn, record, delivery, previous) + changed = True + if not changed: + continue + self._save(txn, record) + if was_terminal and record.is_inflight(): + transitioned += 1 + return transitioned + + def pending( + self, + limit: int = 100, + older_than: str | None = None, + sink: str | None = None, + ) -> list[StoredRecord]: + records = self.list_by_state(RecordState.pending, limit=limit, sink=sink) + if older_than is None: + return records + selected: list[StoredRecord] = [] + for record in records: + for delivery in record.deliveries.values(): + if sink is not None and delivery.sink != sink: + continue + if delivery.state == RecordState.pending and delivery.queued_at <= older_than: + selected.append(record) + break + return selected[:limit] + + def ready_for_retry( + self, + limit: int = 100, + now: str | None = None, + sink: str | None = None, + ) -> list[StoredRecord]: + now_str = now or datetime.now(UTC).isoformat() + env = self._ensure_env() results: list[StoredRecord] = [] seen: set[str] = set() with env.begin() as txn: - cursor = txn.cursor() - if not cursor.set_range(_IDX_RETRY): - return results - for key, _ in cursor: - if not key.startswith(_IDX_RETRY): - break - record_id = self._record_id_from_idx(key, _IDX_RETRY) - retry_ts = key[len(_IDX_RETRY) :].decode().split("\x00", 1)[0] - if retry_ts > now_str: - break + prefix = _retry_prefix(sink) if sink else _RETRY_PREFIX + for key in self._scan(txn, prefix): + parts = key.split(b"\x00") + if len(parts) < 4: + continue + retry_at = parts[2].decode() + if retry_at > now_str: + if sink is not None: + break + continue + record_id = parts[3].decode() + sink_name = parts[1].decode() if record_id in seen: continue + record = self._read(txn, record_id) + delivery = record.deliveries.get(sink_name) if record else None + if record is None or delivery is None or delivery.state != RecordState.failed: + continue seen.add(record_id) - record = self._read_record(txn, record_id) - if record and record.state == RecordState.failed: - results.append(record) - if len(results) >= limit: - break + results.append(record) + if len(results) >= limit: + break return results - def ack(self, record_id: str) -> None: - self.update_state(record_id, RecordState.delivered) - - def nack(self, record_id: str, error: str, retry_delay: float = 10.0) -> None: - from datetime import timedelta - - next_retry = (datetime.now(UTC) + timedelta(seconds=retry_delay)).isoformat() - self.update_state( - record_id, - RecordState.failed, - last_error=error, - next_retry_at=next_retry, - ) - - def dead(self, record_id: str, error: str) -> None: - self.update_state(record_id, RecordState.dead, last_error=error) - - def count(self, state: RecordState | None = None) -> int: + def count(self, state: RecordState | None = None, sink: str | None = None) -> int: env = self._ensure_env() with env.begin() as txn: - if state is None: - cursor = txn.cursor() - count = 0 - if cursor.set_range(_REC_PREFIX): - for key, _ in cursor: - if not key.startswith(_REC_PREFIX): - break - count += 1 - return count - prefix = _PREFIX_MAP.get(state) - if prefix is None: - return 0 - cursor = txn.cursor() - count = 0 - if cursor.set_range(prefix): - for key, _ in cursor: - if not key.startswith(prefix): - break - record_id = self._record_id_from_idx(key, prefix) - record = self._read_record(txn, record_id) - if record and record.state == state: - count += 1 - return count + if state is None and sink is None: + return self._counter_get(txn, _META_RECORDS) + total = 0 + prefix = _META_STATE + (sink.encode() + b"\x00" if sink else b"") + for key, value in self._scan_items(txn, prefix): + if state is not None and not key.endswith(b"\x00" + state.value.encode()): + continue + total += int(value) + return total + + def sink_counts(self) -> dict[str, dict[str, int]]: + env = self._ensure_env() + counts: dict[str, dict[str, int]] = {} + with env.begin() as txn: + for key, value in self._scan_items(txn, _META_STATE): + rest = key[len(_META_STATE) :] + sink_raw, _, state_raw = rest.partition(b"\x00") + sink_name = sink_raw.decode() + counts.setdefault(sink_name, {})[state_raw.decode()] = int(value) + return counts + + def count_inflight(self) -> int: + env = self._ensure_env() + total = 0 + with env.begin() as txn: + for record in self._iter_records(txn): + if record.is_inflight(): + total += 1 + return total + + def earliest_retry_at(self) -> str | None: + earliest: str | None = None + env = self._ensure_env() + with env.begin() as txn: + for key in self._scan(txn, _RETRY_PREFIX): + parts = key.split(b"\x00") + if len(parts) < 4: + continue + ts = parts[2].decode() + if earliest is None or ts < earliest: + earliest = ts + return earliest + + def oldest_queued_at(self) -> str | None: + interesting = { + RecordState.pending.value.encode(), + RecordState.delivering.value.encode(), + RecordState.failed.value.encode(), + } + oldest: str | None = None + env = self._ensure_env() + with env.begin() as txn: + for key in self._scan(txn, _IDX_PREFIX): + parts = key.split(b"\x00") + if len(parts) < 5 or parts[2] not in interesting: + continue + ts = parts[3].decode() + if oldest is None or ts < oldest: + oldest = ts + return oldest def list_by_state( - self, state: RecordState, limit: int = 100, offset: int = 0 - ) -> list[StoredRecord]: - prefix = _PREFIX_MAP.get(state) - if prefix is None: - return [] - return self._list_by_state_prefix(prefix, state, limit, offset) - - def iter_pending(self) -> Iterator[StoredRecord]: - seen: set[str] = set() - env = self._ensure_env() - with env.begin() as txn: - cursor = txn.cursor() - if cursor.set_range(_IDX_PENDING): - for key, _ in cursor: - if not key.startswith(_IDX_PENDING): - break - record_id = self._record_id_from_idx(key, _IDX_PENDING) - if record_id in seen: - continue - seen.add(record_id) - record = self._read_record(txn, record_id) - if record and record.state == RecordState.pending: - yield record - - def delete_old(self, before: str, state: RecordState | None = None) -> int: - env = self._ensure_env() - removed = 0 - - prefixes = [_IDX_DELIVERED, _IDX_DEAD] - if state: - p = _PREFIX_MAP.get(state) - prefixes = [p] if p else [] - - with env.begin(write=True) as txn: - for prefix in prefixes: - cursor = txn.cursor() - if not cursor.set_range(prefix): - continue - seen_del: set[str] = set() - for key, _ in cursor: - if not key.startswith(prefix): - break - suffix = key[len(prefix) :].decode() - ts_part = suffix.split("\x00", 1)[0] - if ts_part > before: - break - record_id = suffix.split("\x00", 1)[1] - if record_id in seen_del: - continue - seen_del.add(record_id) - txn.delete(self._rec_key(record_id)) - txn.delete(key) - removed += 1 - return removed - - def cleanup(self) -> None: - pass - - def _list_by_state_prefix( self, - prefix: bytes, state: RecordState, limit: int = 100, offset: int = 0, + sink: str | None = None, + ) -> list[StoredRecord]: + env = self._ensure_env() + with env.begin() as txn: + return self._records_in_state(txn, state, sink, limit, offset) + + def iter_pending(self) -> Iterator[StoredRecord]: + yield from self.list_by_state(RecordState.pending, limit=1_000_000) + + def delete_old(self, before: str, state: RecordState | None = None) -> int: + required = {state} if state is not None else {RecordState.delivered, RecordState.dead} + return self._delete_where( + lambda record: ( + bool(record.deliveries) + and all(item.state in required for item in record.deliveries.values()) + and all(item.updated_at <= before for item in record.deliveries.values()) + ) + ) + + def delete_expired(self, delivered_before: str, dead_before: str) -> int: + def eligible(record: StoredRecord) -> bool: + if not record.is_terminal(): + return False + for delivery in record.deliveries.values(): + cutoff = ( + delivered_before if delivery.state == RecordState.delivered else dead_before + ) + if delivery.updated_at > cutoff: + return False + return True + + return self._delete_where(eligible) + + def index_count(self) -> int: + env = self._ensure_env() + total = 0 + with env.begin() as txn: + total += sum(1 for _ in self._scan(txn, _IDX_PREFIX)) + total += sum(1 for _ in self._scan(txn, _RETRY_PREFIX)) + return total + + def cleanup(self) -> None: + env = self._ensure_env() + with env.begin(write=True) as txn: + self._wipe_prefix(txn, _META_STATE) + txn.delete(_META_RECORDS) + records = 0 + for record in self._iter_records(txn): + records += 1 + for delivery in record.deliveries.values(): + self._counter_add(txn, _meta_state(delivery.sink, delivery.state), 1) + if records: + txn.put(_META_RECORDS, str(records).encode()) + + def _delete_where(self, predicate: Callable[[StoredRecord], bool]) -> int: + env = self._ensure_env() + removed = 0 + with env.begin(write=True) as txn: + matched = [record for record in self._iter_records(txn) if predicate(record)] + for record in matched: + self._delete_record(txn, record) + removed += 1 + return removed + + def _records_in_state( + self, + txn: lmdb.Transaction, + state: RecordState, + sink: str | None, + limit: int, + offset: int = 0, ) -> list[StoredRecord]: seen: set[str] = set() - env = self._ensure_env() + skipped = 0 results: list[StoredRecord] = [] - with env.begin() as txn: - cursor = txn.cursor() - if not cursor.set_range(prefix): - return results - skipped = 0 - for key, _ in cursor: - if not key.startswith(prefix): - break - record_id = self._record_id_from_idx(key, prefix) + prefixes = [_state_prefix(sink, state)] if sink is not None else [_IDX_PREFIX] + for prefix in prefixes: + for key in self._scan(txn, prefix): + parts = key.split(b"\x00") + if len(parts) < 5: + continue + if parts[2].decode() != state.value: + continue + record_id = parts[4].decode() if record_id in seen: continue - seen.add(record_id) - record = self._read_record(txn, record_id) - if record is None or record.state != state: + record = self._read(txn, record_id) + if record is None: continue + sink_name = parts[1].decode() + delivery = record.deliveries.get(sink_name) + if delivery is None or delivery.state != state: + continue + seen.add(record_id) if skipped < offset: skipped += 1 continue results.append(record) if len(results) >= limit: - break + return results return results + + def _iter_records(self, txn: lmdb.Transaction) -> Iterator[StoredRecord]: + for _key, value in self._scan_items(txn, _REC_PREFIX): + payload: dict[str, Any] = json.loads(value) + yield StoredRecord.from_dict(payload) + + def _scan(self, txn: lmdb.Transaction, prefix: bytes) -> Iterator[bytes]: + for key, _value in self._scan_items(txn, prefix): + yield key + + def _scan_items(self, txn: lmdb.Transaction, prefix: bytes) -> Iterator[tuple[bytes, bytes]]: + cursor = txn.cursor() + if not cursor.set_range(prefix): + return + for key, value in cursor: + key_bytes = _as_bytes(key) + if not key_bytes.startswith(prefix): + break + yield key_bytes, _as_bytes(value) + + def _wipe_prefix(self, txn: lmdb.Transaction, prefix: bytes) -> None: + for key in list(self._scan(txn, prefix)): + txn.delete(key) + + def _migrate(self) -> None: + env = self._ensure_env() + with env.begin(write=True) as txn: + if txn.get(_META_SCHEMA) == _SCHEMA_VERSION: + return + records = list(self._iter_records(txn)) + self._wipe_prefix(txn, _IDX_PREFIX) + self._wipe_prefix(txn, _RETRY_PREFIX) + self._wipe_prefix(txn, _META_STATE) + for prefix in _LEGACY_INDEX_PREFIXES: + self._wipe_prefix(txn, prefix) + txn.delete(_META_RECORDS) + stored = 0 + for record in records: + for delivery in record.deliveries.values(): + _check_sink(delivery.sink) + delivery.index_key = None + delivery.retry_index_key = None + self._reindex(txn, record, delivery, None) + self._save(txn, record) + stored += 1 + if stored: + txn.put(_META_RECORDS, str(stored).encode()) + txn.put(_META_SCHEMA, _SCHEMA_VERSION) diff --git a/src/daena/persistence/models.py b/src/daena/persistence/models.py index 74e1af7..acaf8af 100644 --- a/src/daena/persistence/models.py +++ b/src/daena/persistence/models.py @@ -14,8 +14,92 @@ class RecordState(StrEnum): dead = "dead" +def _now() -> str: + return datetime.now(UTC).isoformat() + + +_TERMINAL = frozenset({RecordState.delivered, RecordState.dead}) +_INFLIGHT = frozenset({RecordState.pending, RecordState.delivering, RecordState.failed}) + + +class DeliveryStatus: + """Delivery progress of one record toward one sink.""" + + def __init__( + self, + sink: str, + state: RecordState = RecordState.pending, + retry_count: int = 0, + next_retry_at: str | None = None, + last_error: str | None = None, + updated_at: str | None = None, + queued_at: str | None = None, + index_key: str | None = None, + retry_index_key: str | None = None, + ) -> None: + now = _now() + self.sink = sink + self.state = state + self.retry_count = retry_count + self.next_retry_at = next_retry_at + self.last_error = last_error + self.updated_at = updated_at or now + self.queued_at = queued_at or now + self.index_key = index_key + self.retry_index_key = retry_index_key + + def to_dict(self) -> dict[str, Any]: + return { + "sink": self.sink, + "state": self.state.value, + "retry_count": self.retry_count, + "next_retry_at": self.next_retry_at, + "last_error": self.last_error, + "updated_at": self.updated_at, + "queued_at": self.queued_at, + "index_key": self.index_key, + "retry_index_key": self.retry_index_key, + } + + @classmethod + def from_dict(cls, sink: str, data: dict[str, Any]) -> DeliveryStatus: + return cls( + sink=sink, + state=RecordState(data.get("state", RecordState.pending)), + retry_count=int(data.get("retry_count", 0)), + next_retry_at=data.get("next_retry_at"), + last_error=data.get("last_error"), + updated_at=data.get("updated_at"), + queued_at=data.get("queued_at") or data.get("updated_at"), + index_key=data.get("index_key"), + retry_index_key=data.get("retry_index_key"), + ) + + +def aggregate_state(deliveries: dict[str, DeliveryStatus]) -> RecordState: + """Single state for display. In-flight work wins over terminal states.""" + states = {item.state for item in deliveries.values()} + if not states: + return RecordState.pending + if RecordState.delivering in states: + return RecordState.delivering + if RecordState.pending in states: + return RecordState.pending + if RecordState.failed in states: + return RecordState.failed + if states <= {RecordState.delivered}: + return RecordState.delivered + if RecordState.dead in states: + return RecordState.dead + return RecordState.delivered + + class StoredRecord: - """A record as stored in the persistence backend.""" + """A record as stored in the persistence backend. + + Payload is stored once. Each configured sink has its own DeliveryStatus, + so one sink's failure cannot rewind another sink's acknowledgement. + """ def __init__( self, @@ -24,26 +108,81 @@ class StoredRecord: record_type: str = "", body: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None, - state: RecordState = RecordState.pending, created_at: str | None = None, - updated_at: str | None = None, + destination: str | None = None, + deliveries: dict[str, DeliveryStatus] | None = None, + sinks: list[str] | None = None, + state: RecordState = RecordState.pending, retry_count: int = 0, next_retry_at: str | None = None, last_error: str | None = None, - destination: str | None = None, + updated_at: str | None = None, ) -> None: + now = created_at or _now() self.record_id = record_id or str(uuid.uuid4()) self.source = source self.record_type = record_type self.body = body or {} self.metadata = metadata or {} - self.state = state - self.created_at = created_at or datetime.now(UTC).isoformat() - self.updated_at = updated_at or self.created_at - self.retry_count = retry_count - self.next_retry_at = next_retry_at - self.last_error = last_error + self.created_at = now self.destination = destination + if deliveries is not None: + self.deliveries = deliveries + elif sinks: + self.deliveries = { + name: DeliveryStatus( + sink=name, + state=state, + retry_count=retry_count, + next_retry_at=next_retry_at, + last_error=last_error, + updated_at=updated_at or now, + queued_at=now, + ) + for name in sinks + } + else: + self.deliveries = {} + + @property + def state(self) -> RecordState: + return aggregate_state(self.deliveries) + + @property + def updated_at(self) -> str: + times = [item.updated_at for item in self.deliveries.values()] + return max(times) if times else self.created_at + + @property + def retry_count(self) -> int: + if not self.deliveries: + return 0 + return max(item.retry_count for item in self.deliveries.values()) + + @property + def next_retry_at(self) -> str | None: + times = [ + item.next_retry_at + for item in self.deliveries.values() + if item.next_retry_at is not None + ] + return min(times) if times else None + + @property + def last_error(self) -> str | None: + ordered = sorted(self.deliveries.values(), key=lambda item: item.updated_at, reverse=True) + for item in ordered: + if item.last_error: + return item.last_error + return None + + def is_terminal(self) -> bool: + if not self.deliveries: + return False + return all(item.state in _TERMINAL for item in self.deliveries.values()) + + def is_inflight(self) -> bool: + return any(item.state in _INFLIGHT for item in self.deliveries.values()) def to_dict(self) -> dict[str, Any]: return { @@ -52,28 +191,43 @@ class StoredRecord: "record_type": self.record_type, "body": self.body, "metadata": self.metadata, - "state": self.state.value, - "created_at": self.created_at, - "updated_at": self.updated_at, - "retry_count": self.retry_count, - "next_retry_at": self.next_retry_at, - "last_error": self.last_error, "destination": self.destination, + "created_at": self.created_at, + "deliveries": {name: item.to_dict() for name, item in self.deliveries.items()}, } @classmethod def from_dict(cls, data: dict[str, Any]) -> StoredRecord: + raw_deliveries = data.get("deliveries") + deliveries: dict[str, DeliveryStatus] | None + if isinstance(raw_deliveries, dict): + deliveries = { + name: DeliveryStatus.from_dict(name, payload) + for name, payload in raw_deliveries.items() + if isinstance(payload, dict) + } + elif "state" in data: + sink_name = data.get("destination") or "default" + deliveries = { + sink_name: DeliveryStatus( + sink=sink_name, + state=RecordState(data.get("state", RecordState.pending)), + retry_count=int(data.get("retry_count", 0)), + next_retry_at=data.get("next_retry_at"), + last_error=data.get("last_error"), + updated_at=data.get("updated_at") or data.get("created_at"), + queued_at=data.get("created_at"), + ) + } + else: + deliveries = {} return cls( record_id=data.get("record_id"), source=data.get("source", ""), record_type=data.get("record_type", ""), body=data.get("body", {}), metadata=data.get("metadata", {}), - state=RecordState(data.get("state", "pending")), created_at=data.get("created_at"), - updated_at=data.get("updated_at"), - retry_count=data.get("retry_count", 0), - next_retry_at=data.get("next_retry_at"), - last_error=data.get("last_error"), destination=data.get("destination"), + deliveries=deliveries, ) diff --git a/src/daena/pipeline/base.py b/src/daena/pipeline/base.py index 65a2f22..df6c4b1 100644 --- a/src/daena/pipeline/base.py +++ b/src/daena/pipeline/base.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any @@ -26,6 +27,11 @@ class Source(ABC): def __init__(self, name: str, config: dict[str, Any] | None = None) -> None: self.name = name self.config = config or {} + self._on_record: Callable[[Record], Awaitable[None]] | None = None + + def set_callback(self, callback: Callable[[Record], Awaitable[None]]) -> None: + """Register the pipeline ingest coroutine. Called before start.""" + self._on_record = callback @abstractmethod async def start(self) -> None: diff --git a/src/daena/pipeline/processors/enricher.py b/src/daena/pipeline/processors/enricher.py new file mode 100644 index 0000000..a16d7ac --- /dev/null +++ b/src/daena/pipeline/processors/enricher.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import socket +from typing import Any + +from daena.pipeline.base import Processor +from daena.pipeline.record import Record + + +class EnricherProcessor(Processor): + """Adds hostname and static fields to each record. + + Options: + hostname: write the local hostname (default true) + static: mapping copied onto every record + target: `metadata` (default) or `body` + """ + + def __init__(self, name: str, config: dict[str, Any] | None = None) -> None: + super().__init__(name, config) + self._add_hostname = bool(self.config.get("hostname", True)) + static = self.config.get("static", {}) + self._static: dict[str, Any] = {} + if isinstance(static, dict): + self._static = {str(key): value for key, value in static.items()} + target = str(self.config.get("target", "metadata")) + if target not in ("metadata", "body"): + target = "metadata" + self._target = target + self._hostname = socket.gethostname() + + async def process(self, record: Record) -> Record | None: + destination = record.metadata if self._target == "metadata" else record.body + if self._add_hostname: + destination["hostname"] = self._hostname + destination.update(self._static) + return record diff --git a/src/daena/pipeline/processors/filter.py b/src/daena/pipeline/processors/filter.py new file mode 100644 index 0000000..7c5cef0 --- /dev/null +++ b/src/daena/pipeline/processors/filter.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from typing import Any + +from daena.pipeline.base import Processor +from daena.pipeline.record import Record + + +class FilterProcessor(Processor): + """Drops records that match `drop`, or that fail to match `keep`. + + Keys are `source`, `record_type`, `body.`, or `metadata.`. + Dotted paths walk nested mappings. Comparison is equality, with string + fallback so YAML numbers match string fields. + """ + + def __init__(self, name: str, config: dict[str, Any] | None = None) -> None: + super().__init__(name, config) + self._drop = _rules(self.config.get("drop")) + self._keep = _rules(self.config.get("keep")) + + async def process(self, record: Record) -> Record | None: + if self._drop and _matches_all(record, self._drop): + return None + if self._keep and not _matches_all(record, self._keep): + return None + return record + + +def _rules(raw: Any) -> dict[str, Any]: + if not isinstance(raw, dict): + return {} + return {str(key): value for key, value in raw.items()} + + +def _matches_all(record: Record, rules: dict[str, Any]) -> bool: + return all(_equals(_lookup(record, path), expected) for path, expected in rules.items()) + + +def _equals(actual: Any, expected: Any) -> bool: + if actual == expected: + return True + if actual is None: + return False + return str(actual) == str(expected) + + +def _lookup(record: Record, path: str) -> Any: + if path == "source": + return record.source + if path == "record_type": + return record.record_type + if path.startswith("body."): + return _dig(record.body, path.removeprefix("body.")) + if path.startswith("metadata."): + return _dig(record.metadata, path.removeprefix("metadata.")) + return _dig(record.body, path) + + +def _dig(value: Any, path: str) -> Any: + current = value + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current diff --git a/src/daena/pipeline/sinks/dummy.py b/src/daena/pipeline/sinks/dummy.py index 2c4bbc6..221424f 100644 --- a/src/daena/pipeline/sinks/dummy.py +++ b/src/daena/pipeline/sinks/dummy.py @@ -1,14 +1,35 @@ from __future__ import annotations +import asyncio +from typing import Any + from daena.pipeline.base import DeliveryResult, Sink from daena.pipeline.record import Record class DummySink(Sink): - """Logs delivered records without sending anywhere. + """Accepts records without sending them anywhere. - Useful for testing and development. + `fail_times` injects that many failures before success. + `permanent` marks those failures as non-retryable. + `delay` sleeps before each attempt so tests can observe backpressure. """ + def __init__(self, name: str, config: dict[str, Any] | None = None) -> None: + super().__init__(name, config) + self._fail_remaining = int(self.config.get("fail_times", 0)) + self._permanent = bool(self.config.get("permanent", False)) + self._delay = float(self.config.get("delay", 0)) + async def deliver(self, record: Record) -> DeliveryResult: + if self._delay: + await asyncio.sleep(self._delay) + if self._fail_remaining > 0: + self._fail_remaining -= 1 + return DeliveryResult( + success=False, + record_id=record.record_id, + error="dummy failure", + permanent=self._permanent, + ) return DeliveryResult(success=True, record_id=record.record_id) diff --git a/src/daena/pipeline/sinks/loki.py b/src/daena/pipeline/sinks/loki.py new file mode 100644 index 0000000..b0fe107 --- /dev/null +++ b/src/daena/pipeline/sinks/loki.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +import httpx + +from daena.pipeline.base import DeliveryResult, Sink +from daena.pipeline.record import Record + + +class LokiSink(Sink): + """Pushes one record to a Loki `/loki/api/v1/push` endpoint. + + Options: + url: push URL + labels: static stream labels + tenant: optional `X-Scope-OrgID` value + timeout: request timeout in seconds + """ + + def __init__(self, name: str, config: dict[str, Any] | None = None) -> None: + super().__init__(name, config) + self._url = str(self.config.get("url", "http://127.0.0.1:3100/loki/api/v1/push")) + labels = self.config.get("labels", {}) + self._labels = ( + {str(key): _label_value(value) for key, value in labels.items()} + if isinstance(labels, dict) + else {} + ) + tenant = self.config.get("tenant") + self._tenant = str(tenant) if tenant else None + self._timeout = float(self.config.get("timeout", 30.0)) + self._client: httpx.AsyncClient | None = None + + async def start(self) -> None: + headers: dict[str, str] = {} + if self._tenant: + headers["X-Scope-OrgID"] = self._tenant + self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._timeout), headers=headers) + + async def stop(self) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + async def deliver(self, record: Record) -> DeliveryResult: + if self._client is None: + return DeliveryResult( + success=False, + record_id=record.record_id, + error="Loki sink not initialized", + ) + payload = { + "streams": [ + { + "stream": {**self._labels, "source": _label_value(record.source)}, + "values": [[_timestamp_ns(record.timestamp), _line(record)]], + } + ] + } + try: + response = await self._client.post(self._url, json=payload) + except httpx.TimeoutException as exc: + return DeliveryResult( + success=False, + record_id=record.record_id, + error=f"Timeout: {exc}", + ) + except httpx.RequestError as exc: + return DeliveryResult(success=False, record_id=record.record_id, error=str(exc)) + if response.is_success: + return DeliveryResult(success=True, record_id=record.record_id) + permanent = 400 <= response.status_code < 500 and response.status_code != 429 + return DeliveryResult( + success=False, + record_id=record.record_id, + error=f"HTTP {response.status_code}: {response.text[:200]}", + permanent=permanent, + ) + + +def _line(record: Record) -> str: + message = record.body.get("message") + if isinstance(message, str) and message: + return message + return json.dumps(record.body, default=str, separators=(",", ":")) + + +def _timestamp_ns(timestamp: str) -> str: + try: + parsed = datetime.fromisoformat(timestamp) + except ValueError: + parsed = datetime.now(UTC) + return str(int(parsed.timestamp() * 1_000_000_000)) + + +def _label_value(value: Any) -> str: + text = str(value).replace("\n", " ").replace('"', "'").strip() + return text or "unknown" diff --git a/src/daena/pipeline/sources/dummy.py b/src/daena/pipeline/sources/dummy.py index 0ef3cf1..bb8b364 100644 --- a/src/daena/pipeline/sources/dummy.py +++ b/src/daena/pipeline/sources/dummy.py @@ -20,12 +20,9 @@ class DummySource(Source): self._message: str = ( config.get("message", "daena test event") if config else "daena test event" ) + self._max_count = int(self.config.get("count", 0)) self._task: asyncio.Task[None] | None = None self._running = False - self._on_record: callable | None = None - - def set_callback(self, cb: callable) -> None: - self._on_record = cb async def start(self) -> None: self._running = True @@ -53,4 +50,7 @@ class DummySource(Source): ) if self._on_record: await self._on_record(record) + if self._max_count and counter >= self._max_count: + self._running = False + break await asyncio.sleep(self._interval) diff --git a/src/daena/pipeline/sources/journald.py b/src/daena/pipeline/sources/journald.py new file mode 100644 index 0000000..a730c1e --- /dev/null +++ b/src/daena/pipeline/sources/journald.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import asyncio +import contextlib +import json +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Protocol + +from daena.exceptions import SourceError +from daena.logutil import get_logger +from daena.pipeline.base import Source +from daena.pipeline.record import Record + +log = get_logger("daena.journald") + +JournalHandler = Callable[[dict[str, Any]], Awaitable[None]] + +_JOURNAL_FIELDS = ( + "MESSAGE", + "SYSLOG_IDENTIFIER", + "PRIORITY", + "_PID", + "_UID", + "_GID", + "_SYSTEMD_UNIT", + "_HOSTNAME", + "_COMM", + "SYSLOG_FACILITY", +) + + +class JournalReader(Protocol): + async def run(self, on_entry: JournalHandler) -> None: + """Emit journal entries until stopped.""" + + async def stop(self) -> None: + """Stop emitting.""" + + +def build_journalctl_command( + *, + cursor: str | None, + units: list[str], + matches: list[str], +) -> list[str]: + """Build a follow command. Without a cursor, only entries after start are read.""" + command = ["journalctl", "--follow", "--output=json", "--no-pager"] + if cursor: + command.append(f"--after-cursor={cursor}") + else: + command.append("--lines=0") + for unit in units: + command.extend(["--unit", unit]) + command.extend(matches) + return command + + +def _log_reader_error(task: asyncio.Task[None]) -> None: + if task.cancelled(): + return + error = task.exception() + if error is not None: + log.error("journald_reader_failed", error=str(error)) + + +def entry_to_record(source: str, entry: dict[str, Any]) -> Record: + message = entry.get("MESSAGE", "") + if not isinstance(message, str): + message = str(message) + body: dict[str, Any] = {"message": message} + for field in _JOURNAL_FIELDS: + if field == "MESSAGE" or field not in entry: + continue + value = entry[field] + if isinstance(value, (str, int, float, bool)) or value is None: + body[field.lower()] = value + timestamp = _journal_timestamp(entry.get("__REALTIME_TIMESTAMP")) + metadata = { + "cursor": entry.get("__CURSOR", ""), + "unit": entry.get("_SYSTEMD_UNIT", ""), + } + return Record( + source=source, + record_type="journald", + body=body, + metadata=metadata, + timestamp=timestamp, + ) + + +def _journal_timestamp(raw: Any) -> str: + if isinstance(raw, str) and raw.isdigit(): + seconds = int(raw) / 1_000_000 + return datetime.fromtimestamp(seconds, UTC).isoformat() + return datetime.now(UTC).isoformat() + + +class JournalctlReader: + """Follows `journalctl --output=json` and parses one object per line.""" + + def __init__(self, command: list[str]) -> None: + self._command = command + self._process: asyncio.subprocess.Process | None = None + self._stopped = False + + async def run(self, on_entry: JournalHandler) -> None: + try: + self._process = await asyncio.create_subprocess_exec( + *self._command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise SourceError("journalctl was not found on PATH") from exc + stdout = self._process.stdout + if stdout is None: + raise SourceError("journalctl produced no stdout pipe") + while not self._stopped: + line = await stdout.readline() + if not line: + break + text = line.strip() + if not text: + continue + try: + payload = json.loads(text) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + await on_entry(payload) + + async def stop(self) -> None: + self._stopped = True + process = self._process + if process is None or process.returncode is not None: + return + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=2) + except TimeoutError: + process.kill() + await process.wait() + + +class JournaldSource(Source): + """Tails the systemd journal. + + Options: + units: systemd unit names passed to `--unit` + matches: extra journalctl match arguments + cursor_path: file storing the last `__CURSOR` so restarts do not replay + """ + + def __init__( + self, + name: str, + config: dict[str, Any] | None = None, + reader: JournalReader | None = None, + ) -> None: + super().__init__(name, config) + self._units = [str(unit) for unit in self.config.get("units", [])] + self._matches = [str(match) for match in self.config.get("matches", [])] + cursor_path = self.config.get("cursor_path") + self._cursor_path = Path(cursor_path) if cursor_path else None + self._reader = reader + self._task: asyncio.Task[None] | None = None + + async def start(self) -> None: + if self._reader is None: + cursor = self._read_cursor() + command = build_journalctl_command( + cursor=cursor, + units=self._units, + matches=self._matches, + ) + self._reader = JournalctlReader(command) + self._task = asyncio.create_task(self._reader.run(self._on_entry)) + self._task.add_done_callback(_log_reader_error) + + async def stop(self) -> None: + if self._reader is not None: + await self._reader.stop() + if self._task is not None: + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _on_entry(self, entry: dict[str, Any]) -> None: + record = entry_to_record(self.name, entry) + if self._on_record is not None: + await self._on_record(record) + cursor = entry.get("__CURSOR") + if isinstance(cursor, str) and self._cursor_path is not None: + await asyncio.to_thread(self._write_cursor, cursor) + + def _read_cursor(self) -> str | None: + if self._cursor_path is None or not self._cursor_path.exists(): + return None + text = self._cursor_path.read_text(encoding="utf-8").strip() + return text or None + + def _write_cursor(self, cursor: str) -> None: + if self._cursor_path is None: + return + self._cursor_path.parent.mkdir(parents=True, exist_ok=True) + temporary = self._cursor_path.with_suffix(self._cursor_path.suffix + ".tmp") + temporary.write_text(cursor + "\n", encoding="utf-8") + temporary.replace(self._cursor_path) diff --git a/src/daena/plugins/loader.py b/src/daena/plugins/loader.py index 435197e..d905a40 100644 --- a/src/daena/plugins/loader.py +++ b/src/daena/plugins/loader.py @@ -3,7 +3,15 @@ from __future__ import annotations from importlib.metadata import entry_points from daena.exceptions import PluginError -from daena.plugins.registry import get_registry +from daena.pipeline.processors.enricher import EnricherProcessor +from daena.pipeline.processors.filter import FilterProcessor +from daena.pipeline.processors.passthrough import PassthroughProcessor +from daena.pipeline.sinks.dummy import DummySink +from daena.pipeline.sinks.http import HTTPSink +from daena.pipeline.sinks.loki import LokiSink +from daena.pipeline.sources.dummy import DummySource +from daena.pipeline.sources.journald import JournaldSource +from daena.plugins.registry import PluginRegistry, get_registry SOURCE_EP = "daena.sources" PROCESSOR_EP = "daena.processors" @@ -13,6 +21,7 @@ SINK_EP = "daena.sinks" def discover_plugins() -> None: """Discover plugins via package entry points and register them.""" registry = get_registry() + registry.clear() errors: list[str] = [] for ep in entry_points(group=SOURCE_EP): @@ -38,3 +47,25 @@ def discover_plugins() -> None: if errors: raise PluginError("Plugin loading encountered errors:\n " + "\n ".join(errors)) + _register_builtins(registry) + + +def _register_builtins(registry: PluginRegistry) -> None: + """Register in-tree plugins when an entry point did not already provide them.""" + builtins: tuple[tuple[str, str, type], ...] = ( + ("source", "dummy", DummySource), + ("source", "journald", JournaldSource), + ("processor", "passthrough", PassthroughProcessor), + ("processor", "enricher", EnricherProcessor), + ("processor", "filter", FilterProcessor), + ("sink", "dummy", DummySink), + ("sink", "http", HTTPSink), + ("sink", "loki", LokiSink), + ) + for kind, name, cls in builtins: + if kind == "source" and registry.get_source(name) is None: + registry.register_source(name, cls) + elif kind == "processor" and registry.get_processor(name) is None: + registry.register_processor(name, cls) + elif kind == "sink" and registry.get_sink(name) is None: + registry.register_sink(name, cls) diff --git a/src/daena/plugins/registry.py b/src/daena/plugins/registry.py index 6660f36..48c12a9 100644 --- a/src/daena/plugins/registry.py +++ b/src/daena/plugins/registry.py @@ -47,6 +47,11 @@ class PluginRegistry: "sinks": self.list_sinks(), } + def clear(self) -> None: + self._sources.clear() + self._processors.clear() + self._sinks.clear() + _registry = PluginRegistry() diff --git a/tests/conftest.py b/tests/conftest.py index f7efefa..d56ef14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ def sample_record() -> StoredRecord: record_type="test", body={"message": "hello"}, metadata={"env": "test"}, + sinks=["out"], ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..203a12f --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typer.testing import CliRunner + +from daena.cli.main import app + +runner = CliRunner() + + +def test_version_command() -> None: + result = runner.invoke(app, ["version"]) + assert result.exit_code == 0 + assert "daena v" in result.output + + +def test_status_reports_connection_failure() -> None: + result = runner.invoke(app, ["--target", "127.0.0.1:1", "status"]) + assert result.exit_code == 1 + assert "Cannot connect" in result.output + + +def test_records_rejects_unknown_kind() -> None: + result = runner.invoke(app, ["records", "other"]) + assert result.exit_code == 2 + assert "pending or dead" in result.output diff --git a/tests/test_dispatch.py b/tests/test_dispatch.py new file mode 100644 index 0000000..f3cc1b4 --- /dev/null +++ b/tests/test_dispatch.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest +import yaml + +from daena.config import ( + DaenaConfig, + HealthConfig, + PipelineConfig, + ProcessorConfig, + SinkConfig, + SinkRetryConfig, + SourceConfig, + load_config, +) +from daena.core.runtime import DaenaRuntime +from daena.persistence.models import RecordState + + +def _runtime(tmp_path: Path, **overrides: object) -> DaenaRuntime: + config = DaenaConfig() + config.persistence.lmdb.path = str(tmp_path / "lmdb") + config.pipeline = PipelineConfig(batch_size=10, flush_interval=0.05, max_queue_size=10_000) + config.retention.interval_seconds = 3600 + for key, value in overrides.items(): + setattr(config, key, value) + return DaenaRuntime(config) + + +@pytest.mark.asyncio +async def test_retry_does_not_block_until_success(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.sources = [ + SourceConfig( + name="once", + type="dummy", + options={"interval": 0.05, "count": 1, "message": "retry"}, + ) + ] + runtime.config.sinks = [ + SinkConfig( + name="out", + type="dummy", + retry=SinkRetryConfig(max_attempts=3, min_delay=0, max_delay=0, multiplier=1), + options={"fail_times": 1}, + ) + ] + await runtime.start() + delivered = 0 + for _ in range(40): + assert runtime.backend is not None + delivered = runtime.backend.count(RecordState.delivered) + if delivered == 1: + break + await asyncio.sleep(0.05) + dead = runtime.backend.count(RecordState.dead) if runtime.backend else -1 + await runtime.stop() + assert delivered == 1 + assert dead == 0 + + +@pytest.mark.asyncio +async def test_sinks_keep_independent_delivery_state(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.sources = [ + SourceConfig(name="once", type="dummy", options={"interval": 0.05, "count": 1}) + ] + runtime.config.sinks = [ + SinkConfig(name="good", type="dummy"), + SinkConfig(name="bad", type="dummy", options={"fail_times": 5, "permanent": True}), + ] + await runtime.start() + stored = None + for _ in range(40): + assert runtime.backend is not None + records = runtime.backend.list_by_state(RecordState.dead, limit=10) + if records and records[0].deliveries["good"].state == RecordState.delivered: + stored = records[0] + break + await asyncio.sleep(0.05) + await runtime.stop() + assert stored is not None + assert stored.deliveries["good"].state == RecordState.delivered + assert stored.deliveries["bad"].state == RecordState.dead + + +@pytest.mark.asyncio +async def test_backpressure_stops_ingest(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.pipeline = PipelineConfig(batch_size=10, flush_interval=0.05, max_queue_size=2) + runtime.config.sources = [ + SourceConfig(name="fast", type="dummy", options={"interval": 0.01, "message": "bp"}) + ] + runtime.config.sinks = [SinkConfig(name="slow", type="dummy", options={"delay": 30})] + await runtime.start() + await asyncio.sleep(0.4) + assert runtime.backend is not None + stored = runtime.backend.count() + await runtime.stop() + assert stored <= 2 + + +@pytest.mark.asyncio +async def test_disabled_source_emits_nothing(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.sources = [ + SourceConfig(name="off", type="dummy", enabled=False, options={"interval": 0.05}) + ] + runtime.config.sinks = [SinkConfig(name="out", type="dummy")] + await runtime.start() + await asyncio.sleep(0.3) + assert runtime.backend is not None + total = runtime.backend.count() + await runtime.stop() + assert total == 0 + + +@pytest.mark.asyncio +async def test_reload_rereads_file(tmp_path: Path) -> None: + path = tmp_path / "daena.yaml" + data = { + "persistence": {"lmdb": {"path": str(tmp_path / "lmdb")}}, + "pipeline": {"batch_size": 10, "flush_interval": 0.05, "max_queue_size": 100}, + "retention": {"interval_seconds": 3600}, + "sources": [{"name": "live", "type": "dummy", "enabled": True, "options": {"interval": 1}}], + "sinks": [{"name": "out", "type": "dummy", "enabled": True}], + } + path.write_text(yaml.dump(data), encoding="utf-8") + runtime = DaenaRuntime(load_config(path), config_path=path) + await runtime.start() + data["sources"][0]["enabled"] = False + path.write_text(yaml.dump(data), encoding="utf-8") + await runtime.reload() + assert runtime.config.sources[0].enabled is False + assert runtime.running is True + await runtime.stop() + + +@pytest.mark.asyncio +async def test_health_degrades_when_delivery_stalls(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.health = HealthConfig(max_pending_age_seconds=0) + runtime.config.sources = [ + SourceConfig(name="fast", type="dummy", options={"interval": 0.05, "count": 1}) + ] + runtime.config.sinks = [SinkConfig(name="slow", type="dummy", options={"delay": 30})] + await runtime.start() + await asyncio.sleep(1.2) + report = runtime.health() + await runtime.stop() + assert report.status == "degraded" + assert report.healthy is False + assert report.oldest_pending_age_seconds >= 1 + + +@pytest.mark.asyncio +async def test_filter_drops_record_before_persist(tmp_path: Path) -> None: + runtime = _runtime(tmp_path) + runtime.config.sources = [ + SourceConfig( + name="once", + type="dummy", + options={"interval": 0.05, "count": 1, "message": "drop-me"}, + ) + ] + runtime.config.processors = [ + ProcessorConfig( + name="drop", + type="filter", + options={"drop": {"body.message": "drop-me #1"}}, + ) + ] + runtime.config.sinks = [SinkConfig(name="out", type="dummy")] + await runtime.start() + await asyncio.sleep(0.3) + assert runtime.backend is not None + total = runtime.backend.count() + await runtime.stop() + assert total == 0 diff --git a/tests/test_grpc.py b/tests/test_grpc.py index dd136f7..43d98f8 100644 --- a/tests/test_grpc.py +++ b/tests/test_grpc.py @@ -8,18 +8,20 @@ import pytest from daena.api.grpc_server import DaenaGRPCServer from daena.api.pb2 import daena_pb2 as pb2 from daena.api.pb2 import daena_pb2_grpc as pb2_grpc -from daena.config import DaenaConfig, SinkConfig, SourceConfig +from daena.config import DaenaConfig, PipelineConfig, SinkConfig, SourceConfig from daena.core.runtime import DaenaRuntime +from daena.persistence.models import RecordState @pytest.fixture -async def grpc_env(): +async def grpc_env(tmp_path): config = DaenaConfig() config.sources = [ SourceConfig(name="dummy", type="dummy", options={"interval": 0.2, "message": "test"}) ] config.sinks = [SinkConfig(name="log", type="dummy")] - config.persistence.lmdb.path = "/tmp/daena_grpc_test" + config.persistence.lmdb.path = str(tmp_path / "lmdb") + config.pipeline = PipelineConfig(batch_size=100, flush_interval=5.0, max_queue_size=10_000) config.api.grpc.port = 18642 runtime = DaenaRuntime(config) @@ -76,7 +78,7 @@ async def test_get_queue_state(grpc_env) -> None: await asyncio.sleep(1) resp = await stub.GetQueueState(pb2.Empty(), timeout=5) assert resp.total > 0 - assert resp.pending > 0 + assert resp.pending + resp.delivered > 0 @pytest.mark.asyncio @@ -98,6 +100,30 @@ async def test_list_sinks(grpc_env) -> None: assert resp.sinks[0].name == "log" +@pytest.mark.asyncio +async def test_replay_dead(grpc_env) -> None: + runtime, stub, _ = grpc_env + await asyncio.sleep(1) + assert runtime.backend is not None + stored = None + for state in (RecordState.pending, RecordState.delivered, RecordState.failed): + found = runtime.backend.list_by_state(state, limit=1) + if found: + stored = found[0] + break + assert stored is not None + for sink in stored.deliveries: + runtime.backend.dead(stored.record_id, "grpc replay", sink=sink) + resp = await stub.ReplayDead( + pb2.ReplayRequest(record_id=stored.record_id, limit=10), + timeout=5, + ) + assert resp.replayed == 1 + again = runtime.backend.get(stored.record_id) + assert again is not None + assert again.deliveries["log"].state == RecordState.pending + + @pytest.mark.asyncio async def test_reload_config(grpc_env) -> None: _, stub, _ = grpc_env diff --git a/tests/test_http_sink.py b/tests/test_http_sink.py index 3746ecc..0271250 100644 --- a/tests/test_http_sink.py +++ b/tests/test_http_sink.py @@ -10,7 +10,7 @@ from typing import Any import pytest -from daena.config import DaenaConfig, SinkConfig, SourceConfig +from daena.config import DaenaConfig, PipelineConfig, SinkConfig, SourceConfig from daena.core.runtime import DaenaRuntime from daena.logutil import setup_logging from daena.persistence.models import RecordState @@ -20,6 +20,20 @@ setup_logging(level="error", fmt="json") pytestmark = pytest.mark.asyncio +def _fast_pipeline(config: DaenaConfig) -> None: + config.pipeline = PipelineConfig(batch_size=100, flush_interval=0.05, max_queue_size=10_000) + + +async def _wait_until(predicate: Any, timeout: float = 3.0) -> bool: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if predicate(): + return True + await asyncio.sleep(0.05) + return False + + class _TestHTTPServer: """Minimal async HTTP server to receive forwarded records.""" @@ -74,11 +88,12 @@ async def test_http_sink_forwards_records() -> None: config.data_dir = str(tmpdir) config.log_dir = str(tmpdir) config.persistence.lmdb.path = str(data_dir / "lmdb") + _fast_pipeline(config) config.sources = [ SourceConfig( name="test_source", type="dummy", - options={"interval": 0.2, "message": "http_sink_test"}, + options={"interval": 0.05, "count": 1, "message": "http_sink_test"}, ) ] config.sinks = [ @@ -91,7 +106,7 @@ async def test_http_sink_forwards_records() -> None: runtime = DaenaRuntime(config) await runtime.start() - await asyncio.sleep(3.0) + await _wait_until(lambda: len(server.received) > 0) await runtime.stop() await server.stop() @@ -119,11 +134,12 @@ async def test_http_sink_permanent_failure_on_404() -> None: config.data_dir = str(tmpdir) config.log_dir = str(tmpdir) config.persistence.lmdb.path = str(data_dir / "lmdb") + _fast_pipeline(config) config.sources = [ SourceConfig( name="test_source", type="dummy", - options={"interval": 0.2, "message": "http_404_test"}, + options={"interval": 0.05, "count": 1, "message": "http_404_test"}, ) ] config.sinks = [ @@ -140,11 +156,13 @@ async def test_http_sink_permanent_failure_on_404() -> None: runtime = DaenaRuntime(config) await runtime.start() - await asyncio.sleep(3.0) + assert runtime.backend is not None + backend = runtime.backend + await _wait_until(lambda: backend.count(RecordState.dead) > 0) - dead = runtime.backend.count(RecordState.dead) - failed = runtime.backend.count(RecordState.failed) - total = runtime.backend.count() + dead = backend.count(RecordState.dead) + failed = backend.count(RecordState.failed) + total = backend.count() await runtime.stop() @@ -168,11 +186,12 @@ async def test_http_sink_delivery_result() -> None: config.data_dir = str(tmpdir) config.log_dir = str(tmpdir) config.persistence.lmdb.path = str(data_dir / "lmdb") + _fast_pipeline(config) config.sources = [ SourceConfig( name="test_source", type="dummy", - options={"interval": 0.2, "message": "http_delivery_test"}, + options={"interval": 0.05, "count": 1, "message": "http_delivery_test"}, ) ] config.sinks = [ @@ -185,9 +204,11 @@ async def test_http_sink_delivery_result() -> None: runtime = DaenaRuntime(config) await runtime.start() - await asyncio.sleep(3.0) + assert runtime.backend is not None + backend = runtime.backend + await _wait_until(lambda: backend.count(RecordState.delivered) > 0) - delivered = runtime.backend.count(RecordState.delivered) + delivered = backend.count(RecordState.delivered) await runtime.stop() await server.stop() diff --git a/tests/test_layout.py b/tests/test_layout.py new file mode 100644 index 0000000..7113242 --- /dev/null +++ b/tests/test_layout.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from daena.config import ( + DaenaConfig, + MetricsConfig, + ProcessorConfig, + SinkConfig, + SourceConfig, +) +from daena.core.layout import preview_message, render_layout + + +def _config() -> DaenaConfig: + config = DaenaConfig() + config.persistence.lmdb.path = "/tmp/daena/data" + config.sources = [ + SourceConfig(name="dummy", type="dummy", options={"interval": 2.0}), + SourceConfig( + name="system", + type="journald", + enabled=False, + options={"units": ["ssh.service"]}, + ), + ] + config.processors = [ + ProcessorConfig(name="host", type="enricher", options={"static": {"env": "local"}}), + ProcessorConfig(name="keep", type="filter", options={"drop": {"body.priority": "7"}}), + ] + config.sinks = [ + SinkConfig(name="log", type="dummy"), + SinkConfig( + name="loki", + type="loki", + options={"url": "http://127.0.0.1:3100/loki/api/v1/push"}, + ), + ] + config.logging.verbose = True + return config + + +def test_layout_is_a_single_pipeline_line() -> None: + text = render_layout(_config(), "./daena.yaml") + assert "┌" not in text + assert "▼" not in text + flow = text.splitlines()[1] + assert flow.index("dummy") < flow.index("host") + assert flow.index("host") < flow.index("lmdb") + assert flow.index("lmdb") < flow.index("log") + assert "daena 0.1.0 ./daena.yaml" in text + assert "/tmp/daena/data" in text + assert "127.0.0.1:8642 127.0.0.1:8643" in text + + +def test_layout_names_each_plugin_once() -> None: + text = render_layout(_config(), None) + flow = text.splitlines()[1] + assert flow == " → ".join( + [ + "dummy (2s), system (journald, off)", + "host (enricher), keep (filter)", + "lmdb", + "log (dummy), loki (127.0.0.1:3100)", + ] + ) + assert "daena 0.1.0 defaults" in text + + +def test_layout_skips_empty_stages_and_disabled_metrics() -> None: + config = DaenaConfig() + config.api.metrics = MetricsConfig(enabled=False) + text = render_layout(config, None) + assert "→ lmdb" not in text + assert text.splitlines()[1] == "lmdb" + assert text.splitlines()[-1] == "127.0.0.1:8642" + + +def test_preview_message_uses_message_then_clips() -> None: + assert preview_message({"message": "daena heartbeat #1"}) == "daena heartbeat #1" + assert preview_message({"message": "a\nb"}) == "a b" + assert preview_message({"counter": 1}) == '{"counter":1}' + clipped = preview_message({"message": "x" * 130}, limit=10) + assert clipped.endswith("…") + assert len(clipped) == 10 diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..08f7cac --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from daena.api.metrics import MetricsServer, MetricsSnapshot, render_prometheus + + +def test_render_prometheus_includes_sink_state() -> None: + text = render_prometheus( + MetricsSnapshot( + running=True, + stored_records=3, + oldest_inflight_seconds=12, + deliveries={("central", "pending"): 2, ("central", "delivered"): 1}, + sink_results={"central": {"delivered": 4, "failed": 1, "dead": 0}}, + ) + ) + assert "daena_up 1" in text + assert "daena_records 3" in text + assert "daena_oldest_inflight_seconds 12" in text + assert 'daena_delivery_records{sink="central",state="pending"} 2' in text + assert 'daena_sink_deliveries_total{sink="central",result="failed"} 1' in text + + +@pytest.mark.asyncio +async def test_metrics_server_serves_text() -> None: + snapshot = MetricsSnapshot( + running=True, + stored_records=0, + oldest_inflight_seconds=0, + deliveries={}, + sink_results={}, + ) + server = MetricsServer("127.0.0.1", 0, lambda: snapshot, lambda: {"status": "ok"}) + await server.start() + assert server._server is not None + sockets = server._server.sockets + assert sockets is not None + port = sockets[0].getsockname()[1] + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n") + await writer.drain() + data = await reader.read() + writer.close() + await writer.wait_closed() + await server.stop() + assert b"200 OK" in data + assert b"daena_up 1" in data diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 7244e90..75580c2 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pathlib import Path + from daena.persistence.lmdb import LMDBBackend from daena.persistence.models import RecordState, StoredRecord @@ -105,3 +107,109 @@ def test_ready_for_retry(lmdb_backend: LMDBBackend, sample_record: StoredRecord) lmdb_backend.nack(sample_record.record_id, "retry later", retry_delay=0) ready = lmdb_backend.ready_for_retry(limit=10) assert len(ready) == 1 + + +def test_index_keys_do_not_accumulate( + lmdb_backend: LMDBBackend, sample_record: StoredRecord +) -> None: + lmdb_backend.put(sample_record) + assert lmdb_backend.index_count() == 1 + lmdb_backend.nack(sample_record.record_id, "later", retry_delay=0) + assert lmdb_backend.index_count() == 2 + lmdb_backend.ack(sample_record.record_id) + assert lmdb_backend.ready_for_retry() == [] + assert lmdb_backend.pending() == [] + assert lmdb_backend.count(RecordState.failed) == 0 + assert lmdb_backend.count(RecordState.delivered) == 1 + assert lmdb_backend.index_count() == 1 + + +def test_claim_and_recover(lmdb_backend: LMDBBackend, sample_record: StoredRecord) -> None: + lmdb_backend.put(sample_record) + claimed = lmdb_backend.claim("out", limit=10) + assert len(claimed) == 1 + assert claimed[0].deliveries["out"].state == RecordState.delivering + assert lmdb_backend.pending() == [] + assert lmdb_backend.recover_inflight() == 1 + pending = lmdb_backend.pending() + assert len(pending) == 1 + assert pending[0].deliveries["out"].state == RecordState.pending + + +def test_per_sink_ack_leaves_other_sink_pending(lmdb_backend: LMDBBackend) -> None: + record = StoredRecord(source="s", record_type="t", body={"n": 1}, sinks=["a", "b"]) + lmdb_backend.put(record) + lmdb_backend.ack(record.record_id, sink="a") + stored = lmdb_backend.get(record.record_id) + assert stored is not None + assert stored.deliveries["a"].state == RecordState.delivered + assert stored.deliveries["b"].state == RecordState.pending + assert lmdb_backend.count(RecordState.pending) == 1 + assert lmdb_backend.count(RecordState.delivered) == 1 + assert lmdb_backend.pending(sink="a") == [] + assert len(lmdb_backend.pending(sink="b")) == 1 + removed = lmdb_backend.delete_old("2100-01-01T00:00:00+00:00", state=RecordState.delivered) + assert removed == 0 + assert lmdb_backend.get(record.record_id) is not None + + +def test_replay_dead_becomes_pending( + lmdb_backend: LMDBBackend, sample_record: StoredRecord +) -> None: + lmdb_backend.put(sample_record) + lmdb_backend.dead(sample_record.record_id, error="nope") + replayed = lmdb_backend.replay(record_id=sample_record.record_id) + stored = lmdb_backend.get(sample_record.record_id) + assert replayed == 1 + assert stored is not None + assert stored.deliveries["out"].state == RecordState.pending + assert stored.deliveries["out"].retry_count == 0 + assert stored.deliveries["out"].last_error is None + + +def test_delete_expired_keeps_inflight(lmdb_backend: LMDBBackend) -> None: + record = StoredRecord(source="s", record_type="t", body={}, sinks=["a", "b"]) + lmdb_backend.put(record) + lmdb_backend.ack(record.record_id, sink="a") + lmdb_backend.dead(record.record_id, "boom", sink="b") + removed = lmdb_backend.delete_expired("2100-01-01T00:00:00+00:00", "2000-01-01T00:00:00+00:00") + assert removed == 0 + removed = lmdb_backend.delete_expired("2100-01-01T00:00:00+00:00", "2100-01-01T00:00:00+00:00") + assert removed == 1 + assert lmdb_backend.get(record.record_id) is None + + +def test_migrates_legacy_record(tmp_db_path: Path) -> None: + import json + + import lmdb + + tmp_db_path.mkdir(parents=True) + env = lmdb.open(str(tmp_db_path), map_size=10_485_760, subdir=True) + payload = { + "record_id": "legacy", + "source": "s", + "record_type": "t", + "body": {"message": "old"}, + "metadata": {}, + "state": "failed", + "created_at": "2020-01-01T00:00:00+00:00", + "updated_at": "2020-01-01T00:00:00+00:00", + "retry_count": 2, + "next_retry_at": "2020-01-01T00:01:00+00:00", + "last_error": "boom", + "destination": "central", + } + with env.begin(write=True) as txn: + txn.put(b"rec:legacy", json.dumps(payload).encode()) + env.close() + + backend = LMDBBackend(path=str(tmp_db_path), map_size=10_485_760) + backend.open() + stored = backend.get("legacy") + assert stored is not None + assert stored.deliveries["central"].state == RecordState.failed + assert stored.deliveries["central"].retry_count == 2 + assert stored.deliveries["central"].last_error == "boom" + assert backend.count(RecordState.failed) == 1 + backend.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..1142f63 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from daena.pipeline.processors.enricher import EnricherProcessor +from daena.pipeline.processors.filter import FilterProcessor +from daena.pipeline.record import Record +from daena.pipeline.sinks.loki import LokiSink +from daena.pipeline.sources.journald import ( + JournaldSource, + build_journalctl_command, + entry_to_record, +) + + +def test_journalctl_command_tails_new_entries_without_cursor() -> None: + command = build_journalctl_command(cursor=None, units=["ssh.service"], matches=["_PID=1"]) + assert command[:4] == ["journalctl", "--follow", "--output=json", "--no-pager"] + assert "--lines=0" in command + assert command[command.index("--unit") + 1] == "ssh.service" + assert "_PID=1" in command + + +def test_journalctl_command_uses_cursor() -> None: + command = build_journalctl_command(cursor="abc", units=[], matches=[]) + assert "--after-cursor=abc" in command + assert "--lines=0" not in command + + +def test_entry_to_record_uses_journal_timestamp() -> None: + record = entry_to_record( + "system", + { + "MESSAGE": "hello", + "SYSLOG_IDENTIFIER": "sshd", + "_SYSTEMD_UNIT": "ssh.service", + "__CURSOR": "cur", + "__REALTIME_TIMESTAMP": "1600000000000000", + }, + ) + assert record.source == "system" + assert record.record_type == "journald" + assert record.body["message"] == "hello" + assert record.body["syslog_identifier"] == "sshd" + assert record.metadata["unit"] == "ssh.service" + assert record.timestamp.startswith("2020-") + + +class _ListReader: + def __init__(self, entries: list[dict[str, Any]]) -> None: + self._entries = entries + self.stopped = False + + async def run(self, on_entry) -> None: # type: ignore[no-untyped-def] + for entry in self._entries: + await on_entry(entry) + + async def stop(self) -> None: + self.stopped = True + + +@pytest.mark.asyncio +async def test_journald_source_emits_and_stores_cursor(tmp_path: Path) -> None: + cursor_path = tmp_path / "cursor" + reader = _ListReader( + [ + {"MESSAGE": "one", "__CURSOR": "c1"}, + {"MESSAGE": "two", "__CURSOR": "c2"}, + ] + ) + source = JournaldSource( + "journal", + config={"cursor_path": str(cursor_path)}, + reader=reader, + ) + seen: list[Record] = [] + + async def capture(record: Record) -> None: + seen.append(record) + + source.set_callback(capture) + await source.start() + await asyncio.sleep(0.05) + await source.stop() + assert [record.body["message"] for record in seen] == ["one", "two"] + assert cursor_path.read_text(encoding="utf-8").strip() == "c2" + assert reader.stopped is True + + +@pytest.mark.asyncio +async def test_enricher_sets_hostname_and_static_fields() -> None: + processor = EnricherProcessor( + "host", + config={"hostname": True, "static": {"env": "prod"}, "target": "metadata"}, + ) + record = Record(source="s", record_type="t", body={"message": "m"}) + result = await processor.process(record) + assert result is not None + assert result.metadata["hostname"] + assert result.metadata["env"] == "prod" + + +@pytest.mark.asyncio +async def test_filter_drop_and_keep() -> None: + dropper = FilterProcessor("drop", config={"drop": {"body.level": "debug"}}) + debug = Record(source="s", record_type="t", body={"level": "debug"}) + info = Record(source="s", record_type="t", body={"level": "info"}) + assert await dropper.process(debug) is None + assert await dropper.process(info) is not None + + keeper = FilterProcessor("keep", config={"keep": {"source": "journal"}}) + assert await keeper.process(Record(source="other", record_type="t", body={})) is None + kept = await keeper.process(Record(source="journal", record_type="t", body={})) + assert kept is not None + + +@pytest.mark.asyncio +async def test_loki_sink_posts_stream() -> None: + response = MagicMock() + response.is_success = True + response.status_code = 204 + client = AsyncMock() + client.post = AsyncMock(return_value=response) + client.aclose = AsyncMock() + with patch("daena.pipeline.sinks.loki.httpx.AsyncClient", return_value=client): + sink = LokiSink("loki", config={"url": "http://loki/push", "labels": {"job": "daena"}}) + await sink.start() + record = Record( + source="system", + record_type="journald", + body={"message": "hello"}, + timestamp="2020-09-13T12:26:40+00:00", + ) + result = await sink.deliver(record) + await sink.stop() + assert result.success is True + payload = client.post.await_args.kwargs["json"] + stream = payload["streams"][0] + assert stream["stream"]["job"] == "daena" + assert stream["stream"]["source"] == "system" + assert stream["values"][0][1] == "hello" + assert stream["values"][0][0].isdigit() + json.dumps(payload)