feat(m8): event-based alerts — Xid/OOM/MCE/PCIe/disk from the kernel log — 0.34.0
AlertMonitor now scans the kernel log (journalctl -k) every ~30s and fires one-shot, cooldown-gated desktop alerts on critical events: NVIDIA Xid, OOM kills, CPU machine-checks, PCIe AER, and disk I/O errors — so users are warned the moment something goes wrong, not only on a temperature threshold. Disk I/O errors come from the kernel log (no root needed, unlike smartctl). Edge/spam protection reuses the existing cooldown model. syslogs.scan_critical() does the matching; init seeds last-scan to "now" so old boot logs don't alert on launch. Tests for the matcher + monitor gating/cooldown; Settings note updated. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,3 @@
|
||||
"""RigDoctor — modular hardware monitoring & crash diagnostics for Linux gamers."""
|
||||
|
||||
__version__ = "0.33.0"
|
||||
__version__ = "0.34.0"
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Desktop alerts (M8): notify on overheat / GPU-lost / new version via notify-send.
|
||||
"""Desktop alerts (M8): notify on overheat / GPU-lost / critical kernel events / new version.
|
||||
|
||||
Edge-triggered: an alert fires when a condition becomes true (not every sample), and
|
||||
can fire again only after it has cleared and a cooldown has passed — so a hot GPU or a
|
||||
1-Hz sample loop doesn't spam notifications. Degrades to a no-op if notify-send is absent.
|
||||
Edge-triggered: a sustained condition (hot GPU, GPU-lost) fires once when it becomes true and
|
||||
can re-fire only after it clears + a cooldown; momentary **kernel events** (Xid, OOM-kill, MCE,
|
||||
PCIe AER, disk I/O errors) are scanned from the kernel log every `event_interval` seconds and
|
||||
fire one-shot (cooldown-gated). So a 1-Hz sample loop never spams. No-op if notify-send absent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -57,13 +58,16 @@ def notify(title: str, message: str, urgency: str = "normal") -> bool:
|
||||
class AlertMonitor:
|
||||
"""Evaluate samples and raise edge-triggered desktop alerts."""
|
||||
|
||||
def __init__(self, gpu_temp: float = 90.0, cpu_temp: float = 95.0, cooldown: float = 300.0):
|
||||
def __init__(self, gpu_temp: float = 90.0, cpu_temp: float = 95.0, cooldown: float = 300.0,
|
||||
event_interval: float = 30.0):
|
||||
self.gpu_temp = gpu_temp
|
||||
self.cpu_temp = cpu_temp
|
||||
self.cooldown = cooldown
|
||||
self.event_interval = event_interval # how often to scan the kernel log
|
||||
self.enabled = True
|
||||
self._active: dict[str, bool] = {}
|
||||
self._last: dict[str, float] = {}
|
||||
self._last_kernel_scan = time.time() # only alert on events after the monitor starts
|
||||
|
||||
def _fire(self, key: str, title: str, message: str, urgency: str = "critical") -> None:
|
||||
if self._active.get(key):
|
||||
@@ -75,9 +79,39 @@ class AlertMonitor:
|
||||
self._last[key] = now
|
||||
notify(title, message, urgency)
|
||||
|
||||
def _notify_once(self, key: str, title: str, message: str, urgency: str = "critical") -> None:
|
||||
"""One-shot alert for a momentary event (cooldown-gated, no active latch)."""
|
||||
now = time.time()
|
||||
if now - self._last.get(key, 0.0) < self.cooldown:
|
||||
return
|
||||
self._last[key] = now
|
||||
notify(title, message, urgency)
|
||||
|
||||
def _clear(self, key: str) -> None:
|
||||
self._active[key] = False
|
||||
|
||||
def _scan_kernel_events(self) -> None:
|
||||
"""Periodically scan the kernel log for new critical events (Xid/OOM/MCE/PCIe/disk)."""
|
||||
now = time.time()
|
||||
if now - self._last_kernel_scan < self.event_interval:
|
||||
return
|
||||
since = self._last_kernel_scan
|
||||
self._last_kernel_scan = now
|
||||
try:
|
||||
from . import syslogs
|
||||
|
||||
text = syslogs.kernel_log(since=since)
|
||||
except Exception: # alerting must never crash the sample loop
|
||||
return
|
||||
if not text:
|
||||
return
|
||||
seen: set[str] = set()
|
||||
for label, line in syslogs.scan_critical(text):
|
||||
if label in seen: # one alert per category per scan
|
||||
continue
|
||||
seen.add(label)
|
||||
self._notify_once(f"kernel:{label}", label, line[:180])
|
||||
|
||||
def check(self, sample: Sample) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
@@ -107,3 +141,5 @@ class AlertMonitor:
|
||||
self._fire("gpu_lost", "GPU not responding", "nvidia-smi query timed out — the GPU may have dropped")
|
||||
else:
|
||||
self._clear("gpu_lost")
|
||||
|
||||
self._scan_kernel_events() # Xid / OOM / MCE / PCIe / disk I/O from the kernel log
|
||||
|
||||
@@ -13,6 +13,7 @@ Best-effort and size-bounded: degrades silently if a tool is missing or access i
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
@@ -118,6 +119,29 @@ def display_log(since: float | None = None, max_bytes: int = _MAX) -> str:
|
||||
return _tail_file(log, max_bytes) if log else ""
|
||||
|
||||
|
||||
# Kernel-log patterns worth alerting on in real time (M8 event alerts). (label, regex).
|
||||
_CRITICAL = [
|
||||
("GPU error (Xid)", re.compile(r"NVRM:\s*Xid", re.I)),
|
||||
("Out of memory", re.compile(r"out of memory|oom-kill|killed process \d+", re.I)),
|
||||
("CPU machine-check", re.compile(r"\bmce:|machine check", re.I)),
|
||||
("PCIe error", re.compile(r"\bAER:|pcie bus error", re.I)),
|
||||
("Disk I/O error", re.compile(
|
||||
r"buffer i/o error|\bi/o error\b|critical medium error|ext4-fs error|"
|
||||
r"blk_update_request:.*error|ata\d+.*(?:failed|error)", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def scan_critical(text: str) -> list[tuple[str, str]]:
|
||||
"""(label, line) for kernel lines matching a critical pattern (first match per line)."""
|
||||
events: list[tuple[str, str]] = []
|
||||
for line in text.splitlines():
|
||||
for label, pat in _CRITICAL:
|
||||
if pat.search(line):
|
||||
events.append((label, line.strip()))
|
||||
break
|
||||
return events
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
return bool(shutil.which("journalctl") or shutil.which("coredumpctl")
|
||||
or shutil.which("nvidia-smi") or _xorg_log())
|
||||
|
||||
@@ -114,7 +114,8 @@ class SetupPage(QWidget):
|
||||
grid.addWidget(QLabel("CPU temperature alert"), 1, 0)
|
||||
grid.addWidget(self._cpu_alert, 1, 1)
|
||||
alerts_layout.addLayout(grid)
|
||||
alerts_note = QLabel("GPU-lost and new-version alerts are included whenever notifications are enabled.")
|
||||
alerts_note = QLabel("GPU-lost, critical kernel events (Xid, out-of-memory, disk I/O, PCIe), "
|
||||
"and new-version alerts are included whenever notifications are enabled.")
|
||||
alerts_note.setObjectName("Muted")
|
||||
alerts_note.setWordWrap(True)
|
||||
alerts_layout.addWidget(alerts_note)
|
||||
|
||||
Reference in New Issue
Block a user