feat: Enhance inference management with device tracking and telemetry updates

This commit is contained in:
2026-05-13 22:39:08 +02:00
parent 83346dc985
commit 6c401b62bb
7 changed files with 630 additions and 27 deletions

View File

@@ -5,8 +5,10 @@ safely be imported and executed in a child process via multiprocessing.
IPC protocol IPC protocol
------------ ------------
input_queue receives : FramePacket (frame_id, raw_bytes, width, height, channels) input_queue receives : FramePacket (frame_id, raw_bytes, width, height, channels)
output_queue sends : ResultPacket (frame_id, detections, width, height) output_queue sends : WorkerReadyPacket (device) — once after model load
: ResultPacket (frame_id, detections, width, height, elapsed_ms)
: None — on fatal load failure
stop_event : multiprocessing.Event — set by parent to request clean exit stop_event : multiprocessing.Event — set by parent to request clean exit
Detection format (namedtuple-compatible plain tuple): Detection format (namedtuple-compatible plain tuple):
@@ -37,6 +39,14 @@ class FramePacket(NamedTuple):
channels: int # always 3 (RGB) channels: int # always 3 (RGB)
class WorkerReadyPacket(NamedTuple):
"""
Sent once by the worker right after the model is loaded.
Carries the device string so the GUI can display it.
"""
device: str # e.g. "cpu", "mps"
class ResultPacket(NamedTuple): class ResultPacket(NamedTuple):
frame_id: int frame_id: int
detections: list # list of (x1, y1, x2, y2, conf, label) tuples detections: list # list of (x1, y1, x2, y2, conf, label) tuples
@@ -59,8 +69,9 @@ def run_worker(
""" """
Main loop of the inference worker process. Main loop of the inference worker process.
Loads the YOLO model once, then processes frames from input_queue Loads the YOLO model once, sends WorkerReadyPacket, then processes
until stop_event is set. Results are posted to output_queue. frames from input_queue until stop_event is set.
Results are posted to output_queue.
This function is designed to be the target of multiprocessing.Process. This function is designed to be the target of multiprocessing.Process.
It must NOT import PySide6 or any Qt module. It must NOT import PySide6 or any Qt module.
@@ -68,15 +79,21 @@ def run_worker(
_configure_worker_logging(log_level) _configure_worker_logging(log_level)
logger.info("Inference worker starting (pid=%d)", _getpid()) logger.info("Inference worker starting (pid=%d)", _getpid())
# Select device once — never changes during the lifetime of this process
device = _select_device()
try: try:
model = _load_model(model_path) model = _load_model(model_path, device)
except Exception as exc: except Exception as exc:
logger.error("Failed to load model '%s': %s", model_path, exc) logger.error("Failed to load model '%s': %s", model_path, exc)
# Signal failure by putting None — manager treats it as error # Signal failure by putting None — manager treats it as error
output_queue.put(None) output_queue.put(None)
return return
logger.info("Model loaded: %s", model_path) logger.info("Model loaded: %s device=%s", model_path, device)
# Notify GUI thread of the device being used
output_queue.put(WorkerReadyPacket(device=device))
while not stop_event.is_set(): while not stop_event.is_set():
try: try:
@@ -88,7 +105,7 @@ def run_worker(
break break
try: try:
result = _infer(model, packet) result = _infer(model, packet, device)
output_queue.put(result) output_queue.put(result)
except Exception as exc: except Exception as exc:
logger.error("Inference error (frame %d): %s", packet.frame_id, exc) logger.error("Inference error (frame %d): %s", packet.frame_id, exc)
@@ -107,11 +124,10 @@ def run_worker(
# Helpers # Helpers
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _load_model(model_path: str): def _load_model(model_path: str, device: str):
"""Load YOLO model with best available device.""" """Load YOLO model and warm up on the selected device."""
from ultralytics import YOLO # noqa: PLC0415 from ultralytics import YOLO # noqa: PLC0415
device = _select_device()
logger.info("Loading YOLO model on device='%s'", device) logger.info("Loading YOLO model on device='%s'", device)
model = YOLO(model_path) model = YOLO(model_path)
# Warm up — run on a tiny dummy to JIT-compile kernels # Warm up — run on a tiny dummy to JIT-compile kernels
@@ -126,11 +142,13 @@ def _load_model(model_path: str):
def _select_device() -> str: def _select_device() -> str:
""" """
Choose inference device. Choose the best available inference device.
Priority: Priority:
- macOS → "mps" if available (Metal GPU), else "cpu" - macOS → "mps" if torch.backends.mps.is_available(), else "cpu"
- others → "cpu" - others → "cpu"
Called once at worker startup — not per frame.
""" """
system = platform.system() system = platform.system()
if system == "Darwin": if system == "Darwin":
@@ -145,7 +163,7 @@ def _select_device() -> str:
return "cpu" return "cpu"
def _infer(model, packet: FramePacket) -> ResultPacket: def _infer(model, packet: FramePacket, device: str) -> ResultPacket:
"""Run model on one frame, return ResultPacket with elapsed_ms.""" """Run model on one frame, return ResultPacket with elapsed_ms."""
import time # noqa: PLC0415 import time # noqa: PLC0415
@@ -155,7 +173,6 @@ def _infer(model, packet: FramePacket) -> ResultPacket:
(packet.height, packet.width, packet.channels) (packet.height, packet.width, packet.channels)
) )
device = _select_device()
t0 = time.perf_counter() t0 = time.perf_counter()
results = model(frame_np, device=device, verbose=False) results = model(frame_np, device=device, verbose=False)
elapsed_ms = (time.perf_counter() - t0) * 1000.0 elapsed_ms = (time.perf_counter() - t0) * 1000.0

View File

@@ -5,11 +5,12 @@ Responsibilities:
- Submit frames (with drop-if-busy logic) - Submit frames (with drop-if-busy logic)
- Poll result queue via QTimer (never blocks the GUI thread) - Poll result queue via QTimer (never blocks the GUI thread)
- Watch process health via QTimer (auto-restart on crash) - Watch process health via QTimer (auto-restart on crash)
- Emit Qt signals with results for BboxOverlay - Emit Qt signals with results for BboxOverlay and TelemetryCollector
""" """
from __future__ import annotations from __future__ import annotations
import collections
import logging import logging
import multiprocessing import multiprocessing
import time import time
@@ -25,10 +26,13 @@ from app.config import (
INFERENCE_WORKER_TIMEOUT_S, INFERENCE_WORKER_TIMEOUT_S,
) )
from app.inference.bbox_overlay import Detection from app.inference.bbox_overlay import Detection
from app.inference.worker import FramePacket, ResultPacket, run_worker from app.inference.worker import FramePacket, ResultPacket, WorkerReadyPacket, run_worker
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Number of recent inference times to average for the overlay display
_ELAPSED_WINDOW = 10
class InferenceManager(QObject): class InferenceManager(QObject):
""" """
@@ -40,22 +44,26 @@ class InferenceManager(QObject):
detections : list[Detection] detections : list[Detection]
source_size : tuple[int, int] — (width, height) of inferred frame source_size : tuple[int, int] — (width, height) of inferred frame
detection_count_updated(int)
Total number of frames on which at least one detection occurred.
inference_stats_updated(device, avg_ms)
Emitted after every result packet.
device : str — e.g. "cpu", "mps"
avg_ms : float — rolling average of inference time (last 10 frames)
inference_device_changed(str)
Emitted once when the worker reports its device after model load.
inference_started() — worker is up and model is loaded inference_started() — worker is up and model is loaded
inference_stopped() — worker has exited cleanly inference_stopped() — worker has exited cleanly
inference_error(str) — fatal error (max restarts exceeded) inference_error(str) — fatal error (max restarts exceeded)
Usage:
mgr = InferenceManager(parent=self)
mgr.detections_ready.connect(bbox_overlay.on_detections)
mgr.start("path/to/model.pt")
# ...
mgr.submit_frame(video_frame) # called by FrameDispatcher subscriber
# ...
mgr.stop()
""" """
detections_ready = Signal(object, object) # list[Detection], tuple[int,int] detections_ready = Signal(object, object) # list[Detection], tuple[int,int]
detection_count_updated = Signal(int) # total frames with detections so far detection_count_updated = Signal(int) # total frames with detections so far
inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms
inference_device_changed = Signal(str) # emitted once on WorkerReadyPacket
inference_started = Signal() inference_started = Signal()
inference_stopped = Signal() inference_stopped = Signal()
inference_error = Signal(str) inference_error = Signal(str)
@@ -83,6 +91,14 @@ class InferenceManager(QObject):
# Detection counter — frames on which at least one detection occurred # Detection counter — frames on which at least one detection occurred
self._detection_frame_count: int = 0 self._detection_frame_count: int = 0
# Device reported by the worker after model load
self._current_device: str = "cpu"
# Rolling window of recent elapsed_ms values for averaging
self._elapsed_window: collections.deque[float] = collections.deque(
maxlen=_ELAPSED_WINDOW
)
# QTimers (GUI thread) # QTimers (GUI thread)
self._poll_timer = QTimer(self) self._poll_timer = QTimer(self)
self._poll_timer.setInterval(INFERENCE_POLL_INTERVAL_MS) self._poll_timer.setInterval(INFERENCE_POLL_INTERVAL_MS)
@@ -109,6 +125,8 @@ class InferenceManager(QObject):
self._restart_count = 0 self._restart_count = 0
self._paused = False self._paused = False
self._detection_frame_count = 0 self._detection_frame_count = 0
self._elapsed_window.clear()
self._current_device = "cpu"
self._start_worker() self._start_worker()
def stop(self) -> None: def stop(self) -> None:
@@ -140,6 +158,10 @@ class InferenceManager(QObject):
def model_path(self) -> str | None: def model_path(self) -> str | None:
return self._model_path return self._model_path
@property
def current_device(self) -> str:
return self._current_device
@Slot(QVideoFrame) @Slot(QVideoFrame)
def submit_frame(self, frame: QVideoFrame) -> None: def submit_frame(self, frame: QVideoFrame) -> None:
""" """
@@ -204,7 +226,6 @@ class InferenceManager(QObject):
try: try:
self._input_queue.put_nowait(packet) self._input_queue.put_nowait(packet)
self._busy = True self._busy = True
# logger.debug("InferenceManager: submitted frame %d", self._frame_id)
except Exception as exc: except Exception as exc:
logger.warning("InferenceManager: could not enqueue frame: %s", exc) logger.warning("InferenceManager: could not enqueue frame: %s", exc)
@@ -278,16 +299,33 @@ class InferenceManager(QObject):
try: try:
while True: while True:
item = self._output_queue.get_nowait() item = self._output_queue.get_nowait()
if item is None: if item is None:
# Worker signalled a fatal load error # Worker signalled a fatal load error
logger.error("Worker reported model load failure") logger.error("Worker reported model load failure")
self._handle_crash("Model failed to load in worker process") self._handle_crash("Model failed to load in worker process")
return return
# ----------------------------------------------------------
# WorkerReadyPacket — sent once after model load
# ----------------------------------------------------------
if isinstance(item, WorkerReadyPacket):
self._current_device = item.device
logger.info("Inference device: %s", item.device)
self.inference_device_changed.emit(item.device)
continue
# ----------------------------------------------------------
# ResultPacket — regular inference result
# ----------------------------------------------------------
packet: ResultPacket = item packet: ResultPacket = item
self._busy = False self._busy = False
self._last_result_time = time.monotonic() self._last_result_time = time.monotonic()
# Update rolling average of elapsed time
self._elapsed_window.append(packet.elapsed_ms)
avg_ms = sum(self._elapsed_window) / len(self._elapsed_window)
detections = [ detections = [
Detection(x1, y1, x2, y2, conf, label) Detection(x1, y1, x2, y2, conf, label)
for x1, y1, x2, y2, conf, label in packet.detections for x1, y1, x2, y2, conf, label in packet.detections
@@ -308,6 +346,8 @@ class InferenceManager(QObject):
) )
self.detection_count_updated.emit(self._detection_frame_count) self.detection_count_updated.emit(self._detection_frame_count)
# Always emit stats so overlay stays current
self.inference_stats_updated.emit(self._current_device, avg_ms)
self.detections_ready.emit(detections, source_size) self.detections_ready.emit(detections, source_size)
except Exception: except Exception:
@@ -340,7 +380,6 @@ class InferenceManager(QObject):
def _handle_crash(self, reason: str) -> None: def _handle_crash(self, reason: str) -> None:
"""Decide whether to auto-restart or give up.""" """Decide whether to auto-restart or give up."""
# Clean up process handles (already dead)
self._poll_timer.stop() self._poll_timer.stop()
self._watchdog_timer.stop() self._watchdog_timer.stop()
self._process = None self._process = None

View File

@@ -33,6 +33,8 @@ class TelemetryOverlay(IOverlayLayer):
CPU sys 14.8 % ← normalised by cpu_count (matches Task Manager) CPU sys 14.8 % ← normalised by cpu_count (matches Task Manager)
CPU core 118.4 % ← per single core (can exceed 100%) CPU core 118.4 % ← per single core (can exceed 100%)
Mem 68 MB Mem 68 MB
Inf.dev mps ← inference device (only when model loaded)
Inf.time 87 ms ← rolling average of model() call time
""" """
def __init__(self) -> None: def __init__(self) -> None:
@@ -106,4 +108,10 @@ class TelemetryOverlay(IOverlayLayer):
if snap.memory_mb is not None: if snap.memory_mb is not None:
lines.append(f"Mem {snap.memory_mb:>5.0f} MB") lines.append(f"Mem {snap.memory_mb:>5.0f} MB")
if snap.inference_device is not None:
lines.append(f"Inf.dev {snap.inference_device:>6s}")
if snap.inference_time_ms is not None:
lines.append(f"Inf.time {snap.inference_time_ms:>5.0f} ms")
return lines return lines

View File

@@ -26,6 +26,9 @@ class TelemetrySnapshot:
cpu_percent_core: float # process CPU per single core — can exceed 100% cpu_percent_core: float # process CPU per single core — can exceed 100%
memory_mb: float | None # process private working set in MB memory_mb: float | None # process private working set in MB
timestamp: float # time.perf_counter() when snapshot was taken timestamp: float # time.perf_counter() when snapshot was taken
# Inference fields — None when inference is disabled / model not loaded
inference_device: str | None = None # e.g. "cpu", "mps"
inference_time_ms: float | None = None # rolling average of model() call time
class TelemetryCollector(QObject): class TelemetryCollector(QObject):
@@ -69,6 +72,10 @@ class TelemetryCollector(QObject):
self._process.cpu_percent() # first call always returns 0.0; discard self._process.cpu_percent() # first call always returns 0.0; discard
self._cpu_count: int = max(psutil.cpu_count(logical=True) or 1, 1) self._cpu_count: int = max(psutil.cpu_count(logical=True) or 1, 1)
# Inference stats (updated externally via set_inference_stats)
self._inference_device: str | None = None
self._inference_time_ms: float | None = None
# periodic snapshot timer # periodic snapshot timer
self._timer = QTimer(self) self._timer = QTimer(self)
self._timer.setInterval(update_interval_ms) self._timer.setInterval(update_interval_ms)
@@ -85,6 +92,16 @@ class TelemetryCollector(QObject):
"""Record the FPS that was requested from the camera.""" """Record the FPS that was requested from the camera."""
self._target_fps = fps self._target_fps = fps
def set_inference_stats(self, device: str, avg_ms: float) -> None:
"""Update inference device and average inference time (called from MainWindow)."""
self._inference_device: str | None = device
self._inference_time_ms: float | None = avg_ms
def clear_inference_stats(self) -> None:
"""Clear inference stats when inference is disabled."""
self._inference_device = None
self._inference_time_ms = None
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Frame subscriber callback # Frame subscriber callback
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@@ -175,6 +192,12 @@ class TelemetryCollector(QObject):
cpu_percent_core=round(cpu_core, 1), cpu_percent_core=round(cpu_core, 1),
memory_mb=round(mem_mb, 1) if mem_mb is not None else None, memory_mb=round(mem_mb, 1) if mem_mb is not None else None,
timestamp=now, timestamp=now,
inference_device=self._inference_device,
inference_time_ms=(
round(self._inference_time_ms, 1)
if self._inference_time_ms is not None
else None
),
) )
def _make_empty_snapshot(self) -> TelemetrySnapshot: def _make_empty_snapshot(self) -> TelemetrySnapshot:
@@ -187,4 +210,6 @@ class TelemetryCollector(QObject):
cpu_percent_core=0.0, cpu_percent_core=0.0,
memory_mb=None, memory_mb=None,
timestamp=time.perf_counter(), timestamp=time.perf_counter(),
inference_device=None,
inference_time_ms=None,
) )

View File

@@ -184,6 +184,7 @@ class MainWindow(QMainWindow):
# ---- InferenceManager ---- # ---- InferenceManager ----
self._inference.detections_ready.connect(self._bbox_overlay.on_detections) self._inference.detections_ready.connect(self._bbox_overlay.on_detections)
self._inference.detection_count_updated.connect(self._on_detection_count_updated) self._inference.detection_count_updated.connect(self._on_detection_count_updated)
self._inference.inference_stats_updated.connect(self._on_inference_stats_updated)
self._inference.inference_started.connect(self._on_inference_started) self._inference.inference_started.connect(self._on_inference_started)
self._inference.inference_stopped.connect(self._on_inference_stopped) self._inference.inference_stopped.connect(self._on_inference_stopped)
self._inference.inference_error.connect(self._on_inference_error) self._inference.inference_error.connect(self._on_inference_error)
@@ -267,6 +268,9 @@ class MainWindow(QMainWindow):
def _on_detection_count_updated(self, count: int) -> None: def _on_detection_count_updated(self, count: int) -> None:
self._detection_label.setText(f"Detections: {count} frames") self._detection_label.setText(f"Detections: {count} frames")
def _on_inference_stats_updated(self, device: str, avg_ms: float) -> None:
self._telemetry.set_inference_stats(device, avg_ms)
def _on_inference_stopped(self) -> None: def _on_inference_stopped(self) -> None:
self._bbox_overlay.clear() self._bbox_overlay.clear()
@@ -276,6 +280,7 @@ class MainWindow(QMainWindow):
self._menu.set_inference_checked(False) self._menu.set_inference_checked(False)
self._bbox_overlay.visible = False self._bbox_overlay.visible = False
self._detection_label.setVisible(False) self._detection_label.setVisible(False)
self._telemetry.clear_inference_stats()
QMessageBox.critical(self, "Inference Error", message) QMessageBox.critical(self, "Inference Error", message)
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@@ -350,6 +355,7 @@ class MainWindow(QMainWindow):
self._bbox_overlay.clear() self._bbox_overlay.clear()
self._bbox_overlay.visible = False self._bbox_overlay.visible = False
self._detection_label.setVisible(False) self._detection_label.setVisible(False)
self._telemetry.clear_inference_stats()
self._status_label.setText("Inference disabled") self._status_label.setText("Inference disabled")
logger.info("Inference disabled") logger.info("Inference disabled")

505
notes/05-mvp-yolo.md Normal file
View File

@@ -0,0 +1,505 @@
# Stan projektu po sesji: YOLO inference + odtwarzanie wideo
Poprzedni stan: `04-mvp-uvc.md`
---
## Kontekst
Po uruchomieniu aplikacji na Mac Mini z kamerą ELP, kolejny krok to weryfikacja
wytrenowanego modelu YOLO. Wymagania:
- Model działa w **osobnym procesie** — crash workera nie wywala GUI
- Inference **nie blokuje** i **nie spowalnia** podglądu kamery
- Worker **ignoruje klatki** dopóki analizuje poprzednią (drop-if-busy)
- Możliwość wczytania **pliku wideo** zamiast kamery do oceny modelu
- Bbox narysowany na nowej **warstwie overlay**
- W przyszłości OCR będzie działał w tym samym procesie co YOLO
---
## Nowe pakiety
### `app/video/`
```
app/video/
├── __init__.py
└── video_player.py
```
### `app/inference/`
```
app/inference/
├── __init__.py
├── worker.py # funkcja uruchamiana w subprocess
├── worker_manager.py # InferenceManager (QObject) — IPC, polling, auto-restart
└── bbox_overlay.py # BboxOverlay(IOverlayLayer) — rysuje bbox+label+conf
```
---
## Szczegółowy opis zmian
### 1. `app/video/video_player.py` — `VideoPlayer`
Nowa klasa `VideoPlayer(QObject)` — identyczny interfejs sygnałowy jak `CameraService`:
```python
frame_ready = Signal(QVideoFrame)
playback_started = Signal()
playback_stopped = Signal()
playback_error = Signal(str)
```
Wewnętrznie: `QMediaPlayer` + `QVideoSink`. Obsługuje formaty: `.mp4`, `.avi`,
`.mov`, `.mkv`, `.m4v`, `.webm` (cokolwiek obsługuje FFmpeg backend Qt).
Odtwarzanie w czasie rzeczywistym (1×). Brak seek/pauzy — tylko Open + Stop.
`MainWindow` podłącza do `FrameDispatcher` albo `CameraService.frame_ready`,
albo `VideoPlayer.frame_ready` — nigdy obu naraz. Przełączanie przez
`_switch_to_camera()` / `_switch_to_video()`.
---
### 2. `app/inference/worker.py` — worker process
#### Struktury IPC (NamedTuple — pickle-safe)
```python
class FramePacket(NamedTuple):
frame_id: int
raw_bytes: bytes # RGB, (H×W×3)
width: int
height: int
channels: int # zawsze 3
class WorkerReadyPacket(NamedTuple):
device: str # "cpu" | "mps" — wysyłany raz po załadowaniu modelu
class ResultPacket(NamedTuple):
frame_id: int
detections: list # list of (x1, y1, x2, y2, conf, label)
width: int
height: int
elapsed_ms: float # czas wywołania model() w ms
```
#### Protokół IPC
```
input_queue ← FramePacket
output_queue → WorkerReadyPacket (raz, zaraz po załadowaniu modelu)
→ ResultPacket (po każdej analizowanej klatce)
→ None (tylko przy błędzie ładowania modelu)
```
#### `_select_device()` — wybór urządzenia
Wywoływany **raz przy starcie workera** (nie per-frame jak wcześniej):
```python
def _select_device() -> str:
if platform.system() == "Darwin":
if torch.backends.mps.is_available():
return "mps" # Metal GPU na macOS
return "cpu"
```
`device` jest przekazywany do `_load_model()` i do każdego wywołania `_infer()`.
Eliminuje redundantne wykrywanie urządzenia przy każdej klatce.
#### `_infer()` — pomiar czasu
```python
t0 = time.perf_counter()
results = model(frame_np, device=device, verbose=False)
elapsed_ms = (time.perf_counter() - t0) * 1000.0
```
`elapsed_ms` trafia do `ResultPacket` i jest logowany w managerze przy detekcjach.
---
### 3. `app/inference/worker_manager.py` — `InferenceManager`
#### Sygnały
```python
detections_ready = Signal(object, object) # list[Detection], tuple[int,int]
detection_count_updated = Signal(int) # łączna liczba klatek z detekcją
inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms
inference_device_changed = Signal(str) # emitowany raz po WorkerReadyPacket
inference_started = Signal()
inference_stopped = Signal()
inference_error = Signal(str)
```
#### Mechanizm drop-if-busy
```python
def submit_frame(self, frame: QVideoFrame) -> None:
if not self.is_running or self._paused or self._busy:
return # klatka odrzucona cicho
# konwersja + put_nowait → self._busy = True
```
`self._busy` wraca do `False` dopiero gdy `_poll_output()` odbierze `ResultPacket`.
Gwarantuje że nigdy nie ma więcej niż jedna klatka w locie.
#### Konwersja klatki w GUI thread
Zamiast `frame.bits(0)` (dawało tylko płaszczyznę Y dla NV12):
```python
image = frame.toImage() # Qt dekoduje NV12/YUV/MJPG → RGB
image = image.convertToFormat(Format_RGB32) # packed BGRX
arr = np.frombuffer(image.bits(), dtype=np.uint8).reshape((H, W, 4))
rgb = arr[:, :, [2, 1, 0]].copy() # BGRX → RGB, drop X
```
Obsługuje każdy pixel format jaki kamera może dostarczyć.
#### Rolling average elapsed_ms
```python
_elapsed_window: deque[float] # maxlen=10
avg_ms = sum(_elapsed_window) / len(_elapsed_window)
```
Emitowany przez `inference_stats_updated(device, avg_ms)` po każdym `ResultPacket`.
#### `_poll_output()` — obsługa `WorkerReadyPacket`
```python
if isinstance(item, WorkerReadyPacket):
self._current_device = item.device
self.inference_device_changed.emit(item.device)
continue
```
Odróżnienie od `ResultPacket` przez `isinstance` — nie wymaga sentinel wartości.
#### Auto-restart
- Watchdog co 2s sprawdza `process.is_alive()`
- Timeout 10s bez odpowiedzi → terminate + restart
- Max 3 restartów (konfigurowalny przez `INFERENCE_MAX_RESTARTS`)
- Po przekroczeniu: `QMessageBox.critical` + overlay wyłączony
#### Logowanie — tylko detekcje
```python
if detections:
logger.info(
"frame %d: %d detection(s) in %.1f ms — %s",
packet.frame_id, len(detections), packet.elapsed_ms, conf_summary,
)
```
Klatki bez detekcji: brak logu. `conf_summary = "label 0.94, label 0.81"`.
---
### 4. `app/inference/bbox_overlay.py` — `BboxOverlay`
```python
class Detection(NamedTuple):
x1: float; y1: float; x2: float; y2: float
conf: float
label: str
```
Współrzędne w pikselach **oryginalnej klatki**. `paint()` skaluje do `video_rect`:
```python
scale_x = video_rect.width() / src_w
scale_y = video_rect.height() / src_h
wx1 = video_rect.x() + int(det.x1 * scale_x)
# ...
```
Każdy bbox: prostokąt w kolorze `BBOX_COLOR` + label `"label 0.87"` na tle
`BBOX_LABEL_BG_COLOR` nad lewym górnym rogiem boxa (lub wewnątrz gdy brakuje miejsca).
`BboxOverlay.visible = False` domyślnie — pojawia się dopiero po włączeniu inference toggle.
---
### 5. Menu — zmiany w `app/ui/menu_bar.py`
Dodano dwa nowe menu przed istniejącymi:
```
File
├── Open Video… QFileDialog (.mp4 .avi .mov .mkv .m4v .webm)
└── Close Video disabled gdy źródło = kamera
Model
├── Load Model… QFileDialog (.pt .pth)
├── Enable Inference QAction checkable, disabled do momentu załadowania modelu
└── Model: (none) disabled — info o załadowanym pliku
```
Nowe sygnały:
- `video_file_selected(str)` — ścieżka pliku wideo
- `video_closed()` — powrót do kamery
- `model_file_selected(str)` — ścieżka modelu
- `inference_toggled(bool)` — włącz/wyłącz inference
---
### 6. `app/ui/main_window.py` — integracja
#### Przełączanie źródła klatek
```python
def _switch_to_camera(self):
video_player.frame_ready.disconnect(dispatcher.dispatch)
camera_service.frame_ready.connect(dispatcher.dispatch)
def _switch_to_video(self):
camera_service.frame_ready.disconnect(dispatcher.dispatch)
video_player.frame_ready.connect(dispatcher.dispatch)
```
Dispatcher i wszyscy subskrybenci (CameraView, TelemetryCollector,
InferenceManager) są podłączeni do dispatchera — źródło klatek jest dla nich
transparentne.
#### Inference toggle
```python
def _on_inference_toggled(self, enabled: bool):
if enabled:
inference.resume()
dispatcher.subscribe(inference.submit_frame, drop_if_busy=True)
bbox_overlay.visible = True
detection_label.setVisible(True)
else:
inference.pause()
dispatcher.unsubscribe(inference.submit_frame)
bbox_overlay.clear()
bbox_overlay.visible = False
detection_label.setVisible(False)
telemetry.clear_inference_stats()
```
`pause()` nie zatrzymuje procesu — tylko blokuje `submit_frame`. Proces
pozostaje załadowany w pamięci.
#### Status bar — counter detekcji
```python
self._detection_label = QLabel("") # addPermanentWidget (prawa strona)
```
Pokazywany tylko gdy inference włączone. Aktualizowany przez
`inference.detection_count_updated(int)``"Detections: 17 frames"`.
---
### 7. Telemetria + overlay — nowe pola inference
#### `TelemetrySnapshot` — nowe pola
```python
@dataclass
class TelemetrySnapshot:
# ... istniejące pola ...
inference_device: str | None = None # "cpu" | "mps" | None
inference_time_ms: float | None = None # rolling avg, None gdy wyłączone
```
#### `TelemetryCollector` — nowe metody
```python
def set_inference_stats(self, device: str, avg_ms: float) -> None: ...
def clear_inference_stats(self) -> None: ...
```
Wywoływane z `MainWindow` przy każdym `inference_stats_updated` i przy
wyłączeniu inference.
#### `TelemetryOverlay` — nowe wiersze
```
FPS req 30.0
FPS got 29.8
Frame 33.5 ms
Drop 0
CPU sys 8.2 %
CPU core 65.7 %
Mem 71 MB
Inf.dev mps ← widoczny tylko gdy model załadowany
Inf.time 87 ms ← rolling avg ostatnich 10 klatek
```
Wiersze `Inf.dev` i `Inf.time` znikają gdy inference jest wyłączone
(`clear_inference_stats()` → pola `None``_format_lines` ich nie emituje).
---
### 8. Bugfixes (zidentyfikowane po uruchomieniu)
#### `Unexpected frame size: 921600 bytes for 1280×720`
**Przyczyna:** `frame.bits(0)` zwraca tylko płaszczyznę 0 (luma Y) dla formatów
planarnych NV12/YUV420P — `1280 × 720 × 1 = 921600` zamiast `1280 × 720 × 3`.
**Naprawa:** zamiana na `frame.toImage() → Format_RGB32 → bits()`. Qt dekoduje
każdy format wewnętrznie. Identyczna ścieżka jak `CameraView.on_frame()`.
#### `Subscriber not found for removal` (WARNING w logu)
**Przyczyna:** `_on_inference_toggled(False)` wywoływał `dispatcher.unsubscribe()`
zanim subscriber był dodany (pierwsze wyłączenie przed włączeniem).
**Naprawa:** zmiana poziomu logu z `WARNING` na `DEBUG` w
`FrameDispatcher.unsubscribe()` — brak subscribera nie jest błędem.
---
## Decyzje architektoniczne
### Osobny proces zamiast wątku
`multiprocessing.Process(context="spawn")` zamiast `QThread` lub `threading.Thread`:
- Osobny GIL — inference nie blokuje Python event loop GUI
- Crash workera (segfault, OOM) nie wywala aplikacji
- `spawn` zamiast `fork` — wymagane na macOS od Python 3.12 (Apple deprecuje `fork`)
### `toImage()` zamiast `bits(0)`
`QVideoFrame.bits(plane)` daje surowe bajty jednej płaszczyzny. W formatach
planarnych (NV12: Y w plane 0, UV w plane 1) to tylko część obrazu. `toImage()`
wywołuje wewnętrzny dekoder Qt i zawsze zwraca kompletny obraz niezależnie od
pixel formatu.
### `WorkerReadyPacket` zamiast osobnego IPC kanału
Worker wysyła `WorkerReadyPacket(device)` do tej samej `output_queue` zaraz
po załadowaniu modelu. Manager odróżnia go przez `isinstance`. Eliminuje
potrzebę dodatkowej kolejki lub pipe tylko dla metadanych startu.
### Inference stats przez `TelemetryCollector`, nie bezpośrednio do overlay
`InferenceManager.inference_stats_updated``MainWindow._on_inference_stats_updated`
`TelemetryCollector.set_inference_stats()``TelemetrySnapshot.inference_*`
`TelemetryOverlay.paint()`.
Alternatywa: bezpośrednie połączenie `InferenceManager → TelemetryOverlay`.
Wybrano pośrednie przez `TelemetryCollector` bo:
- `TelemetrySnapshot` jest jedyną strukturą danych opisującą stan systemu
- CSV logger automatycznie dostaje inference stats bez dodatkowego kodu
- Overlay ma jeden spójny model danych
---
## Pliki dodane
| Plik | Zawartość |
|---|---|
| `app/video/__init__.py` | pusty |
| `app/video/video_player.py` | `VideoPlayer(QObject)` |
| `app/inference/__init__.py` | pusty |
| `app/inference/worker.py` | `run_worker()`, `FramePacket`, `WorkerReadyPacket`, `ResultPacket`, `_select_device()`, `_infer()` |
| `app/inference/worker_manager.py` | `InferenceManager(QObject)` |
| `app/inference/bbox_overlay.py` | `Detection(NamedTuple)`, `BboxOverlay(IOverlayLayer)` |
| `tests/test_bbox_overlay.py` | 16 testów — `Detection`, stan overlay, mapowanie współrzędnych bbox |
| `tests/test_inference_manager.py` | 13 testów — drop-if-busy, pause/resume, restart counter, is_running |
## Pliki zmienione
| Plik | Co zmieniono |
|---|---|
| `app/config.py` | `INFERENCE_WORKER_TIMEOUT_S`, `INFERENCE_MAX_RESTARTS`, `INFERENCE_POLL_INTERVAL_MS`, `INFERENCE_WATCHDOG_INTERVAL_MS`, `BBOX_COLOR`, `BBOX_LABEL_BG_COLOR`, `BBOX_LABEL_TEXT_COLOR`, `BBOX_LINE_WIDTH`, `BBOX_FONT_SIZE`, `VIDEO_FILE_EXTENSIONS`, `MODEL_FILE_EXTENSIONS` |
| `app/ui/menu_bar.py` | Menu `File` (Open Video…, Close Video), menu `Model` (Load Model…, Enable Inference, Model info) |
| `app/ui/main_window.py` | `VideoPlayer` lifecycle, `InferenceManager` lifecycle, source switching, detection counter w statusbar, `_on_inference_stats_updated` |
| `app/telemetry/telemetry_collector.py` | `TelemetrySnapshot.inference_device`, `TelemetrySnapshot.inference_time_ms`, `set_inference_stats()`, `clear_inference_stats()` |
| `app/overlay/telemetry_overlay.py` | Wiersze `Inf.dev` i `Inf.time` w `_format_lines()` |
| `app/pipeline/frame_dispatcher.py` | `unsubscribe()` brak subscribera: WARNING → DEBUG |
| `pyproject.toml` | `[project.optional-dependencies] inference = ["ultralytics>=8.0", "numpy>=1.24"]` |
| `tests/test_telemetry_collector.py` | `_make_collector()` uzupełniony o `_inference_device=None`, `_inference_time_ms=None` |
---
## Łączna liczba testów
**69 testów, wszystkie zielone.**
| Plik | Liczba testów |
|---|---|
| `test_frame_dispatcher.py` | 8 |
| `test_telemetry_collector.py` | 12 |
| `test_uvc.py` | 15 |
| `test_bbox_overlay.py` | 16 |
| `test_inference_manager.py` | 18 |
---
## Instalacja
```bash
# Wymagane do inference:
pip install ultralytics numpy
# lub:
pip install -e ".[inference]"
```
Aplikacja startuje bez tych pakietów — `Load Model…` zostaje aktywne, ale
`InferenceManager.start()` zgłosi błąd jeśli `ultralytics` nie jest zainstalowany
(obsłużony przez `try/except ImportError` w workerze → `output_queue.put(None)`
→ manager emituje `inference_error`).
---
## Uruchamianie
```bash
# Windows dev
.venv-win\Scripts\python.exe -m app.main
# Mac Mini
.venv/bin/python -m app.main
# Mac Mini z plikiem wideo od razu (CLI nie zaimplementowany — użyj File → Open Video…)
.venv/bin/python -m app.main
```
---
## Next Steps
1. Przetestować na Mac Mini z kamerą ELP:
- czy `_select_device()` wykrywa MPS i loguje `"MPS (Metal) available"`
- czy `Inf.dev mps` pojawia się w overlayzie
- czy `Inf.time` jest znacząco niższy niż na CPU
2. OCR w tym samym procesie co YOLO:
- Worker process może obsługiwać wiele zadań — dodać `OcrTask` do `FramePacket`
- lub uruchomić OCR jako osobny subscriber `FrameDispatcher` w osobnym procesie
3. Dodać możliwość regulacji progu confidence (`conf_threshold`) przez menu/dialog
— przekazać jako parametr do `run_worker()` w `FramePacket` lub przy starcie
4. `set_active_format()` call po `_log_actual_format()` żeby menu zaznaczało
faktycznie działający format (nie żądany) — z poprzedniej sesji
---
## Critical Context
- `WorkerReadyPacket` jest rozróżniany od `ResultPacket` przez `isinstance`
nie używaj `None` jako sentinela dla obu typów
- `_select_device()` wywołany raz przy starcie — jeśli zmienisz device w trakcie
działania, trzeba zrestartować workera
- `BboxOverlay.on_detections(detections, source_size)``source_size` to
`tuple[int, int]` (width, height) klatki która była inferowana, nie aktualnego
widgetu; potrzebne do poprawnego skalowania przy zmianie rozdzielczości
- `InferenceManager.pause()` nie zatrzymuje procesu — `submit_frame` tylko
sprawdza flagę; model pozostaje załadowany, można szybko wznowić
- `multiprocessing.get_context("spawn")` — wymagane na macOS/Windows;
`fork` jest domyślny na Linux ale niebezpieczny z Qt

View File

@@ -33,6 +33,9 @@ class TestTelemetryCollector:
mem_info.rss = 70 * 1024 * 1024 # RSS (larger, includes shared) mem_info.rss = 70 * 1024 * 1024 # RSS (larger, includes shared)
col._process.memory_info.return_value = mem_info col._process.memory_info.return_value = mem_info
col._process.cpu_percent.return_value = 0.0 col._process.cpu_percent.return_value = 0.0
# Inference stats — None by default (inference disabled)
col._inference_device = None
col._inference_time_ms = None
return col return col
def test_initial_snapshot_has_zero_fps(self): def test_initial_snapshot_has_zero_fps(self):