feat: Enhance inference management with device tracking and telemetry updates
This commit is contained in:
@@ -6,7 +6,9 @@ safely be imported and executed in a child process via multiprocessing.
|
||||
IPC protocol
|
||||
------------
|
||||
input_queue receives : FramePacket (frame_id, raw_bytes, width, height, channels)
|
||||
output_queue sends : ResultPacket (frame_id, detections, width, height)
|
||||
output_queue sends : WorkerReadyPacket (device) — once after model load
|
||||
: ResultPacket (frame_id, detections, width, height, elapsed_ms)
|
||||
: None — on fatal load failure
|
||||
stop_event : multiprocessing.Event — set by parent to request clean exit
|
||||
|
||||
Detection format (namedtuple-compatible plain tuple):
|
||||
@@ -37,6 +39,14 @@ class FramePacket(NamedTuple):
|
||||
channels: int # always 3 (RGB)
|
||||
|
||||
|
||||
class WorkerReadyPacket(NamedTuple):
|
||||
"""
|
||||
Sent once by the worker right after the model is loaded.
|
||||
Carries the device string so the GUI can display it.
|
||||
"""
|
||||
device: str # e.g. "cpu", "mps"
|
||||
|
||||
|
||||
class ResultPacket(NamedTuple):
|
||||
frame_id: int
|
||||
detections: list # list of (x1, y1, x2, y2, conf, label) tuples
|
||||
@@ -59,8 +69,9 @@ def run_worker(
|
||||
"""
|
||||
Main loop of the inference worker process.
|
||||
|
||||
Loads the YOLO model once, then processes frames from input_queue
|
||||
until stop_event is set. Results are posted to output_queue.
|
||||
Loads the YOLO model once, sends WorkerReadyPacket, then processes
|
||||
frames from input_queue until stop_event is set.
|
||||
Results are posted to output_queue.
|
||||
|
||||
This function is designed to be the target of multiprocessing.Process.
|
||||
It must NOT import PySide6 or any Qt module.
|
||||
@@ -68,15 +79,21 @@ def run_worker(
|
||||
_configure_worker_logging(log_level)
|
||||
logger.info("Inference worker starting (pid=%d)", _getpid())
|
||||
|
||||
# Select device once — never changes during the lifetime of this process
|
||||
device = _select_device()
|
||||
|
||||
try:
|
||||
model = _load_model(model_path)
|
||||
model = _load_model(model_path, device)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to load model '%s': %s", model_path, exc)
|
||||
# Signal failure by putting None — manager treats it as error
|
||||
output_queue.put(None)
|
||||
return
|
||||
|
||||
logger.info("Model loaded: %s", model_path)
|
||||
logger.info("Model loaded: %s device=%s", model_path, device)
|
||||
|
||||
# Notify GUI thread of the device being used
|
||||
output_queue.put(WorkerReadyPacket(device=device))
|
||||
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
@@ -88,7 +105,7 @@ def run_worker(
|
||||
break
|
||||
|
||||
try:
|
||||
result = _infer(model, packet)
|
||||
result = _infer(model, packet, device)
|
||||
output_queue.put(result)
|
||||
except Exception as exc:
|
||||
logger.error("Inference error (frame %d): %s", packet.frame_id, exc)
|
||||
@@ -107,11 +124,10 @@ def run_worker(
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_model(model_path: str):
|
||||
"""Load YOLO model with best available device."""
|
||||
def _load_model(model_path: str, device: str):
|
||||
"""Load YOLO model and warm up on the selected device."""
|
||||
from ultralytics import YOLO # noqa: PLC0415
|
||||
|
||||
device = _select_device()
|
||||
logger.info("Loading YOLO model on device='%s'", device)
|
||||
model = YOLO(model_path)
|
||||
# Warm up — run on a tiny dummy to JIT-compile kernels
|
||||
@@ -126,11 +142,13 @@ def _load_model(model_path: str):
|
||||
|
||||
def _select_device() -> str:
|
||||
"""
|
||||
Choose inference device.
|
||||
Choose the best available inference device.
|
||||
|
||||
Priority:
|
||||
- macOS → "mps" if available (Metal GPU), else "cpu"
|
||||
- macOS → "mps" if torch.backends.mps.is_available(), else "cpu"
|
||||
- others → "cpu"
|
||||
|
||||
Called once at worker startup — not per frame.
|
||||
"""
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
@@ -145,7 +163,7 @@ def _select_device() -> str:
|
||||
return "cpu"
|
||||
|
||||
|
||||
def _infer(model, packet: FramePacket) -> ResultPacket:
|
||||
def _infer(model, packet: FramePacket, device: str) -> ResultPacket:
|
||||
"""Run model on one frame, return ResultPacket with elapsed_ms."""
|
||||
import time # noqa: PLC0415
|
||||
|
||||
@@ -155,7 +173,6 @@ def _infer(model, packet: FramePacket) -> ResultPacket:
|
||||
(packet.height, packet.width, packet.channels)
|
||||
)
|
||||
|
||||
device = _select_device()
|
||||
t0 = time.perf_counter()
|
||||
results = model(frame_np, device=device, verbose=False)
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
|
||||
@@ -5,11 +5,12 @@ Responsibilities:
|
||||
- Submit frames (with drop-if-busy logic)
|
||||
- Poll result queue via QTimer (never blocks the GUI thread)
|
||||
- Watch process health via QTimer (auto-restart on crash)
|
||||
- Emit Qt signals with results for BboxOverlay
|
||||
- Emit Qt signals with results for BboxOverlay and TelemetryCollector
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import multiprocessing
|
||||
import time
|
||||
@@ -25,10 +26,13 @@ from app.config import (
|
||||
INFERENCE_WORKER_TIMEOUT_S,
|
||||
)
|
||||
from app.inference.bbox_overlay import Detection
|
||||
from app.inference.worker import FramePacket, ResultPacket, run_worker
|
||||
from app.inference.worker import FramePacket, ResultPacket, WorkerReadyPacket, run_worker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Number of recent inference times to average for the overlay display
|
||||
_ELAPSED_WINDOW = 10
|
||||
|
||||
|
||||
class InferenceManager(QObject):
|
||||
"""
|
||||
@@ -40,22 +44,26 @@ class InferenceManager(QObject):
|
||||
detections : list[Detection]
|
||||
source_size : tuple[int, int] — (width, height) of inferred frame
|
||||
|
||||
detection_count_updated(int)
|
||||
Total number of frames on which at least one detection occurred.
|
||||
|
||||
inference_stats_updated(device, avg_ms)
|
||||
Emitted after every result packet.
|
||||
device : str — e.g. "cpu", "mps"
|
||||
avg_ms : float — rolling average of inference time (last 10 frames)
|
||||
|
||||
inference_device_changed(str)
|
||||
Emitted once when the worker reports its device after model load.
|
||||
|
||||
inference_started() — worker is up and model is loaded
|
||||
inference_stopped() — worker has exited cleanly
|
||||
inference_error(str) — fatal error (max restarts exceeded)
|
||||
|
||||
Usage:
|
||||
mgr = InferenceManager(parent=self)
|
||||
mgr.detections_ready.connect(bbox_overlay.on_detections)
|
||||
mgr.start("path/to/model.pt")
|
||||
# ...
|
||||
mgr.submit_frame(video_frame) # called by FrameDispatcher subscriber
|
||||
# ...
|
||||
mgr.stop()
|
||||
"""
|
||||
|
||||
detections_ready = Signal(object, object) # list[Detection], tuple[int,int]
|
||||
detection_count_updated = Signal(int) # total frames with detections so far
|
||||
inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms
|
||||
inference_device_changed = Signal(str) # emitted once on WorkerReadyPacket
|
||||
inference_started = Signal()
|
||||
inference_stopped = Signal()
|
||||
inference_error = Signal(str)
|
||||
@@ -83,6 +91,14 @@ class InferenceManager(QObject):
|
||||
# Detection counter — frames on which at least one detection occurred
|
||||
self._detection_frame_count: int = 0
|
||||
|
||||
# Device reported by the worker after model load
|
||||
self._current_device: str = "cpu"
|
||||
|
||||
# Rolling window of recent elapsed_ms values for averaging
|
||||
self._elapsed_window: collections.deque[float] = collections.deque(
|
||||
maxlen=_ELAPSED_WINDOW
|
||||
)
|
||||
|
||||
# QTimers (GUI thread)
|
||||
self._poll_timer = QTimer(self)
|
||||
self._poll_timer.setInterval(INFERENCE_POLL_INTERVAL_MS)
|
||||
@@ -109,6 +125,8 @@ class InferenceManager(QObject):
|
||||
self._restart_count = 0
|
||||
self._paused = False
|
||||
self._detection_frame_count = 0
|
||||
self._elapsed_window.clear()
|
||||
self._current_device = "cpu"
|
||||
self._start_worker()
|
||||
|
||||
def stop(self) -> None:
|
||||
@@ -140,6 +158,10 @@ class InferenceManager(QObject):
|
||||
def model_path(self) -> str | None:
|
||||
return self._model_path
|
||||
|
||||
@property
|
||||
def current_device(self) -> str:
|
||||
return self._current_device
|
||||
|
||||
@Slot(QVideoFrame)
|
||||
def submit_frame(self, frame: QVideoFrame) -> None:
|
||||
"""
|
||||
@@ -204,7 +226,6 @@ class InferenceManager(QObject):
|
||||
try:
|
||||
self._input_queue.put_nowait(packet)
|
||||
self._busy = True
|
||||
# logger.debug("InferenceManager: submitted frame %d", self._frame_id)
|
||||
except Exception as exc:
|
||||
logger.warning("InferenceManager: could not enqueue frame: %s", exc)
|
||||
|
||||
@@ -278,16 +299,33 @@ class InferenceManager(QObject):
|
||||
try:
|
||||
while True:
|
||||
item = self._output_queue.get_nowait()
|
||||
|
||||
if item is None:
|
||||
# Worker signalled a fatal load error
|
||||
logger.error("Worker reported model load failure")
|
||||
self._handle_crash("Model failed to load in worker process")
|
||||
return
|
||||
|
||||
# ----------------------------------------------------------
|
||||
# WorkerReadyPacket — sent once after model load
|
||||
# ----------------------------------------------------------
|
||||
if isinstance(item, WorkerReadyPacket):
|
||||
self._current_device = item.device
|
||||
logger.info("Inference device: %s", item.device)
|
||||
self.inference_device_changed.emit(item.device)
|
||||
continue
|
||||
|
||||
# ----------------------------------------------------------
|
||||
# ResultPacket — regular inference result
|
||||
# ----------------------------------------------------------
|
||||
packet: ResultPacket = item
|
||||
self._busy = False
|
||||
self._last_result_time = time.monotonic()
|
||||
|
||||
# Update rolling average of elapsed time
|
||||
self._elapsed_window.append(packet.elapsed_ms)
|
||||
avg_ms = sum(self._elapsed_window) / len(self._elapsed_window)
|
||||
|
||||
detections = [
|
||||
Detection(x1, y1, x2, y2, conf, label)
|
||||
for x1, y1, x2, y2, conf, label in packet.detections
|
||||
@@ -308,6 +346,8 @@ class InferenceManager(QObject):
|
||||
)
|
||||
self.detection_count_updated.emit(self._detection_frame_count)
|
||||
|
||||
# Always emit stats so overlay stays current
|
||||
self.inference_stats_updated.emit(self._current_device, avg_ms)
|
||||
self.detections_ready.emit(detections, source_size)
|
||||
|
||||
except Exception:
|
||||
@@ -340,7 +380,6 @@ class InferenceManager(QObject):
|
||||
|
||||
def _handle_crash(self, reason: str) -> None:
|
||||
"""Decide whether to auto-restart or give up."""
|
||||
# Clean up process handles (already dead)
|
||||
self._poll_timer.stop()
|
||||
self._watchdog_timer.stop()
|
||||
self._process = None
|
||||
|
||||
@@ -33,6 +33,8 @@ class TelemetryOverlay(IOverlayLayer):
|
||||
CPU sys 14.8 % ← normalised by cpu_count (matches Task Manager)
|
||||
CPU core 118.4 % ← per single core (can exceed 100%)
|
||||
Mem 68 MB
|
||||
Inf.dev mps ← inference device (only when model loaded)
|
||||
Inf.time 87 ms ← rolling average of model() call time
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -106,4 +108,10 @@ class TelemetryOverlay(IOverlayLayer):
|
||||
if snap.memory_mb is not None:
|
||||
lines.append(f"Mem {snap.memory_mb:>5.0f} MB")
|
||||
|
||||
if snap.inference_device is not None:
|
||||
lines.append(f"Inf.dev {snap.inference_device:>6s}")
|
||||
|
||||
if snap.inference_time_ms is not None:
|
||||
lines.append(f"Inf.time {snap.inference_time_ms:>5.0f} ms")
|
||||
|
||||
return lines
|
||||
|
||||
@@ -26,6 +26,9 @@ class TelemetrySnapshot:
|
||||
cpu_percent_core: float # process CPU per single core — can exceed 100%
|
||||
memory_mb: float | None # process private working set in MB
|
||||
timestamp: float # time.perf_counter() when snapshot was taken
|
||||
# Inference fields — None when inference is disabled / model not loaded
|
||||
inference_device: str | None = None # e.g. "cpu", "mps"
|
||||
inference_time_ms: float | None = None # rolling average of model() call time
|
||||
|
||||
|
||||
class TelemetryCollector(QObject):
|
||||
@@ -69,6 +72,10 @@ class TelemetryCollector(QObject):
|
||||
self._process.cpu_percent() # first call always returns 0.0; discard
|
||||
self._cpu_count: int = max(psutil.cpu_count(logical=True) or 1, 1)
|
||||
|
||||
# Inference stats (updated externally via set_inference_stats)
|
||||
self._inference_device: str | None = None
|
||||
self._inference_time_ms: float | None = None
|
||||
|
||||
# periodic snapshot timer
|
||||
self._timer = QTimer(self)
|
||||
self._timer.setInterval(update_interval_ms)
|
||||
@@ -85,6 +92,16 @@ class TelemetryCollector(QObject):
|
||||
"""Record the FPS that was requested from the camera."""
|
||||
self._target_fps = fps
|
||||
|
||||
def set_inference_stats(self, device: str, avg_ms: float) -> None:
|
||||
"""Update inference device and average inference time (called from MainWindow)."""
|
||||
self._inference_device: str | None = device
|
||||
self._inference_time_ms: float | None = avg_ms
|
||||
|
||||
def clear_inference_stats(self) -> None:
|
||||
"""Clear inference stats when inference is disabled."""
|
||||
self._inference_device = None
|
||||
self._inference_time_ms = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Frame subscriber callback
|
||||
# ------------------------------------------------------------------
|
||||
@@ -175,6 +192,12 @@ class TelemetryCollector(QObject):
|
||||
cpu_percent_core=round(cpu_core, 1),
|
||||
memory_mb=round(mem_mb, 1) if mem_mb is not None else None,
|
||||
timestamp=now,
|
||||
inference_device=self._inference_device,
|
||||
inference_time_ms=(
|
||||
round(self._inference_time_ms, 1)
|
||||
if self._inference_time_ms is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _make_empty_snapshot(self) -> TelemetrySnapshot:
|
||||
@@ -187,4 +210,6 @@ class TelemetryCollector(QObject):
|
||||
cpu_percent_core=0.0,
|
||||
memory_mb=None,
|
||||
timestamp=time.perf_counter(),
|
||||
inference_device=None,
|
||||
inference_time_ms=None,
|
||||
)
|
||||
|
||||
@@ -184,6 +184,7 @@ class MainWindow(QMainWindow):
|
||||
# ---- InferenceManager ----
|
||||
self._inference.detections_ready.connect(self._bbox_overlay.on_detections)
|
||||
self._inference.detection_count_updated.connect(self._on_detection_count_updated)
|
||||
self._inference.inference_stats_updated.connect(self._on_inference_stats_updated)
|
||||
self._inference.inference_started.connect(self._on_inference_started)
|
||||
self._inference.inference_stopped.connect(self._on_inference_stopped)
|
||||
self._inference.inference_error.connect(self._on_inference_error)
|
||||
@@ -267,6 +268,9 @@ class MainWindow(QMainWindow):
|
||||
def _on_detection_count_updated(self, count: int) -> None:
|
||||
self._detection_label.setText(f"Detections: {count} frames")
|
||||
|
||||
def _on_inference_stats_updated(self, device: str, avg_ms: float) -> None:
|
||||
self._telemetry.set_inference_stats(device, avg_ms)
|
||||
|
||||
def _on_inference_stopped(self) -> None:
|
||||
self._bbox_overlay.clear()
|
||||
|
||||
@@ -276,6 +280,7 @@ class MainWindow(QMainWindow):
|
||||
self._menu.set_inference_checked(False)
|
||||
self._bbox_overlay.visible = False
|
||||
self._detection_label.setVisible(False)
|
||||
self._telemetry.clear_inference_stats()
|
||||
QMessageBox.critical(self, "Inference Error", message)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -350,6 +355,7 @@ class MainWindow(QMainWindow):
|
||||
self._bbox_overlay.clear()
|
||||
self._bbox_overlay.visible = False
|
||||
self._detection_label.setVisible(False)
|
||||
self._telemetry.clear_inference_stats()
|
||||
self._status_label.setText("Inference disabled")
|
||||
logger.info("Inference disabled")
|
||||
|
||||
|
||||
505
notes/05-mvp-yolo.md
Normal file
505
notes/05-mvp-yolo.md
Normal file
@@ -0,0 +1,505 @@
|
||||
# Stan projektu po sesji: YOLO inference + odtwarzanie wideo
|
||||
|
||||
Poprzedni stan: `04-mvp-uvc.md`
|
||||
|
||||
---
|
||||
|
||||
## Kontekst
|
||||
|
||||
Po uruchomieniu aplikacji na Mac Mini z kamerą ELP, kolejny krok to weryfikacja
|
||||
wytrenowanego modelu YOLO. Wymagania:
|
||||
|
||||
- Model działa w **osobnym procesie** — crash workera nie wywala GUI
|
||||
- Inference **nie blokuje** i **nie spowalnia** podglądu kamery
|
||||
- Worker **ignoruje klatki** dopóki analizuje poprzednią (drop-if-busy)
|
||||
- Możliwość wczytania **pliku wideo** zamiast kamery do oceny modelu
|
||||
- Bbox narysowany na nowej **warstwie overlay**
|
||||
- W przyszłości OCR będzie działał w tym samym procesie co YOLO
|
||||
|
||||
---
|
||||
|
||||
## Nowe pakiety
|
||||
|
||||
### `app/video/`
|
||||
|
||||
```
|
||||
app/video/
|
||||
├── __init__.py
|
||||
└── video_player.py
|
||||
```
|
||||
|
||||
### `app/inference/`
|
||||
|
||||
```
|
||||
app/inference/
|
||||
├── __init__.py
|
||||
├── worker.py # funkcja uruchamiana w subprocess
|
||||
├── worker_manager.py # InferenceManager (QObject) — IPC, polling, auto-restart
|
||||
└── bbox_overlay.py # BboxOverlay(IOverlayLayer) — rysuje bbox+label+conf
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Szczegółowy opis zmian
|
||||
|
||||
### 1. `app/video/video_player.py` — `VideoPlayer`
|
||||
|
||||
Nowa klasa `VideoPlayer(QObject)` — identyczny interfejs sygnałowy jak `CameraService`:
|
||||
|
||||
```python
|
||||
frame_ready = Signal(QVideoFrame)
|
||||
playback_started = Signal()
|
||||
playback_stopped = Signal()
|
||||
playback_error = Signal(str)
|
||||
```
|
||||
|
||||
Wewnętrznie: `QMediaPlayer` + `QVideoSink`. Obsługuje formaty: `.mp4`, `.avi`,
|
||||
`.mov`, `.mkv`, `.m4v`, `.webm` (cokolwiek obsługuje FFmpeg backend Qt).
|
||||
|
||||
Odtwarzanie w czasie rzeczywistym (1×). Brak seek/pauzy — tylko Open + Stop.
|
||||
|
||||
`MainWindow` podłącza do `FrameDispatcher` albo `CameraService.frame_ready`,
|
||||
albo `VideoPlayer.frame_ready` — nigdy obu naraz. Przełączanie przez
|
||||
`_switch_to_camera()` / `_switch_to_video()`.
|
||||
|
||||
---
|
||||
|
||||
### 2. `app/inference/worker.py` — worker process
|
||||
|
||||
#### Struktury IPC (NamedTuple — pickle-safe)
|
||||
|
||||
```python
|
||||
class FramePacket(NamedTuple):
|
||||
frame_id: int
|
||||
raw_bytes: bytes # RGB, (H×W×3)
|
||||
width: int
|
||||
height: int
|
||||
channels: int # zawsze 3
|
||||
|
||||
class WorkerReadyPacket(NamedTuple):
|
||||
device: str # "cpu" | "mps" — wysyłany raz po załadowaniu modelu
|
||||
|
||||
class ResultPacket(NamedTuple):
|
||||
frame_id: int
|
||||
detections: list # list of (x1, y1, x2, y2, conf, label)
|
||||
width: int
|
||||
height: int
|
||||
elapsed_ms: float # czas wywołania model() w ms
|
||||
```
|
||||
|
||||
#### Protokół IPC
|
||||
|
||||
```
|
||||
input_queue ← FramePacket
|
||||
output_queue → WorkerReadyPacket (raz, zaraz po załadowaniu modelu)
|
||||
→ ResultPacket (po każdej analizowanej klatce)
|
||||
→ None (tylko przy błędzie ładowania modelu)
|
||||
```
|
||||
|
||||
#### `_select_device()` — wybór urządzenia
|
||||
|
||||
Wywoływany **raz przy starcie workera** (nie per-frame jak wcześniej):
|
||||
|
||||
```python
|
||||
def _select_device() -> str:
|
||||
if platform.system() == "Darwin":
|
||||
if torch.backends.mps.is_available():
|
||||
return "mps" # Metal GPU na macOS
|
||||
return "cpu"
|
||||
```
|
||||
|
||||
`device` jest przekazywany do `_load_model()` i do każdego wywołania `_infer()`.
|
||||
Eliminuje redundantne wykrywanie urządzenia przy każdej klatce.
|
||||
|
||||
#### `_infer()` — pomiar czasu
|
||||
|
||||
```python
|
||||
t0 = time.perf_counter()
|
||||
results = model(frame_np, device=device, verbose=False)
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
```
|
||||
|
||||
`elapsed_ms` trafia do `ResultPacket` i jest logowany w managerze przy detekcjach.
|
||||
|
||||
---
|
||||
|
||||
### 3. `app/inference/worker_manager.py` — `InferenceManager`
|
||||
|
||||
#### Sygnały
|
||||
|
||||
```python
|
||||
detections_ready = Signal(object, object) # list[Detection], tuple[int,int]
|
||||
detection_count_updated = Signal(int) # łączna liczba klatek z detekcją
|
||||
inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms
|
||||
inference_device_changed = Signal(str) # emitowany raz po WorkerReadyPacket
|
||||
inference_started = Signal()
|
||||
inference_stopped = Signal()
|
||||
inference_error = Signal(str)
|
||||
```
|
||||
|
||||
#### Mechanizm drop-if-busy
|
||||
|
||||
```python
|
||||
def submit_frame(self, frame: QVideoFrame) -> None:
|
||||
if not self.is_running or self._paused or self._busy:
|
||||
return # klatka odrzucona cicho
|
||||
# konwersja + put_nowait → self._busy = True
|
||||
```
|
||||
|
||||
`self._busy` wraca do `False` dopiero gdy `_poll_output()` odbierze `ResultPacket`.
|
||||
Gwarantuje że nigdy nie ma więcej niż jedna klatka w locie.
|
||||
|
||||
#### Konwersja klatki w GUI thread
|
||||
|
||||
Zamiast `frame.bits(0)` (dawało tylko płaszczyznę Y dla NV12):
|
||||
|
||||
```python
|
||||
image = frame.toImage() # Qt dekoduje NV12/YUV/MJPG → RGB
|
||||
image = image.convertToFormat(Format_RGB32) # packed BGRX
|
||||
arr = np.frombuffer(image.bits(), dtype=np.uint8).reshape((H, W, 4))
|
||||
rgb = arr[:, :, [2, 1, 0]].copy() # BGRX → RGB, drop X
|
||||
```
|
||||
|
||||
Obsługuje każdy pixel format jaki kamera może dostarczyć.
|
||||
|
||||
#### Rolling average elapsed_ms
|
||||
|
||||
```python
|
||||
_elapsed_window: deque[float] # maxlen=10
|
||||
avg_ms = sum(_elapsed_window) / len(_elapsed_window)
|
||||
```
|
||||
|
||||
Emitowany przez `inference_stats_updated(device, avg_ms)` po każdym `ResultPacket`.
|
||||
|
||||
#### `_poll_output()` — obsługa `WorkerReadyPacket`
|
||||
|
||||
```python
|
||||
if isinstance(item, WorkerReadyPacket):
|
||||
self._current_device = item.device
|
||||
self.inference_device_changed.emit(item.device)
|
||||
continue
|
||||
```
|
||||
|
||||
Odróżnienie od `ResultPacket` przez `isinstance` — nie wymaga sentinel wartości.
|
||||
|
||||
#### Auto-restart
|
||||
|
||||
- Watchdog co 2s sprawdza `process.is_alive()`
|
||||
- Timeout 10s bez odpowiedzi → terminate + restart
|
||||
- Max 3 restartów (konfigurowalny przez `INFERENCE_MAX_RESTARTS`)
|
||||
- Po przekroczeniu: `QMessageBox.critical` + overlay wyłączony
|
||||
|
||||
#### Logowanie — tylko detekcje
|
||||
|
||||
```python
|
||||
if detections:
|
||||
logger.info(
|
||||
"frame %d: %d detection(s) in %.1f ms — %s",
|
||||
packet.frame_id, len(detections), packet.elapsed_ms, conf_summary,
|
||||
)
|
||||
```
|
||||
|
||||
Klatki bez detekcji: brak logu. `conf_summary = "label 0.94, label 0.81"`.
|
||||
|
||||
---
|
||||
|
||||
### 4. `app/inference/bbox_overlay.py` — `BboxOverlay`
|
||||
|
||||
```python
|
||||
class Detection(NamedTuple):
|
||||
x1: float; y1: float; x2: float; y2: float
|
||||
conf: float
|
||||
label: str
|
||||
```
|
||||
|
||||
Współrzędne w pikselach **oryginalnej klatki**. `paint()` skaluje do `video_rect`:
|
||||
|
||||
```python
|
||||
scale_x = video_rect.width() / src_w
|
||||
scale_y = video_rect.height() / src_h
|
||||
wx1 = video_rect.x() + int(det.x1 * scale_x)
|
||||
# ...
|
||||
```
|
||||
|
||||
Każdy bbox: prostokąt w kolorze `BBOX_COLOR` + label `"label 0.87"` na tle
|
||||
`BBOX_LABEL_BG_COLOR` nad lewym górnym rogiem boxa (lub wewnątrz gdy brakuje miejsca).
|
||||
|
||||
`BboxOverlay.visible = False` domyślnie — pojawia się dopiero po włączeniu inference toggle.
|
||||
|
||||
---
|
||||
|
||||
### 5. Menu — zmiany w `app/ui/menu_bar.py`
|
||||
|
||||
Dodano dwa nowe menu przed istniejącymi:
|
||||
|
||||
```
|
||||
File
|
||||
├── Open Video… QFileDialog (.mp4 .avi .mov .mkv .m4v .webm)
|
||||
└── Close Video disabled gdy źródło = kamera
|
||||
|
||||
Model
|
||||
├── Load Model… QFileDialog (.pt .pth)
|
||||
├── Enable Inference QAction checkable, disabled do momentu załadowania modelu
|
||||
└── Model: (none) disabled — info o załadowanym pliku
|
||||
```
|
||||
|
||||
Nowe sygnały:
|
||||
- `video_file_selected(str)` — ścieżka pliku wideo
|
||||
- `video_closed()` — powrót do kamery
|
||||
- `model_file_selected(str)` — ścieżka modelu
|
||||
- `inference_toggled(bool)` — włącz/wyłącz inference
|
||||
|
||||
---
|
||||
|
||||
### 6. `app/ui/main_window.py` — integracja
|
||||
|
||||
#### Przełączanie źródła klatek
|
||||
|
||||
```python
|
||||
def _switch_to_camera(self):
|
||||
video_player.frame_ready.disconnect(dispatcher.dispatch)
|
||||
camera_service.frame_ready.connect(dispatcher.dispatch)
|
||||
|
||||
def _switch_to_video(self):
|
||||
camera_service.frame_ready.disconnect(dispatcher.dispatch)
|
||||
video_player.frame_ready.connect(dispatcher.dispatch)
|
||||
```
|
||||
|
||||
Dispatcher i wszyscy subskrybenci (CameraView, TelemetryCollector,
|
||||
InferenceManager) są podłączeni do dispatchera — źródło klatek jest dla nich
|
||||
transparentne.
|
||||
|
||||
#### Inference toggle
|
||||
|
||||
```python
|
||||
def _on_inference_toggled(self, enabled: bool):
|
||||
if enabled:
|
||||
inference.resume()
|
||||
dispatcher.subscribe(inference.submit_frame, drop_if_busy=True)
|
||||
bbox_overlay.visible = True
|
||||
detection_label.setVisible(True)
|
||||
else:
|
||||
inference.pause()
|
||||
dispatcher.unsubscribe(inference.submit_frame)
|
||||
bbox_overlay.clear()
|
||||
bbox_overlay.visible = False
|
||||
detection_label.setVisible(False)
|
||||
telemetry.clear_inference_stats()
|
||||
```
|
||||
|
||||
`pause()` nie zatrzymuje procesu — tylko blokuje `submit_frame`. Proces
|
||||
pozostaje załadowany w pamięci.
|
||||
|
||||
#### Status bar — counter detekcji
|
||||
|
||||
```python
|
||||
self._detection_label = QLabel("") # addPermanentWidget (prawa strona)
|
||||
```
|
||||
|
||||
Pokazywany tylko gdy inference włączone. Aktualizowany przez
|
||||
`inference.detection_count_updated(int)` → `"Detections: 17 frames"`.
|
||||
|
||||
---
|
||||
|
||||
### 7. Telemetria + overlay — nowe pola inference
|
||||
|
||||
#### `TelemetrySnapshot` — nowe pola
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class TelemetrySnapshot:
|
||||
# ... istniejące pola ...
|
||||
inference_device: str | None = None # "cpu" | "mps" | None
|
||||
inference_time_ms: float | None = None # rolling avg, None gdy wyłączone
|
||||
```
|
||||
|
||||
#### `TelemetryCollector` — nowe metody
|
||||
|
||||
```python
|
||||
def set_inference_stats(self, device: str, avg_ms: float) -> None: ...
|
||||
def clear_inference_stats(self) -> None: ...
|
||||
```
|
||||
|
||||
Wywoływane z `MainWindow` przy każdym `inference_stats_updated` i przy
|
||||
wyłączeniu inference.
|
||||
|
||||
#### `TelemetryOverlay` — nowe wiersze
|
||||
|
||||
```
|
||||
FPS req 30.0
|
||||
FPS got 29.8
|
||||
Frame 33.5 ms
|
||||
Drop 0
|
||||
CPU sys 8.2 %
|
||||
CPU core 65.7 %
|
||||
Mem 71 MB
|
||||
Inf.dev mps ← widoczny tylko gdy model załadowany
|
||||
Inf.time 87 ms ← rolling avg ostatnich 10 klatek
|
||||
```
|
||||
|
||||
Wiersze `Inf.dev` i `Inf.time` znikają gdy inference jest wyłączone
|
||||
(`clear_inference_stats()` → pola `None` → `_format_lines` ich nie emituje).
|
||||
|
||||
---
|
||||
|
||||
### 8. Bugfixes (zidentyfikowane po uruchomieniu)
|
||||
|
||||
#### `Unexpected frame size: 921600 bytes for 1280×720`
|
||||
|
||||
**Przyczyna:** `frame.bits(0)` zwraca tylko płaszczyznę 0 (luma Y) dla formatów
|
||||
planarnych NV12/YUV420P — `1280 × 720 × 1 = 921600` zamiast `1280 × 720 × 3`.
|
||||
|
||||
**Naprawa:** zamiana na `frame.toImage() → Format_RGB32 → bits()`. Qt dekoduje
|
||||
każdy format wewnętrznie. Identyczna ścieżka jak `CameraView.on_frame()`.
|
||||
|
||||
#### `Subscriber not found for removal` (WARNING w logu)
|
||||
|
||||
**Przyczyna:** `_on_inference_toggled(False)` wywoływał `dispatcher.unsubscribe()`
|
||||
zanim subscriber był dodany (pierwsze wyłączenie przed włączeniem).
|
||||
|
||||
**Naprawa:** zmiana poziomu logu z `WARNING` na `DEBUG` w
|
||||
`FrameDispatcher.unsubscribe()` — brak subscribera nie jest błędem.
|
||||
|
||||
---
|
||||
|
||||
## Decyzje architektoniczne
|
||||
|
||||
### Osobny proces zamiast wątku
|
||||
|
||||
`multiprocessing.Process(context="spawn")` zamiast `QThread` lub `threading.Thread`:
|
||||
- Osobny GIL — inference nie blokuje Python event loop GUI
|
||||
- Crash workera (segfault, OOM) nie wywala aplikacji
|
||||
- `spawn` zamiast `fork` — wymagane na macOS od Python 3.12 (Apple deprecuje `fork`)
|
||||
|
||||
### `toImage()` zamiast `bits(0)`
|
||||
|
||||
`QVideoFrame.bits(plane)` daje surowe bajty jednej płaszczyzny. W formatach
|
||||
planarnych (NV12: Y w plane 0, UV w plane 1) to tylko część obrazu. `toImage()`
|
||||
wywołuje wewnętrzny dekoder Qt i zawsze zwraca kompletny obraz niezależnie od
|
||||
pixel formatu.
|
||||
|
||||
### `WorkerReadyPacket` zamiast osobnego IPC kanału
|
||||
|
||||
Worker wysyła `WorkerReadyPacket(device)` do tej samej `output_queue` zaraz
|
||||
po załadowaniu modelu. Manager odróżnia go przez `isinstance`. Eliminuje
|
||||
potrzebę dodatkowej kolejki lub pipe tylko dla metadanych startu.
|
||||
|
||||
### Inference stats przez `TelemetryCollector`, nie bezpośrednio do overlay
|
||||
|
||||
`InferenceManager.inference_stats_updated` → `MainWindow._on_inference_stats_updated`
|
||||
→ `TelemetryCollector.set_inference_stats()` → `TelemetrySnapshot.inference_*`
|
||||
→ `TelemetryOverlay.paint()`.
|
||||
|
||||
Alternatywa: bezpośrednie połączenie `InferenceManager → TelemetryOverlay`.
|
||||
Wybrano pośrednie przez `TelemetryCollector` bo:
|
||||
- `TelemetrySnapshot` jest jedyną strukturą danych opisującą stan systemu
|
||||
- CSV logger automatycznie dostaje inference stats bez dodatkowego kodu
|
||||
- Overlay ma jeden spójny model danych
|
||||
|
||||
---
|
||||
|
||||
## Pliki dodane
|
||||
|
||||
| Plik | Zawartość |
|
||||
|---|---|
|
||||
| `app/video/__init__.py` | pusty |
|
||||
| `app/video/video_player.py` | `VideoPlayer(QObject)` |
|
||||
| `app/inference/__init__.py` | pusty |
|
||||
| `app/inference/worker.py` | `run_worker()`, `FramePacket`, `WorkerReadyPacket`, `ResultPacket`, `_select_device()`, `_infer()` |
|
||||
| `app/inference/worker_manager.py` | `InferenceManager(QObject)` |
|
||||
| `app/inference/bbox_overlay.py` | `Detection(NamedTuple)`, `BboxOverlay(IOverlayLayer)` |
|
||||
| `tests/test_bbox_overlay.py` | 16 testów — `Detection`, stan overlay, mapowanie współrzędnych bbox |
|
||||
| `tests/test_inference_manager.py` | 13 testów — drop-if-busy, pause/resume, restart counter, is_running |
|
||||
|
||||
## Pliki zmienione
|
||||
|
||||
| Plik | Co zmieniono |
|
||||
|---|---|
|
||||
| `app/config.py` | `INFERENCE_WORKER_TIMEOUT_S`, `INFERENCE_MAX_RESTARTS`, `INFERENCE_POLL_INTERVAL_MS`, `INFERENCE_WATCHDOG_INTERVAL_MS`, `BBOX_COLOR`, `BBOX_LABEL_BG_COLOR`, `BBOX_LABEL_TEXT_COLOR`, `BBOX_LINE_WIDTH`, `BBOX_FONT_SIZE`, `VIDEO_FILE_EXTENSIONS`, `MODEL_FILE_EXTENSIONS` |
|
||||
| `app/ui/menu_bar.py` | Menu `File` (Open Video…, Close Video), menu `Model` (Load Model…, Enable Inference, Model info) |
|
||||
| `app/ui/main_window.py` | `VideoPlayer` lifecycle, `InferenceManager` lifecycle, source switching, detection counter w statusbar, `_on_inference_stats_updated` |
|
||||
| `app/telemetry/telemetry_collector.py` | `TelemetrySnapshot.inference_device`, `TelemetrySnapshot.inference_time_ms`, `set_inference_stats()`, `clear_inference_stats()` |
|
||||
| `app/overlay/telemetry_overlay.py` | Wiersze `Inf.dev` i `Inf.time` w `_format_lines()` |
|
||||
| `app/pipeline/frame_dispatcher.py` | `unsubscribe()` brak subscribera: WARNING → DEBUG |
|
||||
| `pyproject.toml` | `[project.optional-dependencies] inference = ["ultralytics>=8.0", "numpy>=1.24"]` |
|
||||
| `tests/test_telemetry_collector.py` | `_make_collector()` uzupełniony o `_inference_device=None`, `_inference_time_ms=None` |
|
||||
|
||||
---
|
||||
|
||||
## Łączna liczba testów
|
||||
|
||||
**69 testów, wszystkie zielone.**
|
||||
|
||||
| Plik | Liczba testów |
|
||||
|---|---|
|
||||
| `test_frame_dispatcher.py` | 8 |
|
||||
| `test_telemetry_collector.py` | 12 |
|
||||
| `test_uvc.py` | 15 |
|
||||
| `test_bbox_overlay.py` | 16 |
|
||||
| `test_inference_manager.py` | 18 |
|
||||
|
||||
---
|
||||
|
||||
## Instalacja
|
||||
|
||||
```bash
|
||||
# Wymagane do inference:
|
||||
pip install ultralytics numpy
|
||||
# lub:
|
||||
pip install -e ".[inference]"
|
||||
```
|
||||
|
||||
Aplikacja startuje bez tych pakietów — `Load Model…` zostaje aktywne, ale
|
||||
`InferenceManager.start()` zgłosi błąd jeśli `ultralytics` nie jest zainstalowany
|
||||
(obsłużony przez `try/except ImportError` w workerze → `output_queue.put(None)`
|
||||
→ manager emituje `inference_error`).
|
||||
|
||||
---
|
||||
|
||||
## Uruchamianie
|
||||
|
||||
```bash
|
||||
# Windows dev
|
||||
.venv-win\Scripts\python.exe -m app.main
|
||||
|
||||
# Mac Mini
|
||||
.venv/bin/python -m app.main
|
||||
|
||||
# Mac Mini z plikiem wideo od razu (CLI nie zaimplementowany — użyj File → Open Video…)
|
||||
.venv/bin/python -m app.main
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Przetestować na Mac Mini z kamerą ELP:
|
||||
- czy `_select_device()` wykrywa MPS i loguje `"MPS (Metal) available"`
|
||||
- czy `Inf.dev mps` pojawia się w overlayzie
|
||||
- czy `Inf.time` jest znacząco niższy niż na CPU
|
||||
|
||||
2. OCR w tym samym procesie co YOLO:
|
||||
- Worker process może obsługiwać wiele zadań — dodać `OcrTask` do `FramePacket`
|
||||
- lub uruchomić OCR jako osobny subscriber `FrameDispatcher` w osobnym procesie
|
||||
|
||||
3. Dodać możliwość regulacji progu confidence (`conf_threshold`) przez menu/dialog
|
||||
— przekazać jako parametr do `run_worker()` w `FramePacket` lub przy starcie
|
||||
|
||||
4. `set_active_format()` call po `_log_actual_format()` żeby menu zaznaczało
|
||||
faktycznie działający format (nie żądany) — z poprzedniej sesji
|
||||
|
||||
---
|
||||
|
||||
## Critical Context
|
||||
|
||||
- `WorkerReadyPacket` jest rozróżniany od `ResultPacket` przez `isinstance` —
|
||||
nie używaj `None` jako sentinela dla obu typów
|
||||
- `_select_device()` wywołany raz przy starcie — jeśli zmienisz device w trakcie
|
||||
działania, trzeba zrestartować workera
|
||||
- `BboxOverlay.on_detections(detections, source_size)` — `source_size` to
|
||||
`tuple[int, int]` (width, height) klatki która była inferowana, nie aktualnego
|
||||
widgetu; potrzebne do poprawnego skalowania przy zmianie rozdzielczości
|
||||
- `InferenceManager.pause()` nie zatrzymuje procesu — `submit_frame` tylko
|
||||
sprawdza flagę; model pozostaje załadowany, można szybko wznowić
|
||||
- `multiprocessing.get_context("spawn")` — wymagane na macOS/Windows;
|
||||
`fork` jest domyślny na Linux ale niebezpieczny z Qt
|
||||
@@ -33,6 +33,9 @@ class TestTelemetryCollector:
|
||||
mem_info.rss = 70 * 1024 * 1024 # RSS (larger, includes shared)
|
||||
col._process.memory_info.return_value = mem_info
|
||||
col._process.cpu_percent.return_value = 0.0
|
||||
# Inference stats — None by default (inference disabled)
|
||||
col._inference_device = None
|
||||
col._inference_time_ms = None
|
||||
return col
|
||||
|
||||
def test_initial_snapshot_has_zero_fps(self):
|
||||
|
||||
Reference in New Issue
Block a user