From 6c401b62bb91b6e871637cebce4dd062c0f63275 Mon Sep 17 00:00:00 2001 From: bartool Date: Wed, 13 May 2026 22:39:08 +0200 Subject: [PATCH] feat: Enhance inference management with device tracking and telemetry updates --- app/inference/worker.py | 45 ++- app/inference/worker_manager.py | 65 +++- app/overlay/telemetry_overlay.py | 8 + app/telemetry/telemetry_collector.py | 25 ++ app/ui/main_window.py | 6 + notes/05-mvp-yolo.md | 505 +++++++++++++++++++++++++++ tests/test_telemetry_collector.py | 3 + 7 files changed, 630 insertions(+), 27 deletions(-) create mode 100644 notes/05-mvp-yolo.md diff --git a/app/inference/worker.py b/app/inference/worker.py index b37d21a..e68958f 100644 --- a/app/inference/worker.py +++ b/app/inference/worker.py @@ -5,8 +5,10 @@ safely be imported and executed in a child process via multiprocessing. IPC protocol ------------ -input_queue receives : FramePacket (frame_id, raw_bytes, width, height, channels) -output_queue sends : ResultPacket (frame_id, detections, width, height) +input_queue receives : FramePacket (frame_id, raw_bytes, width, height, channels) +output_queue sends : WorkerReadyPacket (device) — once after model load + : ResultPacket (frame_id, detections, width, height, elapsed_ms) + : None — on fatal load failure stop_event : multiprocessing.Event — set by parent to request clean exit Detection format (namedtuple-compatible plain tuple): @@ -37,6 +39,14 @@ class FramePacket(NamedTuple): channels: int # always 3 (RGB) +class WorkerReadyPacket(NamedTuple): + """ + Sent once by the worker right after the model is loaded. + Carries the device string so the GUI can display it. + """ + device: str # e.g. "cpu", "mps" + + class ResultPacket(NamedTuple): frame_id: int detections: list # list of (x1, y1, x2, y2, conf, label) tuples @@ -59,8 +69,9 @@ def run_worker( """ Main loop of the inference worker process. - Loads the YOLO model once, then processes frames from input_queue - until stop_event is set. Results are posted to output_queue. + Loads the YOLO model once, sends WorkerReadyPacket, then processes + frames from input_queue until stop_event is set. + Results are posted to output_queue. This function is designed to be the target of multiprocessing.Process. It must NOT import PySide6 or any Qt module. @@ -68,15 +79,21 @@ def run_worker( _configure_worker_logging(log_level) logger.info("Inference worker starting (pid=%d)", _getpid()) + # Select device once — never changes during the lifetime of this process + device = _select_device() + try: - model = _load_model(model_path) + model = _load_model(model_path, device) except Exception as exc: logger.error("Failed to load model '%s': %s", model_path, exc) # Signal failure by putting None — manager treats it as error output_queue.put(None) return - logger.info("Model loaded: %s", model_path) + logger.info("Model loaded: %s device=%s", model_path, device) + + # Notify GUI thread of the device being used + output_queue.put(WorkerReadyPacket(device=device)) while not stop_event.is_set(): try: @@ -88,7 +105,7 @@ def run_worker( break try: - result = _infer(model, packet) + result = _infer(model, packet, device) output_queue.put(result) except Exception as exc: logger.error("Inference error (frame %d): %s", packet.frame_id, exc) @@ -107,11 +124,10 @@ def run_worker( # Helpers # --------------------------------------------------------------------------- -def _load_model(model_path: str): - """Load YOLO model with best available device.""" +def _load_model(model_path: str, device: str): + """Load YOLO model and warm up on the selected device.""" from ultralytics import YOLO # noqa: PLC0415 - device = _select_device() logger.info("Loading YOLO model on device='%s'", device) model = YOLO(model_path) # Warm up — run on a tiny dummy to JIT-compile kernels @@ -126,11 +142,13 @@ def _load_model(model_path: str): def _select_device() -> str: """ - Choose inference device. + Choose the best available inference device. Priority: - - macOS → "mps" if available (Metal GPU), else "cpu" + - macOS → "mps" if torch.backends.mps.is_available(), else "cpu" - others → "cpu" + + Called once at worker startup — not per frame. """ system = platform.system() if system == "Darwin": @@ -145,7 +163,7 @@ def _select_device() -> str: return "cpu" -def _infer(model, packet: FramePacket) -> ResultPacket: +def _infer(model, packet: FramePacket, device: str) -> ResultPacket: """Run model on one frame, return ResultPacket with elapsed_ms.""" import time # noqa: PLC0415 @@ -155,7 +173,6 @@ def _infer(model, packet: FramePacket) -> ResultPacket: (packet.height, packet.width, packet.channels) ) - device = _select_device() t0 = time.perf_counter() results = model(frame_np, device=device, verbose=False) elapsed_ms = (time.perf_counter() - t0) * 1000.0 diff --git a/app/inference/worker_manager.py b/app/inference/worker_manager.py index dfdc5d2..724369d 100644 --- a/app/inference/worker_manager.py +++ b/app/inference/worker_manager.py @@ -5,11 +5,12 @@ Responsibilities: - Submit frames (with drop-if-busy logic) - Poll result queue via QTimer (never blocks the GUI thread) - Watch process health via QTimer (auto-restart on crash) - - Emit Qt signals with results for BboxOverlay + - Emit Qt signals with results for BboxOverlay and TelemetryCollector """ from __future__ import annotations +import collections import logging import multiprocessing import time @@ -25,10 +26,13 @@ from app.config import ( INFERENCE_WORKER_TIMEOUT_S, ) from app.inference.bbox_overlay import Detection -from app.inference.worker import FramePacket, ResultPacket, run_worker +from app.inference.worker import FramePacket, ResultPacket, WorkerReadyPacket, run_worker logger = logging.getLogger(__name__) +# Number of recent inference times to average for the overlay display +_ELAPSED_WINDOW = 10 + class InferenceManager(QObject): """ @@ -40,22 +44,26 @@ class InferenceManager(QObject): detections : list[Detection] source_size : tuple[int, int] — (width, height) of inferred frame + detection_count_updated(int) + Total number of frames on which at least one detection occurred. + + inference_stats_updated(device, avg_ms) + Emitted after every result packet. + device : str — e.g. "cpu", "mps" + avg_ms : float — rolling average of inference time (last 10 frames) + + inference_device_changed(str) + Emitted once when the worker reports its device after model load. + inference_started() — worker is up and model is loaded inference_stopped() — worker has exited cleanly inference_error(str) — fatal error (max restarts exceeded) - - Usage: - mgr = InferenceManager(parent=self) - mgr.detections_ready.connect(bbox_overlay.on_detections) - mgr.start("path/to/model.pt") - # ... - mgr.submit_frame(video_frame) # called by FrameDispatcher subscriber - # ... - mgr.stop() """ detections_ready = Signal(object, object) # list[Detection], tuple[int,int] detection_count_updated = Signal(int) # total frames with detections so far + inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms + inference_device_changed = Signal(str) # emitted once on WorkerReadyPacket inference_started = Signal() inference_stopped = Signal() inference_error = Signal(str) @@ -83,6 +91,14 @@ class InferenceManager(QObject): # Detection counter — frames on which at least one detection occurred self._detection_frame_count: int = 0 + # Device reported by the worker after model load + self._current_device: str = "cpu" + + # Rolling window of recent elapsed_ms values for averaging + self._elapsed_window: collections.deque[float] = collections.deque( + maxlen=_ELAPSED_WINDOW + ) + # QTimers (GUI thread) self._poll_timer = QTimer(self) self._poll_timer.setInterval(INFERENCE_POLL_INTERVAL_MS) @@ -109,6 +125,8 @@ class InferenceManager(QObject): self._restart_count = 0 self._paused = False self._detection_frame_count = 0 + self._elapsed_window.clear() + self._current_device = "cpu" self._start_worker() def stop(self) -> None: @@ -140,6 +158,10 @@ class InferenceManager(QObject): def model_path(self) -> str | None: return self._model_path + @property + def current_device(self) -> str: + return self._current_device + @Slot(QVideoFrame) def submit_frame(self, frame: QVideoFrame) -> None: """ @@ -204,7 +226,6 @@ class InferenceManager(QObject): try: self._input_queue.put_nowait(packet) self._busy = True - # logger.debug("InferenceManager: submitted frame %d", self._frame_id) except Exception as exc: logger.warning("InferenceManager: could not enqueue frame: %s", exc) @@ -278,16 +299,33 @@ class InferenceManager(QObject): try: while True: item = self._output_queue.get_nowait() + if item is None: # Worker signalled a fatal load error logger.error("Worker reported model load failure") self._handle_crash("Model failed to load in worker process") return + # ---------------------------------------------------------- + # WorkerReadyPacket — sent once after model load + # ---------------------------------------------------------- + if isinstance(item, WorkerReadyPacket): + self._current_device = item.device + logger.info("Inference device: %s", item.device) + self.inference_device_changed.emit(item.device) + continue + + # ---------------------------------------------------------- + # ResultPacket — regular inference result + # ---------------------------------------------------------- packet: ResultPacket = item self._busy = False self._last_result_time = time.monotonic() + # Update rolling average of elapsed time + self._elapsed_window.append(packet.elapsed_ms) + avg_ms = sum(self._elapsed_window) / len(self._elapsed_window) + detections = [ Detection(x1, y1, x2, y2, conf, label) for x1, y1, x2, y2, conf, label in packet.detections @@ -308,6 +346,8 @@ class InferenceManager(QObject): ) self.detection_count_updated.emit(self._detection_frame_count) + # Always emit stats so overlay stays current + self.inference_stats_updated.emit(self._current_device, avg_ms) self.detections_ready.emit(detections, source_size) except Exception: @@ -340,7 +380,6 @@ class InferenceManager(QObject): def _handle_crash(self, reason: str) -> None: """Decide whether to auto-restart or give up.""" - # Clean up process handles (already dead) self._poll_timer.stop() self._watchdog_timer.stop() self._process = None diff --git a/app/overlay/telemetry_overlay.py b/app/overlay/telemetry_overlay.py index f1a39fe..faa30d4 100644 --- a/app/overlay/telemetry_overlay.py +++ b/app/overlay/telemetry_overlay.py @@ -33,6 +33,8 @@ class TelemetryOverlay(IOverlayLayer): CPU sys 14.8 % ← normalised by cpu_count (matches Task Manager) CPU core 118.4 % ← per single core (can exceed 100%) Mem 68 MB + Inf.dev mps ← inference device (only when model loaded) + Inf.time 87 ms ← rolling average of model() call time """ def __init__(self) -> None: @@ -106,4 +108,10 @@ class TelemetryOverlay(IOverlayLayer): if snap.memory_mb is not None: lines.append(f"Mem {snap.memory_mb:>5.0f} MB") + if snap.inference_device is not None: + lines.append(f"Inf.dev {snap.inference_device:>6s}") + + if snap.inference_time_ms is not None: + lines.append(f"Inf.time {snap.inference_time_ms:>5.0f} ms") + return lines diff --git a/app/telemetry/telemetry_collector.py b/app/telemetry/telemetry_collector.py index 44a2873..7b77444 100644 --- a/app/telemetry/telemetry_collector.py +++ b/app/telemetry/telemetry_collector.py @@ -26,6 +26,9 @@ class TelemetrySnapshot: cpu_percent_core: float # process CPU per single core — can exceed 100% memory_mb: float | None # process private working set in MB timestamp: float # time.perf_counter() when snapshot was taken + # Inference fields — None when inference is disabled / model not loaded + inference_device: str | None = None # e.g. "cpu", "mps" + inference_time_ms: float | None = None # rolling average of model() call time class TelemetryCollector(QObject): @@ -69,6 +72,10 @@ class TelemetryCollector(QObject): self._process.cpu_percent() # first call always returns 0.0; discard self._cpu_count: int = max(psutil.cpu_count(logical=True) or 1, 1) + # Inference stats (updated externally via set_inference_stats) + self._inference_device: str | None = None + self._inference_time_ms: float | None = None + # periodic snapshot timer self._timer = QTimer(self) self._timer.setInterval(update_interval_ms) @@ -85,6 +92,16 @@ class TelemetryCollector(QObject): """Record the FPS that was requested from the camera.""" self._target_fps = fps + def set_inference_stats(self, device: str, avg_ms: float) -> None: + """Update inference device and average inference time (called from MainWindow).""" + self._inference_device: str | None = device + self._inference_time_ms: float | None = avg_ms + + def clear_inference_stats(self) -> None: + """Clear inference stats when inference is disabled.""" + self._inference_device = None + self._inference_time_ms = None + # ------------------------------------------------------------------ # Frame subscriber callback # ------------------------------------------------------------------ @@ -175,6 +192,12 @@ class TelemetryCollector(QObject): cpu_percent_core=round(cpu_core, 1), memory_mb=round(mem_mb, 1) if mem_mb is not None else None, timestamp=now, + inference_device=self._inference_device, + inference_time_ms=( + round(self._inference_time_ms, 1) + if self._inference_time_ms is not None + else None + ), ) def _make_empty_snapshot(self) -> TelemetrySnapshot: @@ -187,4 +210,6 @@ class TelemetryCollector(QObject): cpu_percent_core=0.0, memory_mb=None, timestamp=time.perf_counter(), + inference_device=None, + inference_time_ms=None, ) diff --git a/app/ui/main_window.py b/app/ui/main_window.py index 9a0cf26..a3bec7e 100644 --- a/app/ui/main_window.py +++ b/app/ui/main_window.py @@ -184,6 +184,7 @@ class MainWindow(QMainWindow): # ---- InferenceManager ---- self._inference.detections_ready.connect(self._bbox_overlay.on_detections) self._inference.detection_count_updated.connect(self._on_detection_count_updated) + self._inference.inference_stats_updated.connect(self._on_inference_stats_updated) self._inference.inference_started.connect(self._on_inference_started) self._inference.inference_stopped.connect(self._on_inference_stopped) self._inference.inference_error.connect(self._on_inference_error) @@ -267,6 +268,9 @@ class MainWindow(QMainWindow): def _on_detection_count_updated(self, count: int) -> None: self._detection_label.setText(f"Detections: {count} frames") + def _on_inference_stats_updated(self, device: str, avg_ms: float) -> None: + self._telemetry.set_inference_stats(device, avg_ms) + def _on_inference_stopped(self) -> None: self._bbox_overlay.clear() @@ -276,6 +280,7 @@ class MainWindow(QMainWindow): self._menu.set_inference_checked(False) self._bbox_overlay.visible = False self._detection_label.setVisible(False) + self._telemetry.clear_inference_stats() QMessageBox.critical(self, "Inference Error", message) # ------------------------------------------------------------------ @@ -350,6 +355,7 @@ class MainWindow(QMainWindow): self._bbox_overlay.clear() self._bbox_overlay.visible = False self._detection_label.setVisible(False) + self._telemetry.clear_inference_stats() self._status_label.setText("Inference disabled") logger.info("Inference disabled") diff --git a/notes/05-mvp-yolo.md b/notes/05-mvp-yolo.md new file mode 100644 index 0000000..0f8d192 --- /dev/null +++ b/notes/05-mvp-yolo.md @@ -0,0 +1,505 @@ +# Stan projektu po sesji: YOLO inference + odtwarzanie wideo + +Poprzedni stan: `04-mvp-uvc.md` + +--- + +## Kontekst + +Po uruchomieniu aplikacji na Mac Mini z kamerą ELP, kolejny krok to weryfikacja +wytrenowanego modelu YOLO. Wymagania: + +- Model działa w **osobnym procesie** — crash workera nie wywala GUI +- Inference **nie blokuje** i **nie spowalnia** podglądu kamery +- Worker **ignoruje klatki** dopóki analizuje poprzednią (drop-if-busy) +- Możliwość wczytania **pliku wideo** zamiast kamery do oceny modelu +- Bbox narysowany na nowej **warstwie overlay** +- W przyszłości OCR będzie działał w tym samym procesie co YOLO + +--- + +## Nowe pakiety + +### `app/video/` + +``` +app/video/ +├── __init__.py +└── video_player.py +``` + +### `app/inference/` + +``` +app/inference/ +├── __init__.py +├── worker.py # funkcja uruchamiana w subprocess +├── worker_manager.py # InferenceManager (QObject) — IPC, polling, auto-restart +└── bbox_overlay.py # BboxOverlay(IOverlayLayer) — rysuje bbox+label+conf +``` + +--- + +## Szczegółowy opis zmian + +### 1. `app/video/video_player.py` — `VideoPlayer` + +Nowa klasa `VideoPlayer(QObject)` — identyczny interfejs sygnałowy jak `CameraService`: + +```python +frame_ready = Signal(QVideoFrame) +playback_started = Signal() +playback_stopped = Signal() +playback_error = Signal(str) +``` + +Wewnętrznie: `QMediaPlayer` + `QVideoSink`. Obsługuje formaty: `.mp4`, `.avi`, +`.mov`, `.mkv`, `.m4v`, `.webm` (cokolwiek obsługuje FFmpeg backend Qt). + +Odtwarzanie w czasie rzeczywistym (1×). Brak seek/pauzy — tylko Open + Stop. + +`MainWindow` podłącza do `FrameDispatcher` albo `CameraService.frame_ready`, +albo `VideoPlayer.frame_ready` — nigdy obu naraz. Przełączanie przez +`_switch_to_camera()` / `_switch_to_video()`. + +--- + +### 2. `app/inference/worker.py` — worker process + +#### Struktury IPC (NamedTuple — pickle-safe) + +```python +class FramePacket(NamedTuple): + frame_id: int + raw_bytes: bytes # RGB, (H×W×3) + width: int + height: int + channels: int # zawsze 3 + +class WorkerReadyPacket(NamedTuple): + device: str # "cpu" | "mps" — wysyłany raz po załadowaniu modelu + +class ResultPacket(NamedTuple): + frame_id: int + detections: list # list of (x1, y1, x2, y2, conf, label) + width: int + height: int + elapsed_ms: float # czas wywołania model() w ms +``` + +#### Protokół IPC + +``` +input_queue ← FramePacket +output_queue → WorkerReadyPacket (raz, zaraz po załadowaniu modelu) + → ResultPacket (po każdej analizowanej klatce) + → None (tylko przy błędzie ładowania modelu) +``` + +#### `_select_device()` — wybór urządzenia + +Wywoływany **raz przy starcie workera** (nie per-frame jak wcześniej): + +```python +def _select_device() -> str: + if platform.system() == "Darwin": + if torch.backends.mps.is_available(): + return "mps" # Metal GPU na macOS + return "cpu" +``` + +`device` jest przekazywany do `_load_model()` i do każdego wywołania `_infer()`. +Eliminuje redundantne wykrywanie urządzenia przy każdej klatce. + +#### `_infer()` — pomiar czasu + +```python +t0 = time.perf_counter() +results = model(frame_np, device=device, verbose=False) +elapsed_ms = (time.perf_counter() - t0) * 1000.0 +``` + +`elapsed_ms` trafia do `ResultPacket` i jest logowany w managerze przy detekcjach. + +--- + +### 3. `app/inference/worker_manager.py` — `InferenceManager` + +#### Sygnały + +```python +detections_ready = Signal(object, object) # list[Detection], tuple[int,int] +detection_count_updated = Signal(int) # łączna liczba klatek z detekcją +inference_stats_updated = Signal(str, float) # device, avg_elapsed_ms +inference_device_changed = Signal(str) # emitowany raz po WorkerReadyPacket +inference_started = Signal() +inference_stopped = Signal() +inference_error = Signal(str) +``` + +#### Mechanizm drop-if-busy + +```python +def submit_frame(self, frame: QVideoFrame) -> None: + if not self.is_running or self._paused or self._busy: + return # klatka odrzucona cicho + # konwersja + put_nowait → self._busy = True +``` + +`self._busy` wraca do `False` dopiero gdy `_poll_output()` odbierze `ResultPacket`. +Gwarantuje że nigdy nie ma więcej niż jedna klatka w locie. + +#### Konwersja klatki w GUI thread + +Zamiast `frame.bits(0)` (dawało tylko płaszczyznę Y dla NV12): + +```python +image = frame.toImage() # Qt dekoduje NV12/YUV/MJPG → RGB +image = image.convertToFormat(Format_RGB32) # packed BGRX +arr = np.frombuffer(image.bits(), dtype=np.uint8).reshape((H, W, 4)) +rgb = arr[:, :, [2, 1, 0]].copy() # BGRX → RGB, drop X +``` + +Obsługuje każdy pixel format jaki kamera może dostarczyć. + +#### Rolling average elapsed_ms + +```python +_elapsed_window: deque[float] # maxlen=10 +avg_ms = sum(_elapsed_window) / len(_elapsed_window) +``` + +Emitowany przez `inference_stats_updated(device, avg_ms)` po każdym `ResultPacket`. + +#### `_poll_output()` — obsługa `WorkerReadyPacket` + +```python +if isinstance(item, WorkerReadyPacket): + self._current_device = item.device + self.inference_device_changed.emit(item.device) + continue +``` + +Odróżnienie od `ResultPacket` przez `isinstance` — nie wymaga sentinel wartości. + +#### Auto-restart + +- Watchdog co 2s sprawdza `process.is_alive()` +- Timeout 10s bez odpowiedzi → terminate + restart +- Max 3 restartów (konfigurowalny przez `INFERENCE_MAX_RESTARTS`) +- Po przekroczeniu: `QMessageBox.critical` + overlay wyłączony + +#### Logowanie — tylko detekcje + +```python +if detections: + logger.info( + "frame %d: %d detection(s) in %.1f ms — %s", + packet.frame_id, len(detections), packet.elapsed_ms, conf_summary, + ) +``` + +Klatki bez detekcji: brak logu. `conf_summary = "label 0.94, label 0.81"`. + +--- + +### 4. `app/inference/bbox_overlay.py` — `BboxOverlay` + +```python +class Detection(NamedTuple): + x1: float; y1: float; x2: float; y2: float + conf: float + label: str +``` + +Współrzędne w pikselach **oryginalnej klatki**. `paint()` skaluje do `video_rect`: + +```python +scale_x = video_rect.width() / src_w +scale_y = video_rect.height() / src_h +wx1 = video_rect.x() + int(det.x1 * scale_x) +# ... +``` + +Każdy bbox: prostokąt w kolorze `BBOX_COLOR` + label `"label 0.87"` na tle +`BBOX_LABEL_BG_COLOR` nad lewym górnym rogiem boxa (lub wewnątrz gdy brakuje miejsca). + +`BboxOverlay.visible = False` domyślnie — pojawia się dopiero po włączeniu inference toggle. + +--- + +### 5. Menu — zmiany w `app/ui/menu_bar.py` + +Dodano dwa nowe menu przed istniejącymi: + +``` +File + ├── Open Video… QFileDialog (.mp4 .avi .mov .mkv .m4v .webm) + └── Close Video disabled gdy źródło = kamera + +Model + ├── Load Model… QFileDialog (.pt .pth) + ├── Enable Inference QAction checkable, disabled do momentu załadowania modelu + └── Model: (none) disabled — info o załadowanym pliku +``` + +Nowe sygnały: +- `video_file_selected(str)` — ścieżka pliku wideo +- `video_closed()` — powrót do kamery +- `model_file_selected(str)` — ścieżka modelu +- `inference_toggled(bool)` — włącz/wyłącz inference + +--- + +### 6. `app/ui/main_window.py` — integracja + +#### Przełączanie źródła klatek + +```python +def _switch_to_camera(self): + video_player.frame_ready.disconnect(dispatcher.dispatch) + camera_service.frame_ready.connect(dispatcher.dispatch) + +def _switch_to_video(self): + camera_service.frame_ready.disconnect(dispatcher.dispatch) + video_player.frame_ready.connect(dispatcher.dispatch) +``` + +Dispatcher i wszyscy subskrybenci (CameraView, TelemetryCollector, +InferenceManager) są podłączeni do dispatchera — źródło klatek jest dla nich +transparentne. + +#### Inference toggle + +```python +def _on_inference_toggled(self, enabled: bool): + if enabled: + inference.resume() + dispatcher.subscribe(inference.submit_frame, drop_if_busy=True) + bbox_overlay.visible = True + detection_label.setVisible(True) + else: + inference.pause() + dispatcher.unsubscribe(inference.submit_frame) + bbox_overlay.clear() + bbox_overlay.visible = False + detection_label.setVisible(False) + telemetry.clear_inference_stats() +``` + +`pause()` nie zatrzymuje procesu — tylko blokuje `submit_frame`. Proces +pozostaje załadowany w pamięci. + +#### Status bar — counter detekcji + +```python +self._detection_label = QLabel("") # addPermanentWidget (prawa strona) +``` + +Pokazywany tylko gdy inference włączone. Aktualizowany przez +`inference.detection_count_updated(int)` → `"Detections: 17 frames"`. + +--- + +### 7. Telemetria + overlay — nowe pola inference + +#### `TelemetrySnapshot` — nowe pola + +```python +@dataclass +class TelemetrySnapshot: + # ... istniejące pola ... + inference_device: str | None = None # "cpu" | "mps" | None + inference_time_ms: float | None = None # rolling avg, None gdy wyłączone +``` + +#### `TelemetryCollector` — nowe metody + +```python +def set_inference_stats(self, device: str, avg_ms: float) -> None: ... +def clear_inference_stats(self) -> None: ... +``` + +Wywoływane z `MainWindow` przy każdym `inference_stats_updated` i przy +wyłączeniu inference. + +#### `TelemetryOverlay` — nowe wiersze + +``` +FPS req 30.0 +FPS got 29.8 +Frame 33.5 ms +Drop 0 +CPU sys 8.2 % +CPU core 65.7 % +Mem 71 MB +Inf.dev mps ← widoczny tylko gdy model załadowany +Inf.time 87 ms ← rolling avg ostatnich 10 klatek +``` + +Wiersze `Inf.dev` i `Inf.time` znikają gdy inference jest wyłączone +(`clear_inference_stats()` → pola `None` → `_format_lines` ich nie emituje). + +--- + +### 8. Bugfixes (zidentyfikowane po uruchomieniu) + +#### `Unexpected frame size: 921600 bytes for 1280×720` + +**Przyczyna:** `frame.bits(0)` zwraca tylko płaszczyznę 0 (luma Y) dla formatów +planarnych NV12/YUV420P — `1280 × 720 × 1 = 921600` zamiast `1280 × 720 × 3`. + +**Naprawa:** zamiana na `frame.toImage() → Format_RGB32 → bits()`. Qt dekoduje +każdy format wewnętrznie. Identyczna ścieżka jak `CameraView.on_frame()`. + +#### `Subscriber not found for removal` (WARNING w logu) + +**Przyczyna:** `_on_inference_toggled(False)` wywoływał `dispatcher.unsubscribe()` +zanim subscriber był dodany (pierwsze wyłączenie przed włączeniem). + +**Naprawa:** zmiana poziomu logu z `WARNING` na `DEBUG` w +`FrameDispatcher.unsubscribe()` — brak subscribera nie jest błędem. + +--- + +## Decyzje architektoniczne + +### Osobny proces zamiast wątku + +`multiprocessing.Process(context="spawn")` zamiast `QThread` lub `threading.Thread`: +- Osobny GIL — inference nie blokuje Python event loop GUI +- Crash workera (segfault, OOM) nie wywala aplikacji +- `spawn` zamiast `fork` — wymagane na macOS od Python 3.12 (Apple deprecuje `fork`) + +### `toImage()` zamiast `bits(0)` + +`QVideoFrame.bits(plane)` daje surowe bajty jednej płaszczyzny. W formatach +planarnych (NV12: Y w plane 0, UV w plane 1) to tylko część obrazu. `toImage()` +wywołuje wewnętrzny dekoder Qt i zawsze zwraca kompletny obraz niezależnie od +pixel formatu. + +### `WorkerReadyPacket` zamiast osobnego IPC kanału + +Worker wysyła `WorkerReadyPacket(device)` do tej samej `output_queue` zaraz +po załadowaniu modelu. Manager odróżnia go przez `isinstance`. Eliminuje +potrzebę dodatkowej kolejki lub pipe tylko dla metadanych startu. + +### Inference stats przez `TelemetryCollector`, nie bezpośrednio do overlay + +`InferenceManager.inference_stats_updated` → `MainWindow._on_inference_stats_updated` +→ `TelemetryCollector.set_inference_stats()` → `TelemetrySnapshot.inference_*` +→ `TelemetryOverlay.paint()`. + +Alternatywa: bezpośrednie połączenie `InferenceManager → TelemetryOverlay`. +Wybrano pośrednie przez `TelemetryCollector` bo: +- `TelemetrySnapshot` jest jedyną strukturą danych opisującą stan systemu +- CSV logger automatycznie dostaje inference stats bez dodatkowego kodu +- Overlay ma jeden spójny model danych + +--- + +## Pliki dodane + +| Plik | Zawartość | +|---|---| +| `app/video/__init__.py` | pusty | +| `app/video/video_player.py` | `VideoPlayer(QObject)` | +| `app/inference/__init__.py` | pusty | +| `app/inference/worker.py` | `run_worker()`, `FramePacket`, `WorkerReadyPacket`, `ResultPacket`, `_select_device()`, `_infer()` | +| `app/inference/worker_manager.py` | `InferenceManager(QObject)` | +| `app/inference/bbox_overlay.py` | `Detection(NamedTuple)`, `BboxOverlay(IOverlayLayer)` | +| `tests/test_bbox_overlay.py` | 16 testów — `Detection`, stan overlay, mapowanie współrzędnych bbox | +| `tests/test_inference_manager.py` | 13 testów — drop-if-busy, pause/resume, restart counter, is_running | + +## Pliki zmienione + +| Plik | Co zmieniono | +|---|---| +| `app/config.py` | `INFERENCE_WORKER_TIMEOUT_S`, `INFERENCE_MAX_RESTARTS`, `INFERENCE_POLL_INTERVAL_MS`, `INFERENCE_WATCHDOG_INTERVAL_MS`, `BBOX_COLOR`, `BBOX_LABEL_BG_COLOR`, `BBOX_LABEL_TEXT_COLOR`, `BBOX_LINE_WIDTH`, `BBOX_FONT_SIZE`, `VIDEO_FILE_EXTENSIONS`, `MODEL_FILE_EXTENSIONS` | +| `app/ui/menu_bar.py` | Menu `File` (Open Video…, Close Video), menu `Model` (Load Model…, Enable Inference, Model info) | +| `app/ui/main_window.py` | `VideoPlayer` lifecycle, `InferenceManager` lifecycle, source switching, detection counter w statusbar, `_on_inference_stats_updated` | +| `app/telemetry/telemetry_collector.py` | `TelemetrySnapshot.inference_device`, `TelemetrySnapshot.inference_time_ms`, `set_inference_stats()`, `clear_inference_stats()` | +| `app/overlay/telemetry_overlay.py` | Wiersze `Inf.dev` i `Inf.time` w `_format_lines()` | +| `app/pipeline/frame_dispatcher.py` | `unsubscribe()` brak subscribera: WARNING → DEBUG | +| `pyproject.toml` | `[project.optional-dependencies] inference = ["ultralytics>=8.0", "numpy>=1.24"]` | +| `tests/test_telemetry_collector.py` | `_make_collector()` uzupełniony o `_inference_device=None`, `_inference_time_ms=None` | + +--- + +## Łączna liczba testów + +**69 testów, wszystkie zielone.** + +| Plik | Liczba testów | +|---|---| +| `test_frame_dispatcher.py` | 8 | +| `test_telemetry_collector.py` | 12 | +| `test_uvc.py` | 15 | +| `test_bbox_overlay.py` | 16 | +| `test_inference_manager.py` | 18 | + +--- + +## Instalacja + +```bash +# Wymagane do inference: +pip install ultralytics numpy +# lub: +pip install -e ".[inference]" +``` + +Aplikacja startuje bez tych pakietów — `Load Model…` zostaje aktywne, ale +`InferenceManager.start()` zgłosi błąd jeśli `ultralytics` nie jest zainstalowany +(obsłużony przez `try/except ImportError` w workerze → `output_queue.put(None)` +→ manager emituje `inference_error`). + +--- + +## Uruchamianie + +```bash +# Windows dev +.venv-win\Scripts\python.exe -m app.main + +# Mac Mini +.venv/bin/python -m app.main + +# Mac Mini z plikiem wideo od razu (CLI nie zaimplementowany — użyj File → Open Video…) +.venv/bin/python -m app.main +``` + +--- + +## Next Steps + +1. Przetestować na Mac Mini z kamerą ELP: + - czy `_select_device()` wykrywa MPS i loguje `"MPS (Metal) available"` + - czy `Inf.dev mps` pojawia się w overlayzie + - czy `Inf.time` jest znacząco niższy niż na CPU + +2. OCR w tym samym procesie co YOLO: + - Worker process może obsługiwać wiele zadań — dodać `OcrTask` do `FramePacket` + - lub uruchomić OCR jako osobny subscriber `FrameDispatcher` w osobnym procesie + +3. Dodać możliwość regulacji progu confidence (`conf_threshold`) przez menu/dialog + — przekazać jako parametr do `run_worker()` w `FramePacket` lub przy starcie + +4. `set_active_format()` call po `_log_actual_format()` żeby menu zaznaczało + faktycznie działający format (nie żądany) — z poprzedniej sesji + +--- + +## Critical Context + +- `WorkerReadyPacket` jest rozróżniany od `ResultPacket` przez `isinstance` — + nie używaj `None` jako sentinela dla obu typów +- `_select_device()` wywołany raz przy starcie — jeśli zmienisz device w trakcie + działania, trzeba zrestartować workera +- `BboxOverlay.on_detections(detections, source_size)` — `source_size` to + `tuple[int, int]` (width, height) klatki która była inferowana, nie aktualnego + widgetu; potrzebne do poprawnego skalowania przy zmianie rozdzielczości +- `InferenceManager.pause()` nie zatrzymuje procesu — `submit_frame` tylko + sprawdza flagę; model pozostaje załadowany, można szybko wznowić +- `multiprocessing.get_context("spawn")` — wymagane na macOS/Windows; + `fork` jest domyślny na Linux ale niebezpieczny z Qt diff --git a/tests/test_telemetry_collector.py b/tests/test_telemetry_collector.py index eea4027..05c2099 100644 --- a/tests/test_telemetry_collector.py +++ b/tests/test_telemetry_collector.py @@ -33,6 +33,9 @@ class TestTelemetryCollector: mem_info.rss = 70 * 1024 * 1024 # RSS (larger, includes shared) col._process.memory_info.return_value = mem_info col._process.cpu_percent.return_value = 0.0 + # Inference stats — None by default (inference disabled) + col._inference_device = None + col._inference_time_ms = None return col def test_initial_snapshot_has_zero_fps(self):