From 500cd59af14fc62fa0cf51f2e16dc10fb469fec5 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Wed, 15 Oct 2025 23:33:38 -0400
Subject: [PATCH 01/14] initial implementation of model probability calibration
---
src/jabs/classifier/classifier.py | 200 ++++++++++++++++++-----
src/jabs/scripts/classify.py | 6 +-
src/jabs/scripts/stats.py | 2 +-
src/jabs/ui/central_widget.py | 2 +-
src/jabs/ui/main_window.py | 14 ++
src/jabs/ui/settings_dialog.py | 262 ++++++++++++++++++++++++++++++
6 files changed, 440 insertions(+), 46 deletions(-)
create mode 100644 src/jabs/ui/settings_dialog.py
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index a38bfed3..df66ca01 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -8,6 +8,7 @@
import joblib
import numpy as np
import pandas as pd
+from sklearn.calibration import CalibratedClassifierCV
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.exceptions import InconsistentVersionWarning
from sklearn.metrics import (
@@ -21,7 +22,7 @@
from jabs.types import ClassifierType
from jabs.utils import hash_file
-_VERSION = 9
+_VERSION = 10
_classifier_choices = [ClassifierType.RANDOM_FOREST, ClassifierType.GRADIENT_BOOSTING]
@@ -60,6 +61,7 @@ class Classifier:
def __init__(self, classifier=ClassifierType.RANDOM_FOREST, n_jobs=1):
self._classifier_type = classifier
self._classifier = None
+ self._behavior_settings = None
self._project_settings = None
self._behavior = None
self._feature_names = None
@@ -141,10 +143,10 @@ def classifier_hash(self) -> str:
return "NO HASH"
@property
- def project_settings(self) -> dict:
- """return a copy of dictionary of project settings for this classifier"""
- if self._project_settings is not None:
- return dict(self._project_settings)
+ def behavior_settings(self) -> dict:
+ """return a copy of dictionary of behavior-specific settings for this classifier"""
+ if self._behavior_settings is not None:
+ return dict(self._behavior_settings)
return {}
@property
@@ -361,19 +363,21 @@ def set_project_settings(self, project: Project):
if no behavior is currently set will use project defaults
"""
if self._behavior is None:
- self._project_settings = project.get_project_defaults()
+ self._behavior_settings = project.get_project_defaults()
else:
- self._project_settings = project.settings_manager.get_behavior(self._behavior)
+ self._behavior_settings = project.settings_manager.get_behavior(self._behavior)
+
+ self._project_settings = project.settings_manager.project_settings.get("settings", {})
def set_dict_settings(self, settings: dict):
- """assign project settings via a dict to the classifier
+ """assign behavior-specific settings via a dict to the classifier
Args:
settings: dict of project settings. Must be same structure as project.settings_manager.get_behavior
TODO: Add checks to enforce conformity to project settings
"""
- self._project_settings = dict(settings)
+ self._behavior_settings = dict(settings)
def classifier_choices(self):
"""get the available classifier types
@@ -403,7 +407,7 @@ def train(self, data, random_seed: int | None = None):
raises ValueError for having either unset project settings or an unset classifier
"""
- if self._project_settings is None:
+ if self._behavior_settings is None:
raise ValueError("Project settings for classifier unset, cannot train classifier.")
# Assume that feature names is provided, otherwise extract it from the dataframe
@@ -416,32 +420,90 @@ def train(self, data, random_seed: int | None = None):
features = data["training_data"]
labels = data["training_labels"]
# Symmetric augmentation should occur before balancing so that the class with more labels can sample from the whole set
- if self._project_settings.get("symmetric_behavior", False):
+ if self._behavior_settings.get("symmetric_behavior", False):
features, labels = self.augment_symmetric(features, labels)
- if self._project_settings.get("balance_labels", False):
+ if self._behavior_settings.get("balance_labels", False):
features, labels = self.downsample_balance(features, labels, random_seed)
- if self._classifier_type == ClassifierType.RANDOM_FOREST:
- self._classifier = self._fit_random_forest(features, labels, random_seed=random_seed)
- elif self._classifier_type == ClassifierType.GRADIENT_BOOSTING:
- self._classifier = self._fit_gradient_boost(features, labels, random_seed=random_seed)
- elif _xgboost is not None and self._classifier_type == ClassifierType.XGBOOST:
+ # Optional probability calibration, this is currently set at the project level
+ calibrate = self._project_settings.get("calibrate_probabilities", False)
+ calibration_method = self._project_settings.get(
+ "calibration_method", "isotonic"
+ ) # or 'sigmoid'
+ calibration_cv = self._project_settings.get("calibration_cv", 3)
+
+ if calibrate:
+ # Build an unfitted base estimator
+ if self._classifier_type == ClassifierType.RANDOM_FOREST:
+ base_estimator = self._make_random_forest(random_seed=random_seed)
+ elif self._classifier_type == ClassifierType.GRADIENT_BOOSTING:
+ base_estimator = self._make_gradient_boost(random_seed=random_seed)
+ elif _xgboost is not None and self._classifier_type == ClassifierType.XGBOOST:
+ base_estimator = self._make_xgboost(random_seed=random_seed)
+ else:
+ raise ValueError("Unsupported classifier")
+
+ # Wrap with calibrated classifier and fit
+ self._classifier = CalibratedClassifierCV(
+ estimator=base_estimator, method=calibration_method, cv=calibration_cv
+ )
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=FutureWarning)
- self._classifier = self._fit_xgboost(features, labels, random_seed=random_seed)
+ self._classifier.fit(self._clean_features_for_training(features), labels)
else:
- raise ValueError("Unsupported classifier")
+ # Fit without calibration (original behavior)
+ if self._classifier_type == ClassifierType.RANDOM_FOREST:
+ self._classifier = self._fit_random_forest(
+ features, labels, random_seed=random_seed
+ )
+ elif self._classifier_type == ClassifierType.GRADIENT_BOOSTING:
+ self._classifier = self._fit_gradient_boost(
+ features, labels, random_seed=random_seed
+ )
+ elif _xgboost is not None and self._classifier_type == ClassifierType.XGBOOST:
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", category=FutureWarning)
+ self._classifier = self._fit_xgboost(features, labels, random_seed=random_seed)
+ else:
+ raise ValueError("Unsupported classifier")
# Classifier may have been re-used from a prior training, blank the logging attributes
self._classifier_file = None
self._classifier_hash = None
self._classifier_source = None
+ def _clean_features_for_training(self, features: pd.DataFrame):
+ """Clean feature matrix prior to fitting based on classifier type.
+
+ For XGBoost, only replace +/- inf with 0 (XGBoost can handle NaN).
+ For sklearn tree models, also fill NaNs with 0.
+ """
+ if self._classifier_type == ClassifierType.XGBOOST:
+ return features.replace([np.inf, -np.inf], 0)
+ return features.replace([np.inf, -np.inf], 0).fillna(0)
+
+ def _make_random_forest(self, random_seed: int | None = None):
+ if random_seed is not None:
+ return RandomForestClassifier(n_jobs=self._n_jobs, random_state=random_seed)
+ return RandomForestClassifier(n_jobs=self._n_jobs)
+
+ def _make_gradient_boost(self, random_seed: int | None = None):
+ if random_seed is not None:
+ return GradientBoostingClassifier(random_state=random_seed)
+ return GradientBoostingClassifier()
+
+ def _make_xgboost(self, random_seed: int | None = None):
+ if random_seed is not None:
+ return _xgboost.XGBClassifier(n_jobs=self._n_jobs, random_state=random_seed)
+ return _xgboost.XGBClassifier(n_jobs=self._n_jobs)
+
def sort_features_to_classify(self, features):
"""sorts features to match the current classifier"""
- if self._classifier_type == ClassifierType.XGBOOST:
+ if isinstance(self._classifier, CalibratedClassifierCV):
+ # Use the training-time feature order we stored
+ classifier_columns = self._feature_names
+ elif self._classifier_type == ClassifierType.XGBOOST:
classifier_columns = self._classifier.get_booster().feature_names
- # sklearn places feature names in the same spot
else:
classifier_columns = self._classifier.feature_names_in_
features_sorted = features[classifier_columns]
@@ -518,6 +580,7 @@ def load(self, path: Path):
self._classifier = c._classifier
self._behavior = c._behavior
+ self._behavior_settings = c._behavior_settings
self._project_settings = c._project_settings
self._classifier_type = c._classifier_type
if c._classifier_file is not None:
@@ -568,27 +631,65 @@ def combine_data(per_frame, window):
return pd.concat([per_frame, window], axis=1)
def _fit_random_forest(self, features, labels, random_seed: int | None = None):
- if random_seed is not None:
- classifier = RandomForestClassifier(n_jobs=self._n_jobs, random_state=random_seed)
- else:
- classifier = RandomForestClassifier(n_jobs=self._n_jobs)
+ classifier = self._make_random_forest(random_seed=random_seed)
return classifier.fit(features.replace([np.inf, -np.inf], 0).fillna(0), labels)
def _fit_gradient_boost(self, features, labels, random_seed: int | None = None):
- if random_seed is not None:
- classifier = GradientBoostingClassifier(random_state=random_seed)
- else:
- classifier = GradientBoostingClassifier()
+ classifier = self._make_gradient_boost(random_seed=random_seed)
return classifier.fit(features.replace([np.inf, -np.inf], 0).fillna(0), labels)
def _fit_xgboost(self, features, labels, random_seed: int | None = None):
- if random_seed is not None:
- classifier = _xgboost.XGBClassifier(n_jobs=self._n_jobs, random_state=random_seed)
- else:
- classifier = _xgboost.XGBClassifier(n_jobs=self._n_jobs)
+ classifier = self._make_xgboost(random_seed=random_seed)
classifier.fit(features.replace([np.inf, -np.inf]), labels)
return classifier
+ def _get_estimator_with_feature_importances(self):
+ """Return the underlying estimator that exposes `feature_importances_`, if available.
+
+ Handles calibrated classifiers by retrieving the estimator from the first
+ calibrated fold. Returns None if no estimator with `feature_importances_` is found.
+ """
+ est = self._classifier
+ # If wrapped by CalibratedClassifierCV, peel off the estimator
+ if isinstance(est, CalibratedClassifierCV):
+ try:
+ cc0 = est.calibrated_classifiers_[0]
+ est = cc0.estimator
+ except Exception:
+ return None
+ # Some sklearn/xgboost estimators expose feature_importances_
+ return est if hasattr(est, "feature_importances_") else None
+
+ def get_calibrated_feature_importances(self):
+ """Return averaged feature importances across calibrated folds.
+
+ For CalibratedClassifierCV with tree-based base estimators (RF/GBT/XGBoost),
+ this computes the mean and std of `feature_importances_` across
+ `calibrated_classifiers_` estimators and returns a list of tuples:
+ [(feature_name, mean_importance, std_importance), ...] sorted by mean desc.
+
+ Returns None if unavailable (e.g., non-tree base estimators).
+ """
+ est = self._classifier
+ if not isinstance(est, CalibratedClassifierCV):
+ return None
+ try:
+ base_ests = [cc.estimator for cc in est.calibrated_classifiers_]
+ except Exception:
+ return None
+ base_ests = [be for be in base_ests if hasattr(be, "feature_importances_")]
+ if not base_ests:
+ return None
+ importances = np.vstack([be.feature_importances_ for be in base_ests])
+ mean_imp = importances.mean(axis=0)
+ std_imp = importances.std(axis=0)
+ names = self._feature_names or [f"feature_{i}" for i in range(mean_imp.shape[0])]
+ if len(names) != len(mean_imp):
+ names = [f"feature_{i}" for i in range(len(mean_imp))]
+ items = list(zip(names, mean_imp, std_imp, strict=True))
+ items.sort(key=lambda t: t[1], reverse=True)
+ return items
+
def print_feature_importance(self, feature_list, limit=20):
"""print the most important features and their importance
@@ -596,20 +697,37 @@ def print_feature_importance(self, feature_list, limit=20):
feature_list: list of feature names used in the classifier
limit: maximum number of features to print, defaults to 20
"""
- # Get numerical feature importance
- importances = list(self._classifier.feature_importances_)
- # List of tuples with variable and importance
+ # Prefer calibrated importances if available
+ if isinstance(self._classifier, CalibratedClassifierCV):
+ items = self.get_calibrated_feature_importances()
+ if items is not None:
+ print(f"{'Feature Name':100} Mean Importance Std")
+ print("-" * 120)
+ for name, mean_imp, std_imp in items[:limit]:
+ print(f"{name:100} {mean_imp:0.4f} {std_imp:0.4f}")
+ return
+ # fall through to base-estimator single-source path if calibrated but no importances
+
+ # Fallback: single estimator feature_importances_
+ est = self._get_estimator_with_feature_importances()
+ if est is None:
+ print(
+ "Feature importances are unavailable for the current classifier (e.g., calibrated logistic/linear models)."
+ )
+ return
+ importances = list(est.feature_importances_)
+ names = feature_list if feature_list is not None else (self._feature_names or [])
+ if len(importances) != len(names):
+ names = [f"feature_{i}" for i in range(len(importances))]
feature_importance = [
- (feature, round(importance, 2))
- for feature, importance in zip(feature_list, importances, strict=True)
+ (feature, round(importance, 4))
+ for feature, importance in zip(names, importances, strict=False)
]
- # Sort the feature importance by most important first
feature_importance = sorted(feature_importance, key=lambda x: x[1], reverse=True)
- # Print out the feature and importance
print(f"{'Feature Name':100} Importance")
print("-" * 120)
for feature, importance in feature_importance[:limit]:
- print(f"{feature:100} {importance:0.2f}")
+ print(f"{feature:100} {importance:0.4f}")
@staticmethod
def count_label_threshold(all_counts: dict):
diff --git a/src/jabs/scripts/classify.py b/src/jabs/scripts/classify.py
index cfd38c1e..90478157 100755
--- a/src/jabs/scripts/classify.py
+++ b/src/jabs/scripts/classify.py
@@ -105,7 +105,7 @@ def classify_pose(
prediction_labels = np.full((pose_est.num_identities, pose_est.num_frames), -1, dtype=np.int8)
prediction_prob = np.zeros_like(prediction_labels, dtype=np.float32)
- classifier_settings = classifier.project_settings
+ classifier_settings = classifier.behavior_settings
print(f"Classifying {input_pose_file}...")
@@ -188,7 +188,7 @@ def train(training_file: Path) -> Classifier:
Classifier: The trained classifier instance.
"""
classifier = Classifier.from_training_file(training_file)
- classifier_settings = classifier.project_settings
+ classifier_settings = classifier.behavior_settings
print("Training classifier for:", classifier.behavior_name)
print(f" Classifier Type: {__CLASSIFIER_CHOICES[classifier.classifier_type]}")
@@ -315,7 +315,7 @@ def classify_main():
sys.exit(str(e))
behavior = classifier.behavior_name
- classifier_settings = classifier.project_settings
+ classifier_settings = classifier.behavior_settings
print(f"Classifying using trained classifier: {args.classifier}")
try:
diff --git a/src/jabs/scripts/stats.py b/src/jabs/scripts/stats.py
index 2eda38ba..6d522e99 100644
--- a/src/jabs/scripts/stats.py
+++ b/src/jabs/scripts/stats.py
@@ -124,7 +124,7 @@ def main():
print(f"\nClassifier: {classifier.classifier_name}")
print(f"Behavior: {features['behavior']}")
unit = (
- "cm" if classifier.project_settings["cm_units"] == ProjectDistanceUnit.CM else "pixel"
+ "cm" if classifier.behavior_settings["cm_units"] == ProjectDistanceUnit.CM else "pixel"
)
print(f"Feature Distance Unit: {unit}")
print("-" * 70)
diff --git a/src/jabs/ui/central_widget.py b/src/jabs/ui/central_widget.py
index f1f0fc64..47856de5 100644
--- a/src/jabs/ui/central_widget.py
+++ b/src/jabs/ui/central_widget.py
@@ -655,7 +655,7 @@ def _update_classifier_controls(self) -> None:
self._controls.set_classifier_selection(self._classifier.classifier_type)
# does the classifier match the current settings?
- classifier_settings = self._classifier.project_settings
+ classifier_settings = self._classifier.behavior_settings
if (
classifier_settings is not None
and classifier_settings.get("window_size", None) == self.window_size
diff --git a/src/jabs/ui/main_window.py b/src/jabs/ui/main_window.py
index e4863f8d..0eead518 100644
--- a/src/jabs/ui/main_window.py
+++ b/src/jabs/ui/main_window.py
@@ -20,6 +20,7 @@
from .progress_dialog import create_progress_dialog
from .project_loader_thread import ProjectLoaderThread
from .project_pruning_dialog import ProjectPruningDialog
+from .settings_dialog import JabsSettingsDialog
from .stacked_timeline_widget import StackedTimelineWidget
from .user_guide_dialog import UserGuideDialog
from .util import send_file_to_recycle_bin
@@ -119,6 +120,13 @@ def __init__(self, app_name: str, app_name_long: str, *args, **kwargs) -> None:
self._clear_cache.triggered.connect(self._clear_cache_action)
app_menu.addAction(self._clear_cache)
+ # model calibration settings
+ self._settings_action = QtGui.QAction("JABS Settings", self)
+ self._settings_action.setStatusTip("Open settings dialog")
+ self._settings_action.setEnabled(False)
+ self._settings_action.triggered.connect(self._open_settings_dialog)
+ app_menu.addAction(self._settings_action)
+
# exit action
exit_action = QtGui.QAction(f" &Quit {self._app_name}", self)
exit_action.setShortcut(QtGui.QKeySequence("Ctrl+Q"))
@@ -725,6 +733,7 @@ def _project_loaded_callback(self) -> None:
self._project.feature_manager.can_use_segmentation_features
)
self._clear_cache.setEnabled(True)
+ self._settings_action.setEnabled(True)
available_objects = self._project.feature_manager.static_objects
for static_object, menu_item in self.enable_landmark_features.items():
if static_object in available_objects:
@@ -990,3 +999,8 @@ def _view_license(self) -> None:
"""View the license agreement (JABS->View License Agreement menu action)"""
dialog = LicenseAgreementDialog(self, view_only=True)
dialog.exec_()
+
+ def _open_settings_dialog(self) -> None:
+ """Open the settings dialog (JABS->Settings menu action)"""
+ dialog = JabsSettingsDialog(parent=self, project_settings=self._project.settings_manager)
+ dialog.exec_()
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
new file mode 100644
index 00000000..be5efaa8
--- /dev/null
+++ b/src/jabs/ui/settings_dialog.py
@@ -0,0 +1,262 @@
+from PySide6.QtCore import Qt, Signal
+from PySide6.QtWidgets import (
+ QAbstractScrollArea,
+ QCheckBox,
+ QComboBox,
+ QDialog,
+ QDialogButtonBox,
+ QFrame,
+ QGridLayout,
+ QGroupBox,
+ QLabel,
+ QLayout,
+ QScrollArea,
+ QSizePolicy,
+ QSpacerItem,
+ QSpinBox,
+ QToolButton,
+ QVBoxLayout,
+ QWidget,
+)
+
+from jabs.project.settings_manager import SettingsManager
+
+
+class CollapsibleSection(QWidget):
+ """A simple collapsible section with a header ToolButton and a content area."""
+
+ sizeChanged = Signal()
+
+ def __init__(self, title: str, content: QWidget, parent: QWidget | None = None) -> None:
+ super().__init__(parent)
+ self._content = content
+ self._toggle_btn = QToolButton(self)
+ self._toggle_btn.setStyleSheet("QToolButton { border: none; }")
+ self._toggle_btn.setToolButtonStyle(Qt.ToolButtonStyle.ToolButtonTextBesideIcon)
+ self._toggle_btn.setArrowType(Qt.ArrowType.RightArrow)
+ self._toggle_btn.setText(title)
+ self._toggle_btn.setCheckable(True)
+ self._toggle_btn.setChecked(False)
+ self._toggle_btn.toggled.connect(self._on_toggled)
+
+ line = QFrame(self)
+ line.setFrameShape(QFrame.Shape.HLine)
+ line.setFrameShadow(QFrame.Shadow.Sunken)
+
+ self._content.setVisible(False)
+ # Ensure the collapsible widget and its content expand to fit content
+ self.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
+ self._content.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
+
+ lay = QVBoxLayout(self)
+ lay.setContentsMargins(0, 0, 0, 0)
+ lay.addWidget(self._toggle_btn)
+ lay.addWidget(line)
+ lay.addWidget(self._content)
+
+ def _on_toggled(self, checked: bool) -> None:
+ self._toggle_btn.setArrowType(
+ Qt.ArrowType.DownArrow if checked else Qt.ArrowType.RightArrow
+ )
+ self._content.setVisible(checked)
+ self._content.updateGeometry()
+ # Ask ancestors to recompute layout so the page grows inside the scroll area
+ p = self.parentWidget()
+ if p is not None and p.layout() is not None:
+ p.layout().activate()
+ lg = self.layout()
+ if lg is not None:
+ lg.activate()
+ # Let ancestors recompute size hints and notify listeners
+ pw = self.parentWidget()
+ if pw is not None:
+ pw.updateGeometry()
+ self.updateGeometry()
+ self.sizeChanged.emit()
+
+
+class JabsSettingsDialog(QDialog):
+ """Dialog for changing project settings.
+
+ Parameters
+ ----------
+ project_settings: ProjectSettings
+ Object providing .get(key, default) and either .set(key, value) or item assignment.
+ parent: QWidget | None
+ """
+
+ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = None) -> None:
+ super().__init__(parent)
+ self.setWindowTitle("Project Settings")
+ self._settings_manager = project_settings
+
+ # Allow resizing and show scrollbars if content overflows
+ self.setSizeGripEnabled(True)
+
+ # Widgets
+ self._calibrate_checkbox = QCheckBox(
+ "Enable probability calibration (calibrate_probabilities)"
+ )
+ self._method_selection = QComboBox()
+ self._method_selection.addItems(
+ ["isotonic", "sigmoid"]
+ ) # default will be set from settings
+ self._cv_selection = QSpinBox()
+ self._cv_selection.setRange(2, 10)
+ self._cv_selection.setAccelerated(True)
+ self._cv_selection.setToolTip("Number of CV folds used inside the calibrator")
+
+ # Load current values from project settings (keys must match classifier usage)
+ current_settings = project_settings.project_settings.get("settings", {})
+ calibrate = current_settings.get("calibrate_probabilities", False)
+ method = current_settings.get("calibration_method", "isotonic")
+ cv = current_settings.get("calibration_cv", 3)
+
+ self._calibrate_checkbox.setChecked(calibrate)
+ idx = max(0, self._method_selection.findText(method))
+ self._method_selection.setCurrentIndex(idx)
+ self._cv_selection.setValue(cv)
+
+ # Layout for form
+ form = QWidget(self)
+ grid = QGridLayout(form)
+ grid.setContentsMargins(0, 0, 0, 0)
+ grid.setHorizontalSpacing(12)
+ grid.setVerticalSpacing(8)
+ grid.setColumnStretch(0, 0) # labels column: natural size
+ grid.setColumnStretch(1, 0) # inputs column: natural size
+ grid.setColumnStretch(2, 1) # expanding whitespace to the right
+
+ # Keep inputs compact; whitespace grows in column 2
+ self._method_selection.setSizeAdjustPolicy(QComboBox.SizeAdjustPolicy.AdjustToContents)
+ self._method_selection.setFixedWidth(self._method_selection.sizeHint().width() + 24)
+ self._method_selection.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+
+ self._cv_selection.setFixedWidth(90)
+ self._cv_selection.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+
+ self._calibrate_checkbox.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+
+ grid.addWidget(QLabel("Calibrate probabilities:"), 0, 0, Qt.AlignRight)
+ grid.addWidget(self._calibrate_checkbox, 0, 1)
+
+ grid.addWidget(QLabel("Calibration method:"), 1, 0, Qt.AlignRight)
+ grid.addWidget(self._method_selection, 1, 1)
+
+ grid.addWidget(QLabel("calibration cv (folds):"), 2, 0, Qt.AlignRight)
+ grid.addWidget(self._cv_selection, 2, 1)
+
+ grid.addItem(QSpacerItem(0, 0, QSizePolicy.Expanding, QSizePolicy.Minimum), 0, 2, 3, 1)
+
+ # Help / inline docs (rich text)
+ help_label = QLabel(self)
+ help_label.setTextFormat(Qt.TextFormat.RichText)
+ help_label.setWordWrap(True)
+ help_label.setText(
+ """
+ What do these parameters do?
+ Calibrate probabilities remaps raw model scores to better probabilities, using a small
+ cross-validation inside training. This improves metrics like log-loss and decision thresholds.
+
+
+ - calibration_method:
+ isotonic (default) is a flexible, non-parametric mapping that often yields the best
+ calibration when you have enough data. sigmoid (a.k.a. Platt scaling) is more constrained and
+ can be better on small datasets.
+ - calibration_cv: Number of folds used internally by the calibrator. Typical values are 3-5.
+ Each fold fits the base model and calibrates on held-out training data to avoid leakage.
+
+ Tip: If you see probabilities stuck near 0/1, try enabling calibration.
+ """
+ )
+ help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
+
+ calibration_help_panel = CollapsibleSection("What do these do?", help_label, self)
+ calibration_help_panel.setSizePolicy(
+ QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred
+ )
+ help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
+ _help_toggle_btn = (
+ calibration_help_panel._toggle_btn
+ ) # internal; used to wire scrolling after scroll area is created
+
+ # When the collapsible expands/collapses, grow the group box to fit content and leave scrolling to the dialog
+ def _reflow_calibration_group():
+ calibration_group.adjustSize()
+ page_layout.activate()
+ scroll.ensureWidgetVisible(calibration_help_panel)
+
+ calibration_help_panel.sizeChanged.connect(_reflow_calibration_group)
+
+ # Group box for Model Calibration section
+ calibration_group = QGroupBox("Model Calibration", self)
+ calibration_group.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Preferred)
+ group_vbox = QVBoxLayout(calibration_group)
+ group_vbox.setContentsMargins(12, 12, 12, 12)
+ group_vbox.setSpacing(8)
+ group_vbox.addWidget(form)
+ group_vbox.addWidget(calibration_help_panel)
+ group_vbox.addStretch(0)
+
+ # Scrollable page to host settings sections
+ page = QWidget(self)
+ page.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Preferred)
+ page_layout = QVBoxLayout(page)
+ page_layout.setSizeConstraint(QLayout.SizeConstraint.SetMinAndMaxSize)
+ page_layout.setContentsMargins(0, 0, 0, 0)
+ page_layout.setSpacing(10)
+ page_layout.addWidget(calibration_group)
+ page_layout.setAlignment(calibration_group, Qt.AlignTop)
+ page_layout.addStretch(1)
+
+ scroll = QScrollArea(self)
+ scroll.setWidget(page)
+ scroll.setWidgetResizable(True)
+ scroll.setFrameShape(QFrame.Shape.NoFrame)
+ scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
+ scroll.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
+ scroll.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
+ # Keep content width constant: viewport gutter above matches scrollbar width
+ scroll.setSizeAdjustPolicy(QAbstractScrollArea.SizeAdjustPolicy.AdjustToContents)
+
+ # When the help section is expanded, ensure it is visible within the scroll area
+ try:
+
+ def _on_help_toggled(checked: bool) -> None:
+ if checked:
+ # Only scroll; don't resize caps — keeps scrollbar on the window only
+ scroll.ensureWidgetVisible(calibration_help_panel)
+
+ _help_toggle_btn.toggled.connect(_on_help_toggled)
+ except Exception:
+ pass
+
+ # Buttons
+ btn_box = QDialogButtonBox(self)
+ btn_save = btn_box.addButton("Save", QDialogButtonBox.ButtonRole.AcceptRole)
+ btn_close = btn_box.addButton("Close", QDialogButtonBox.ButtonRole.RejectRole)
+ btn_save.clicked.connect(self._on_save)
+ btn_close.clicked.connect(self.reject)
+
+ # Main layout
+ main = QVBoxLayout(self)
+ main.addWidget(scroll, 1)
+ main.addWidget(btn_box)
+
+ self.setLayout(main)
+
+ # Size to content initially, then give a taller starting height; user-resize preserved later
+ self.adjustSize()
+ self.resize(max(self.width(), 720), max(self.height(), 1000))
+
+ def _on_save(self) -> None:
+ # Persist back to project settings
+ settings = {
+ "settings": {
+ "calibrate_probabilities": self._calibrate_checkbox.isChecked(),
+ "calibration_method": self._method_selection.currentText(),
+ "calibration_cv": self._cv_selection.value(),
+ }
+ }
+ self._settings_manager.save_project_file(settings)
+ self.accept()
From e4041d0de5017d1ebc0b6b2e66591abd81353c7e Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 16 Oct 2025 08:29:28 -0400
Subject: [PATCH 02/14] fix some Qt flag usage
---
src/jabs/ui/settings_dialog.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index be5efaa8..95b4e40d 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -130,23 +130,25 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
# Keep inputs compact; whitespace grows in column 2
self._method_selection.setSizeAdjustPolicy(QComboBox.SizeAdjustPolicy.AdjustToContents)
self._method_selection.setFixedWidth(self._method_selection.sizeHint().width() + 24)
- self._method_selection.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+ self._method_selection.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
self._cv_selection.setFixedWidth(90)
- self._cv_selection.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+ self._cv_selection.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
- self._calibrate_checkbox.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+ self._calibrate_checkbox.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
- grid.addWidget(QLabel("Calibrate probabilities:"), 0, 0, Qt.AlignRight)
+ grid.addWidget(QLabel("Calibrate probabilities:"), 0, 0, Qt.AlignmentFlag.AlignRight)
grid.addWidget(self._calibrate_checkbox, 0, 1)
- grid.addWidget(QLabel("Calibration method:"), 1, 0, Qt.AlignRight)
+ grid.addWidget(QLabel("Calibration method:"), 1, 0, Qt.AlignmentFlag.AlignRight)
grid.addWidget(self._method_selection, 1, 1)
- grid.addWidget(QLabel("calibration cv (folds):"), 2, 0, Qt.AlignRight)
+ grid.addWidget(QLabel("calibration cv (folds):"), 2, 0, Qt.AlignmentFlag.AlignRight)
grid.addWidget(self._cv_selection, 2, 1)
- grid.addItem(QSpacerItem(0, 0, QSizePolicy.Expanding, QSizePolicy.Minimum), 0, 2, 3, 1)
+ grid.addItem(
+ QSpacerItem(0, 0, QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum), 0, 2, 3, 1
+ )
# Help / inline docs (rich text)
help_label = QLabel(self)
@@ -206,7 +208,7 @@ def _reflow_calibration_group():
page_layout.setContentsMargins(0, 0, 0, 0)
page_layout.setSpacing(10)
page_layout.addWidget(calibration_group)
- page_layout.setAlignment(calibration_group, Qt.AlignTop)
+ page_layout.setAlignment(calibration_group, Qt.AlignmentFlag.AlignTop)
page_layout.addStretch(1)
scroll = QScrollArea(self)
From 76b3bdf9224472a4c194dc69d26b3a91f792f1e4 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 16 Oct 2025 08:45:14 -0400
Subject: [PATCH 03/14] some refactoring in classifier.py
---
src/jabs/classifier/classifier.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index df66ca01..02b81798 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -670,23 +670,25 @@ def get_calibrated_feature_importances(self):
Returns None if unavailable (e.g., non-tree base estimators).
"""
- est = self._classifier
- if not isinstance(est, CalibratedClassifierCV):
+ if not isinstance(self._classifier, CalibratedClassifierCV):
return None
try:
- base_ests = [cc.estimator for cc in est.calibrated_classifiers_]
+ base_ests = [cc.estimator for cc in self._classifier.calibrated_classifiers_]
except Exception:
return None
+
+ # get the base estimators that have feature_importances_
base_ests = [be for be in base_ests if hasattr(be, "feature_importances_")]
if not base_ests:
return None
+
+ # get the mean and standard deviation of feature importances from the base estimators
importances = np.vstack([be.feature_importances_ for be in base_ests])
mean_imp = importances.mean(axis=0)
std_imp = importances.std(axis=0)
- names = self._feature_names or [f"feature_{i}" for i in range(mean_imp.shape[0])]
- if len(names) != len(mean_imp):
- names = [f"feature_{i}" for i in range(len(mean_imp))]
- items = list(zip(names, mean_imp, std_imp, strict=True))
+
+ # combine with feature names and sort by mean importance
+ items = list(zip(self._feature_names, mean_imp, std_imp, strict=True))
items.sort(key=lambda t: t[1], reverse=True)
return items
@@ -701,19 +703,17 @@ def print_feature_importance(self, feature_list, limit=20):
if isinstance(self._classifier, CalibratedClassifierCV):
items = self.get_calibrated_feature_importances()
if items is not None:
- print(f"{'Feature Name':100} Mean Importance Std")
+ print(f"{'Feature Name':100} Mean Importance Std")
print("-" * 120)
for name, mean_imp, std_imp in items[:limit]:
- print(f"{name:100} {mean_imp:0.4f} {std_imp:0.4f}")
+ print(f"{name:100} {mean_imp:0.4f} {std_imp:0.4f}")
return
# fall through to base-estimator single-source path if calibrated but no importances
# Fallback: single estimator feature_importances_
est = self._get_estimator_with_feature_importances()
if est is None:
- print(
- "Feature importances are unavailable for the current classifier (e.g., calibrated logistic/linear models)."
- )
+ print("Feature importances are unavailable for the current classifier.")
return
importances = list(est.feature_importances_)
names = feature_list if feature_list is not None else (self._feature_names or [])
From 3fe7536cf87d2309fdafa1c32ed2fb7011b2a59c Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 16 Oct 2025 09:35:06 -0400
Subject: [PATCH 04/14] refactoring in JabsSettingsDialog
---
src/jabs/ui/settings_dialog.py | 50 ++++++++++++++++------------------
1 file changed, 23 insertions(+), 27 deletions(-)
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index 95b4e40d..e103fd1e 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -60,29 +60,29 @@ def _on_toggled(self, checked: bool) -> None:
)
self._content.setVisible(checked)
self._content.updateGeometry()
+
# Ask ancestors to recompute layout so the page grows inside the scroll area
- p = self.parentWidget()
- if p is not None and p.layout() is not None:
- p.layout().activate()
- lg = self.layout()
- if lg is not None:
- lg.activate()
+ parent = self.parentWidget()
+ if parent is not None and parent.layout() is not None:
+ parent.layout().activate()
+
+ if self.layout() is not None:
+ self.layout().activate()
+
# Let ancestors recompute size hints and notify listeners
- pw = self.parentWidget()
- if pw is not None:
- pw.updateGeometry()
+ if parent is not None:
+ parent.updateGeometry()
self.updateGeometry()
self.sizeChanged.emit()
class JabsSettingsDialog(QDialog):
- """Dialog for changing project settings.
+ """
+ Dialog for changing project settings.
- Parameters
- ----------
- project_settings: ProjectSettings
- Object providing .get(key, default) and either .set(key, value) or item assignment.
- parent: QWidget | None
+ Args:
+ project_settings (SettingsManager): Project settings manager used to load and save settings.
+ parent (QWidget | None, optional): Parent widget for this dialog. Defaults to None.
"""
def __init__(self, project_settings: SettingsManager, parent: QWidget | None = None) -> None:
@@ -218,20 +218,16 @@ def _reflow_calibration_group():
scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
scroll.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
scroll.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
+
# Keep content width constant: viewport gutter above matches scrollbar width
scroll.setSizeAdjustPolicy(QAbstractScrollArea.SizeAdjustPolicy.AdjustToContents)
- # When the help section is expanded, ensure it is visible within the scroll area
- try:
-
- def _on_help_toggled(checked: bool) -> None:
- if checked:
- # Only scroll; don't resize caps — keeps scrollbar on the window only
- scroll.ensureWidgetVisible(calibration_help_panel)
+ def _on_help_toggled(checked: bool) -> None:
+ if checked:
+ # Only scroll; don't resize caps — keeps scrollbar on the window only
+ scroll.ensureWidgetVisible(calibration_help_panel)
- _help_toggle_btn.toggled.connect(_on_help_toggled)
- except Exception:
- pass
+ _help_toggle_btn.toggled.connect(_on_help_toggled)
# Buttons
btn_box = QDialogButtonBox(self)
@@ -249,10 +245,10 @@ def _on_help_toggled(checked: bool) -> None:
# Size to content initially, then give a taller starting height; user-resize preserved later
self.adjustSize()
- self.resize(max(self.width(), 720), max(self.height(), 1000))
+ self.resize(max(self.width(), 700), max(self.height(), 600))
def _on_save(self) -> None:
- # Persist back to project settings
+ """Save settings to project and close dialog."""
settings = {
"settings": {
"calibrate_probabilities": self._calibrate_checkbox.isChecked(),
From 0fe5d90c2e0f1136852a375a9f921397ebfe7b80 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 17 Oct 2025 14:52:47 -0400
Subject: [PATCH 05/14] save the probability calibration settings in exported
training data
---
src/jabs/classifier/classifier.py | 29 +++++++++++++++-----------
src/jabs/constants.py | 3 +++
src/jabs/project/export_training.py | 12 +++++++++++
src/jabs/project/prediction_manager.py | 2 +-
src/jabs/project/project.py | 7 +++++--
src/jabs/project/read_training.py | 19 ++++++++++++++++-
src/jabs/project/settings_manager.py | 11 +++++++++-
src/jabs/project/video_manager.py | 2 +-
src/jabs/ui/central_widget.py | 2 +-
src/jabs/ui/main_window.py | 2 +-
10 files changed, 69 insertions(+), 20 deletions(-)
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index 02b81798..c84e97a3 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -18,6 +18,7 @@
)
from sklearn.model_selection import LeaveOneGroupOut, train_test_split
+from jabs.constants import DEFAULT_CALIBRATION_CV, DEFAULT_CALIBRATION_METHOD
from jabs.project import Project, TrackLabels, load_training_data
from jabs.types import ClassifierType
from jabs.utils import hash_file
@@ -62,7 +63,7 @@ def __init__(self, classifier=ClassifierType.RANDOM_FOREST, n_jobs=1):
self._classifier_type = classifier
self._classifier = None
self._behavior_settings = None
- self._project_settings = None
+ self._jabs_settings = None
self._behavior = None
self._feature_names = None
self._n_jobs = n_jobs
@@ -93,7 +94,9 @@ def from_training_file(cls, path: Path):
classifier = cls()
classifier.behavior_name = behavior
- classifier.set_dict_settings(loaded_training_data["settings"])
+ classifier.set_behavior_settings(loaded_training_data["behavior_settings"])
+ classifier._jabs_settings = loaded_training_data["jabs_settings"]
+
classifier_type = ClassifierType(loaded_training_data["classifier_type"])
if classifier_type in classifier.classifier_choices():
classifier.set_classifier(classifier_type)
@@ -101,6 +104,7 @@ def from_training_file(cls, path: Path):
print(
f"Specified classifier type {classifier_type.name} is unavailable, using default: {classifier.classifier_type.name}"
)
+
training_features = classifier.combine_data(
loaded_training_data["per_frame"], loaded_training_data["window"]
)
@@ -367,9 +371,10 @@ def set_project_settings(self, project: Project):
else:
self._behavior_settings = project.settings_manager.get_behavior(self._behavior)
- self._project_settings = project.settings_manager.project_settings.get("settings", {})
+ # grab other JABS settings from settings manager, some might be used by the classifier
+ self._jabs_settings = project.settings_manager.settings
- def set_dict_settings(self, settings: dict):
+ def set_behavior_settings(self, settings: dict):
"""assign behavior-specific settings via a dict to the classifier
Args:
@@ -425,14 +430,14 @@ def train(self, data, random_seed: int | None = None):
if self._behavior_settings.get("balance_labels", False):
features, labels = self.downsample_balance(features, labels, random_seed)
- # Optional probability calibration, this is currently set at the project level
- calibrate = self._project_settings.get("calibrate_probabilities", False)
- calibration_method = self._project_settings.get(
- "calibration_method", "isotonic"
- ) # or 'sigmoid'
- calibration_cv = self._project_settings.get("calibration_cv", 3)
+ # Optional probability calibration
+ calibrate_probabilities = self._jabs_settings.get("calibrate_probabilities", False)
+ if calibrate_probabilities:
+ calibration_method = self._jabs_settings.get(
+ "calibration_method", DEFAULT_CALIBRATION_METHOD
+ )
+ calibration_cv = self._jabs_settings.get("calibration_cv", DEFAULT_CALIBRATION_CV)
- if calibrate:
# Build an unfitted base estimator
if self._classifier_type == ClassifierType.RANDOM_FOREST:
base_estimator = self._make_random_forest(random_seed=random_seed)
@@ -581,7 +586,7 @@ def load(self, path: Path):
self._classifier = c._classifier
self._behavior = c._behavior
self._behavior_settings = c._behavior_settings
- self._project_settings = c._project_settings
+ self._jabs_settings = c._jabs_settings
self._classifier_type = c._classifier_type
if c._classifier_file is not None:
self._classifier_file = c._classifier_file
diff --git a/src/jabs/constants.py b/src/jabs/constants.py
index a4a585d0..80c0e853 100644
--- a/src/jabs/constants.py
+++ b/src/jabs/constants.py
@@ -8,3 +8,6 @@
# some defaults for compressing hdf5 output
COMPRESSION = "gzip"
COMPRESSION_OPTS_DEFAULT = 6
+
+DEFAULT_CALIBRATION_METHOD = "isotonic"
+DEFAULT_CALIBRATION_CV = 3
diff --git a/src/jabs/project/export_training.py b/src/jabs/project/export_training.py
index 3565454d..12248fae 100644
--- a/src/jabs/project/export_training.py
+++ b/src/jabs/project/export_training.py
@@ -7,6 +7,7 @@
import jabs.feature_extraction
import jabs.version
+from jabs.constants import DEFAULT_CALIBRATION_CV, DEFAULT_CALIBRATION_METHOD
from jabs.project.project_utils import to_safe_name
from jabs.utils import FINAL_TRAIN_SEED
@@ -64,6 +65,17 @@ def export_training_data(
write_project_settings(out_h5, project.settings_manager.get_behavior(behavior), "settings")
out_h5.attrs["classifier_type"] = classifier_type.value
out_h5.attrs["training_seed"] = training_seed
+ out_h5.attrs["calibrate_probabilities"] = project.settings_manager.settings.get(
+ "calibrate_probabilities", False
+ )
+ if out_h5.attrs["calibrate_probabilities"]:
+ out_h5.attrs["calibration_method"] = project.settings_manager.settings.get(
+ "calibration_method", DEFAULT_CALIBRATION_METHOD
+ )
+ out_h5.attrs["calibration_cv"] = project.settings_manager.settings.get(
+ "calibration_cv", DEFAULT_CALIBRATION_CV
+ )
+
feature_group = out_h5.create_group("features")
for feature, data in features["per_frame"].items():
feature_group.create_dataset(f"per_frame/{feature}", data=data)
diff --git a/src/jabs/project/prediction_manager.py b/src/jabs/project/prediction_manager.py
index af3407d3..d4e63caa 100644
--- a/src/jabs/project/prediction_manager.py
+++ b/src/jabs/project/prediction_manager.py
@@ -126,7 +126,7 @@ def load_predictions(self, video: str, behavior: str):
file_base = Path(video).with_suffix("").name + ".h5"
path = self._project.project_paths.prediction_dir / file_base
- nident = self._project.settings_manager.project_settings["video_files"][video][
+ nident = self._project.settings_manager.project_dictionary["video_files"][video][
"identities"
]
diff --git a/src/jabs/project/project.py b/src/jabs/project/project.py
index 03abd2ec..b84cc474 100644
--- a/src/jabs/project/project.py
+++ b/src/jabs/project/project.py
@@ -69,7 +69,10 @@ def __init__(
self._session_tracker = SessionTracker(self, tracking_enabled=enable_session_tracker)
# write out the defaults to the project file
- if self._settings_manager.project_settings.get("defaults") != self.get_project_defaults():
+ if (
+ self._settings_manager.project_dictionary.get("defaults")
+ != self.get_project_defaults()
+ ):
self._settings_manager.save_project_file({"defaults": self.get_project_defaults()})
# Start a session tracker for this project.
@@ -110,7 +113,7 @@ def classifier_dir(self):
@property
def settings(self):
"""get the project metadata and preferences."""
- return self._settings_manager.project_settings
+ return self._settings_manager.project_dictionary
@property
def settings_manager(self) -> SettingsManager:
diff --git a/src/jabs/project/read_training.py b/src/jabs/project/read_training.py
index 5e5ed492..3a6bdbc7 100644
--- a/src/jabs/project/read_training.py
+++ b/src/jabs/project/read_training.py
@@ -4,6 +4,7 @@
import h5py
import pandas as pd
+from jabs.constants import DEFAULT_CALIBRATION_CV, DEFAULT_CALIBRATION_METHOD
from jabs.types import ClassifierType, ProjectDistanceUnit
@@ -79,9 +80,10 @@ def load_training_data(training_file: Path):
with h5py.File(training_file, "r") as in_h5:
features["min_pose_version"] = in_h5.attrs["min_pose_version"]
features["behavior"] = in_h5.attrs["behavior"]
- features["settings"] = read_project_settings(in_h5["settings"])
+ features["behavior_settings"] = read_project_settings(in_h5["settings"])
features["training_seed"] = in_h5.attrs["training_seed"]
features["classifier_type"] = ClassifierType(in_h5.attrs["classifier_type"])
+
# convert the string distance_unit attr to corresponding
# ProjectDistanceUnit enum
unit = in_h5.attrs.get("distance_unit")
@@ -92,6 +94,21 @@ def load_training_data(training_file: Path):
else:
features["distance_unit"] = ProjectDistanceUnit[unit]
+ features["jabs_settings"] = {}
+
+ # load other jabs settings that might or might not be present
+ calibrate_probabilities = in_h5.attrs.get("calibrate_probabilities", False)
+ if calibrate_probabilities:
+ features["jabs_settings"].extend(
+ {
+ "calibrate_probabilities": calibrate_probabilities,
+ "calibration_method": in_h5.attrs.get(
+ "calibration_method", DEFAULT_CALIBRATION_METHOD
+ ),
+ "calibration_cv": in_h5.attrs.get("calibration_cv", DEFAULT_CALIBRATION_CV),
+ }
+ )
+
features["labels"] = in_h5["label"][:]
features["groups"] = in_h5["group"][:]
diff --git a/src/jabs/project/settings_manager.py b/src/jabs/project/settings_manager.py
index d6aaeaf6..f5ddb031 100644
--- a/src/jabs/project/settings_manager.py
+++ b/src/jabs/project/settings_manager.py
@@ -57,7 +57,7 @@ def save_project_file(self, data: dict | None = None):
tmp.replace(self._paths.project_file)
@property
- def project_settings(self) -> dict:
+ def project_dictionary(self) -> dict:
"""Get a copy of the current project properties and settings.
Returns:
@@ -65,6 +65,15 @@ def project_settings(self) -> dict:
"""
return dict(self._project_info)
+ @property
+ def settings(self) -> dict:
+ """Get a copy of general JABS settings from project file
+
+ Returns:
+ dict
+ """
+ return dict(self._project_info.get("settings", {}))
+
@property
def behavior_names(self) -> list[str]:
"""Get a list of all behaviors defined in the project settings.
diff --git a/src/jabs/project/video_manager.py b/src/jabs/project/video_manager.py
index e02ef73d..60e451e3 100644
--- a/src/jabs/project/video_manager.py
+++ b/src/jabs/project/video_manager.py
@@ -142,7 +142,7 @@ def get_video_identity_count(self, video_name: str) -> int:
def _load_video_metadata(self):
"""Load metadata for each video and calculate total identities."""
- video_metadata = self._settings_manager.project_settings.get("video_files", {})
+ video_metadata = self._settings_manager.project_dictionary.get("video_files", {})
flush = False
for video in self._videos:
vinfo = video_metadata.get(video, {})
diff --git a/src/jabs/ui/central_widget.py b/src/jabs/ui/central_widget.py
index 47856de5..39015d5f 100644
--- a/src/jabs/ui/central_widget.py
+++ b/src/jabs/ui/central_widget.py
@@ -675,7 +675,7 @@ def _train_button_clicked(self) -> None:
# make sure video playback is stopped
self._player_widget.stop()
- # setup training thread
+ # setup training thread, training thread will configure self._classifier with current settings
self._training_thread = TrainingThread(
self._classifier,
self._project,
diff --git a/src/jabs/ui/main_window.py b/src/jabs/ui/main_window.py
index 0eead518..aaa92766 100644
--- a/src/jabs/ui/main_window.py
+++ b/src/jabs/ui/main_window.py
@@ -532,7 +532,7 @@ def behavior_label_add_event(self, behaviors: list[str]) -> None:
"""handle project updates required when user adds new behavior labels"""
# check for new behaviors
for behavior in behaviors:
- if behavior not in self._project.settings_manager.project_settings["behavior"]:
+ if behavior not in self._project.settings_manager.project_dictionary["behavior"]:
# save new behavior with default settings
self._project.settings_manager.save_behavior(behavior, {})
From 4b9d258cb42b1242d7f74ec46af0950a17f7da11 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 17 Oct 2025 17:17:39 -0400
Subject: [PATCH 06/14] finalizing jabs settings dialog and classifier
probability calibration
---
src/jabs/behavior_search/behavior_search_util.py | 2 +-
src/jabs/classifier/classifier.py | 10 +++++-----
src/jabs/scripts/classify.py | 1 +
src/jabs/scripts/initialize_project.py | 2 +-
src/jabs/ui/settings_dialog.py | 7 ++++---
5 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/src/jabs/behavior_search/behavior_search_util.py b/src/jabs/behavior_search/behavior_search_util.py
index 5334b5ac..b9a11eb6 100644
--- a/src/jabs/behavior_search/behavior_search_util.py
+++ b/src/jabs/behavior_search/behavior_search_util.py
@@ -114,7 +114,7 @@ def _search_behaviors_gen(
)
case PredictionBehaviorSearchQuery() as pred_query:
- proj_settings = project.settings_manager.project_settings
+ proj_settings = project.settings_manager.project_dictionary
if pred_query.behavior_label is None:
behavior_dict = proj_settings.get("behavior", {})
behaviors = list(behavior_dict.keys())
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index c84e97a3..873cb770 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -32,11 +32,11 @@
# we were able to import xgboost, make it available as an option:
_classifier_choices.append(ClassifierType.XGBOOST)
except Exception:
- # we were unable to import the xgboost module. It's either not
- # installed (it should be if the user used our requirements-old.txt)
- # or it may have been unable to be imported due to a missing
- # libomp. Either way, we won't add it to the available choices and
- # we can otherwise ignore this exception
+ # we were unable to import the xgboost module -- possibly due to a missing
+ # libomp (which is not available by default on macOS). Mac users should
+ # install libomp via Homebrew (brew install libomp) to enable XGBoost support (this is
+ # detailed in the installation instructions).
+ # we won't add it to the available choices and we can otherwise ignore this exception
_xgboost = None
diff --git a/src/jabs/scripts/classify.py b/src/jabs/scripts/classify.py
index 90478157..319cacf8 100755
--- a/src/jabs/scripts/classify.py
+++ b/src/jabs/scripts/classify.py
@@ -197,6 +197,7 @@ def train(training_file: Path) -> Classifier:
print(f" Balanced Labels: {classifier_settings['balance_labels']}")
print(f" Symmetric Behavior: {classifier_settings['symmetric_behavior']}")
print(f" CM Units: {bool(classifier_settings['cm_units'])}")
+ print(f" Calibrate Probabilities: {classifier.calibrate_probabilities}")
return classifier
diff --git a/src/jabs/scripts/initialize_project.py b/src/jabs/scripts/initialize_project.py
index d67cd9b5..f1f7db4e 100755
--- a/src/jabs/scripts/initialize_project.py
+++ b/src/jabs/scripts/initialize_project.py
@@ -295,7 +295,7 @@ def validation_job_producer():
# save window sizes to project settings
deduped_window_sizes = set(
- project.settings_manager.project_settings.get("window_sizes", []) + window_sizes
+ project.settings_manager.project_dictionary.get("window_sizes", []) + window_sizes
)
project.settings_manager.save_project_file({"window_sizes": list(deduped_window_sizes)})
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index e103fd1e..7b0c0e08 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -19,6 +19,7 @@
QWidget,
)
+from jabs.constants import DEFAULT_CALIBRATION_CV, DEFAULT_CALIBRATION_METHOD
from jabs.project.settings_manager import SettingsManager
@@ -107,10 +108,10 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
self._cv_selection.setToolTip("Number of CV folds used inside the calibrator")
# Load current values from project settings (keys must match classifier usage)
- current_settings = project_settings.project_settings.get("settings", {})
+ current_settings = project_settings.project_dictionary.get("settings", {})
calibrate = current_settings.get("calibrate_probabilities", False)
- method = current_settings.get("calibration_method", "isotonic")
- cv = current_settings.get("calibration_cv", 3)
+ method = current_settings.get("calibration_method", DEFAULT_CALIBRATION_METHOD)
+ cv = current_settings.get("calibration_cv", DEFAULT_CALIBRATION_CV)
self._calibrate_checkbox.setChecked(calibrate)
idx = max(0, self._method_selection.findText(method))
From 34cf9eb19bfa69c9299741c84d9ccc1d309876c1 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 17 Oct 2025 17:41:45 -0400
Subject: [PATCH 07/14] edit help text
---
src/jabs/ui/settings_dialog.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index 7b0c0e08..56077d4f 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -169,6 +169,7 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
calibration_cv: Number of folds used internally by the calibrator. Typical values are 3-5.
Each fold fits the base model and calibrates on held-out training data to avoid leakage.
+ Isotonic is recommended, since it should work well in most cases (>~2000 labeled samples).
Tip: If you see probabilities stuck near 0/1, try enabling calibration.
"""
)
From d56f9c7b9ae649eb5db902273d48eef8e8db52c8 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 17 Oct 2025 21:14:27 -0400
Subject: [PATCH 08/14] fix unittests
---
tests/project/test_prediction_manager.py | 2 +-
tests/test_behavior_search.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/project/test_prediction_manager.py b/tests/project/test_prediction_manager.py
index 9f6b3016..899c2268 100644
--- a/tests/project/test_prediction_manager.py
+++ b/tests/project/test_prediction_manager.py
@@ -25,7 +25,7 @@ class MockSettingsManager:
"""Class to simulate project settings."""
def __init__(self):
- self.project_settings = {"video_files": {"test_video.avi": {"identities": 2}}}
+ self.project_dictionary = {"video_files": {"test_video.avi": {"identities": 2}}}
@pytest.fixture
diff --git a/tests/test_behavior_search.py b/tests/test_behavior_search.py
index 6ffbe152..2c22dd60 100644
--- a/tests/test_behavior_search.py
+++ b/tests/test_behavior_search.py
@@ -40,7 +40,7 @@ def load_predictions(video, behavior):
# SettingsManager mock
settings_manager = MagicMock()
- settings_manager.project_settings = project_settings or {}
+ settings_manager.project_dictionary = project_settings or {}
# Project mock
project = MagicMock()
From 745288d2830de7b51eb5d51f33be8de9f2950d33 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 17 Oct 2025 21:16:49 -0400
Subject: [PATCH 09/14] fix typo in docstring
---
tests/test_behavior_search.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test_behavior_search.py b/tests/test_behavior_search.py
index 2c22dd60..c99902ca 100644
--- a/tests/test_behavior_search.py
+++ b/tests/test_behavior_search.py
@@ -51,7 +51,7 @@ def load_predictions(video, behavior):
def _correct_probs(preds, probs):
- """Move 0.0 - 1.0 probaabilities into 0.5 - 1.0 range based on predictions."""
+ """Move 0.0 - 1.0 probabilities into 0.5 - 1.0 range based on predictions."""
probs = probs.copy()
probs[preds == 0] = 1.0 - probs[preds == 0]
From 876f48251991e3a39fa5eed5270c011bef707166 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Mon, 20 Oct 2025 10:15:10 -0400
Subject: [PATCH 10/14] update inline help text
---
src/jabs/ui/settings_dialog.py | 21 ++++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index 56077d4f..6a884766 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -159,18 +159,21 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
"""
What do these parameters do?
Calibrate probabilities remaps raw model scores to better probabilities, using a small
- cross-validation inside training. This improves metrics like log-loss and decision thresholds.
-
+ cross-validation inside training. This improves metrics like log-loss and decision thresholds.
- calibration_method:
- isotonic (default) is a flexible, non-parametric mapping that often yields the best
- calibration when you have enough data. sigmoid (a.k.a. Platt scaling) is more constrained and
- can be better on small datasets.
- - calibration_cv: Number of folds used internally by the calibrator. Typical values are 3-5.
- Each fold fits the base model and calibrates on held-out training data to avoid leakage.
+ isotonic (recommended) works best when you have at least ~2000 labeled samples
+ (ideally balanced). It learns a flexible mapping and produces very accurate probabilities.
+ sigmoid (Platt scaling) is safer for smaller datasets (below ~2000 labels) and is less likely
+ to overfit on limited data.
+ - calibration_cv: Number of folds used internally by the calibrator. Typical values are 3-5.
+ Each fold fits the base model and calibrates on held-out training data to avoid leakage.
+ Larger values slow training without meaningful benefit.
- Isotonic is recommended, since it should work well in most cases (>~2000 labeled samples).
- Tip: If you see probabilities stuck near 0/1, try enabling calibration.
+ When to use isotonic: If you have thousands of labeled frames and probabilities seem too extreme
+ (many near 0 or 1), isotonic will give smoother and more realistic confidence scores.
+ Tip: If training is slow or your dataset is small, start with sigmoid and
+ calibration_cv = 3. You can switch to isotonic later as your dataset grows.
"""
)
help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
From 55cf0944aabe0dd567e4887d8fa73eb489c22786 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 23 Oct 2025 13:56:47 -0400
Subject: [PATCH 11/14] compute Brier score, avoid calling both predict and
predict_proba
---
src/jabs/classifier/classifier.py | 30 +++++++++++++++++++++++--
src/jabs/feature_extraction/features.py | 2 +-
src/jabs/project/read_training.py | 2 +-
src/jabs/scripts/classify.py | 10 ++++-----
src/jabs/ui/classification_thread.py | 27 +++++++++-------------
src/jabs/ui/training_thread.py | 14 +++++++++---
6 files changed, 57 insertions(+), 28 deletions(-)
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index 873cb770..0738573d 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -13,6 +13,7 @@
from sklearn.exceptions import InconsistentVersionWarning
from sklearn.metrics import (
accuracy_score,
+ brier_score_loss,
confusion_matrix,
precision_recall_fscore_support,
)
@@ -52,6 +53,7 @@ class Classifier:
"""
LABEL_THRESHOLD = 20
+ TRUE_THRESHOLD = 0.5
_CLASSIFIER_NAMES: typing.ClassVar[dict] = {
ClassifierType.RANDOM_FOREST: "Random Forest",
@@ -173,6 +175,13 @@ def feature_names(self) -> list:
"""returns the list of feature names used when training this classifier"""
return self._feature_names
+ @property
+ def calibrate_probabilities(self) -> bool:
+ """return whether the classifier is set to calibrate probabilities"""
+ if self._jabs_settings is not None:
+ return self._jabs_settings.get("calibrate_probabilities", False)
+ return False
+
@staticmethod
def train_test_split(per_frame_features, window_features, label_data):
"""split features and labels into training and test datasets
@@ -431,8 +440,7 @@ def train(self, data, random_seed: int | None = None):
features, labels = self.downsample_balance(features, labels, random_seed)
# Optional probability calibration
- calibrate_probabilities = self._jabs_settings.get("calibrate_probabilities", False)
- if calibrate_probabilities:
+ if self.calibrate_probabilities:
calibration_method = self._jabs_settings.get(
"calibration_method", DEFAULT_CALIBRATION_METHOD
)
@@ -622,6 +630,24 @@ def confusion_matrix(truth, predictions):
"""return the confusion matrix using sklearn's confusion_matrix function"""
return confusion_matrix(truth, predictions)
+ @staticmethod
+ def brier_score(truth, proba):
+ """Return the Brier score (lower is better).
+
+ Args:
+ truth: array-like of true binary labels (0/1).
+ proba: array of predicted probabilities for the positive class; can be shape (n_samples,)
+ or a (n_samples, 2) array from `predict_proba`.
+
+ Returns:
+ float Brier score.
+ """
+ proba = np.asarray(proba)
+ if proba.ndim == 2:
+ # assume columns [P(neg), P(pos)] as returned by predict_proba
+ proba = proba[:, 1]
+ return brier_score_loss(truth, proba)
+
@staticmethod
def combine_data(per_frame, window):
"""combine feature sets together
diff --git a/src/jabs/feature_extraction/features.py b/src/jabs/feature_extraction/features.py
index 5991a7f2..98f46da4 100644
--- a/src/jabs/feature_extraction/features.py
+++ b/src/jabs/feature_extraction/features.py
@@ -85,7 +85,7 @@ class IdentityFeatures:
def __init__(
self,
- source_file: str,
+ source_file: str | Path,
identity: int,
directory: str | Path | None,
pose_est: PoseEstimation,
diff --git a/src/jabs/project/read_training.py b/src/jabs/project/read_training.py
index 3a6bdbc7..0cdbf303 100644
--- a/src/jabs/project/read_training.py
+++ b/src/jabs/project/read_training.py
@@ -99,7 +99,7 @@ def load_training_data(training_file: Path):
# load other jabs settings that might or might not be present
calibrate_probabilities = in_h5.attrs.get("calibrate_probabilities", False)
if calibrate_probabilities:
- features["jabs_settings"].extend(
+ features["jabs_settings"].update(
{
"calibrate_probabilities": calibrate_probabilities,
"calibration_method": in_h5.attrs.get(
diff --git a/src/jabs/scripts/classify.py b/src/jabs/scripts/classify.py
index 319cacf8..d8e4a94c 100755
--- a/src/jabs/scripts/classify.py
+++ b/src/jabs/scripts/classify.py
@@ -137,13 +137,13 @@ def classify_pose(
data = Classifier.combine_data(per_frame_features, window_features)
if data.shape[0] > 0:
- pred = classifier.predict(data)
pred_prob = classifier.predict_proba(data)
+ positive_proba = pred_prob[:, 1]
- # Keep the probability for the predicted class only.
- # The following code uses some
- # numpy magic to use the pred array as column indexes
- # for each row of the pred_prob array we just computed.
+ # Derive predicted labels by thresholding at 0.5
+ pred = (positive_proba >= classifier.TRUE_THRESHOLD).astype(int)
+
+ # Keep the probability of the predicted class
pred_prob = pred_prob[np.arange(len(pred_prob)), pred]
# Only copy out predictions where there was a valid pose
diff --git a/src/jabs/ui/classification_thread.py b/src/jabs/ui/classification_thread.py
index 972b067d..4fc8c2c3 100644
--- a/src/jabs/ui/classification_thread.py
+++ b/src/jabs/ui/classification_thread.py
@@ -130,23 +130,18 @@ def check_termination_requested() -> None:
check_termination_requested()
if data.shape[0] > 0:
- # make predictions
- # Note: this makes predictions for all frames in the video, even those without valid pose
- # We will later filter these out when saving the predictions to disk
- # consider changing this to only predict on frames with valid pose
- predictions[video][identity] = self._classifier.predict(data)
-
- # also get the probabilities
+ # get predicted probabilities for the positive class (class 1)
prob = self._classifier.predict_proba(data)
- # Save the probability for the predicted class only.
- # The following code uses some
- # numpy magic to use the _predictions array as column indexes
- # for each row of the 'prob' array we just computed.
- probabilities[video][identity] = prob[
- np.arange(len(prob)), predictions[video][identity]
- ]
-
- # save the indexes for the predicted frames
+ positive_proba = prob[:, 1]
+
+ # derive binary predictions by thresholding probabilities
+ preds = (positive_proba >= self._classifier.TRUE_THRESHOLD).astype(int)
+ predictions[video][identity] = preds
+
+ # save probability of the predicted class for each frame
+ probabilities[video][identity] = prob[np.arange(len(prob)), preds]
+
+ # store the frame indexes corresponding to each prediction
frame_indexes[video][identity] = feature_values["frame_indexes"]
else:
predictions[video][identity] = np.array(0)
diff --git a/src/jabs/ui/training_thread.py b/src/jabs/ui/training_thread.py
index 3061aac3..20d98dd6 100644
--- a/src/jabs/ui/training_thread.py
+++ b/src/jabs/ui/training_thread.py
@@ -130,12 +130,18 @@ def id_processed() -> None:
self._classifier.behavior_name = self._behavior
self._classifier.set_project_settings(self._project)
self._classifier.train(data)
- predictions = self._classifier.predict(data["test_data"])
+ # get predicted probabilities for the positive class (class 1)
+ probabilities = self._classifier.predict_proba(data["test_data"])
+ positive_proba = probabilities[:, 1]
+
+ # derive binary predictions by thresholding probabilities
+ predictions = (positive_proba >= self._classifier.TRUE_THRESHOLD).astype(int)
# calculate some performance metrics using the classifications
accuracy = self._classifier.accuracy_score(data["test_labels"], predictions)
pr = self._classifier.precision_recall_score(data["test_labels"], predictions)
confusion = self._classifier.confusion_matrix(data["test_labels"], predictions)
+ brier_score = self._classifier.brier_score(data["test_labels"], probabilities)
table_rows.append(
[
@@ -146,6 +152,7 @@ def id_processed() -> None:
pr[1][1],
pr[2][0],
pr[2][1],
+ brier_score,
f"{test_info['video']} [{test_info['identity']}]",
]
)
@@ -188,8 +195,9 @@ def id_processed() -> None:
"precision\n(behavior)",
"recall\n(not behavior)",
"recall\n(behavior)",
- "f beta score\n(not behavior)",
- "f beta score\n(behavior)",
+ "F-1 score\n(not behavior)",
+ "F-1 score\n(behavior)",
+ "Brier score",
"test - leave one out:\n(video [identity])",
],
)
From 96e6819dca4e223ec2a8277d383098b2b784fdee Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 23 Oct 2025 16:53:41 -0400
Subject: [PATCH 12/14] add auto calibration method
---
src/jabs/classifier/classifier.py | 75 +++++++++++++++++++++++++++----
src/jabs/constants.py | 2 +-
src/jabs/ui/settings_dialog.py | 35 ++++++++++-----
3 files changed, 91 insertions(+), 21 deletions(-)
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index 0738573d..ac183f9d 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -54,6 +54,7 @@ class Classifier:
LABEL_THRESHOLD = 20
TRUE_THRESHOLD = 0.5
+ CALIBRATION_METHODS: typing.ClassVar[list[str]] = ["auto", "isotonic", "sigmoid"]
_CLASSIFIER_NAMES: typing.ClassVar[dict] = {
ClassifierType.RANDOM_FOREST: "Random Forest",
@@ -182,6 +183,38 @@ def calibrate_probabilities(self) -> bool:
return self._jabs_settings.get("calibrate_probabilities", False)
return False
+ @staticmethod
+ def _choose_auto_calibration_method(
+ labels: np.ndarray, calibration_cv: int
+ ) -> tuple[str, dict]:
+ """Choose 'isotonic' or 'sigmoid' based on data size per calibration fold.
+
+ Heuristic:
+ - Compute class counts on the *training set labels* passed in.
+ - Estimate per-fold calibration set size as min(pos, neg) / calibration_cv
+ (because CalibratedClassifierCV uses 1/cv of the train split for calibration).
+ - If per-fold per-class counts >= 500 ➜ 'isotonic', else 'sigmoid'.
+
+ Returns:
+ (method, info_dict) where info_dict contains counts used for logging.
+ """
+ # count positive and negative labels
+ pos = int(np.sum(labels == TrackLabels.Label.BEHAVIOR))
+ neg = int(np.sum(labels == TrackLabels.Label.NOT_BEHAVIOR))
+ min_per_class = min(pos, neg)
+ per_fold_per_class = max(0, min_per_class // calibration_cv)
+
+ # Threshold for isotonic safety
+ threshold = 500
+ method = "isotonic" if per_fold_per_class >= threshold else "sigmoid"
+ return method, {
+ "pos_total": pos,
+ "neg_total": neg,
+ "cv": calibration_cv,
+ "per_fold_per_class": per_fold_per_class,
+ "threshold": threshold,
+ }
+
@staticmethod
def train_test_split(per_frame_features, window_features, label_data):
"""split features and labels into training and test datasets
@@ -408,7 +441,7 @@ def classifier_choices(self):
"""
return {d: self._CLASSIFIER_NAMES[d] for d in _classifier_choices}
- def train(self, data, random_seed: int | None = None):
+ def train(self, data: dict, random_seed: int | None = None) -> None:
"""train the classifier
Args:
@@ -441,11 +474,38 @@ def train(self, data, random_seed: int | None = None):
# Optional probability calibration
if self.calibrate_probabilities:
+ # get and validate calibration settings
calibration_method = self._jabs_settings.get(
"calibration_method", DEFAULT_CALIBRATION_METHOD
)
+ if calibration_method.lower() not in self.CALIBRATION_METHODS:
+ raise ValueError(
+ f"Invalid calibration method: {calibration_method}. Must be one of {self.CALIBRATION_METHODS}"
+ )
calibration_cv = self._jabs_settings.get("calibration_cv", DEFAULT_CALIBRATION_CV)
+ # Auto-select method if requested, always figure out what the auto method would be because some of the
+ # selection info is still useful for warnings/logging purposes if the user specified a method explicitly
+ auto_method, auto_method_info = self._choose_auto_calibration_method(
+ labels, calibration_cv
+ )
+ if calibration_method.lower() == "auto":
+ calibration_method = auto_method
+ else:
+ # Optional safety warning: isotonic with small per-fold sets can overfit
+ if (
+ str(calibration_method).lower() == "isotonic"
+ and auto_method_info["per_fold_per_class"] < auto_method_info["threshold"]
+ ):
+ warnings.warn(
+ (
+ "Isotonic calibration selected but per-fold per-class count appears small "
+ f"(~{auto_method_info['per_fold_per_class']}). Consider 'sigmoid' or lowering calibration_cv."
+ ),
+ RuntimeWarning,
+ stacklevel=2,
+ )
+
# Build an unfitted base estimator
if self._classifier_type == ClassifierType.RANDOM_FOREST:
base_estimator = self._make_random_forest(random_seed=random_seed)
@@ -631,22 +691,21 @@ def confusion_matrix(truth, predictions):
return confusion_matrix(truth, predictions)
@staticmethod
- def brier_score(truth, proba):
+ def brier_score(truth: np.ndarray, probabilities: np.ndarray) -> float:
"""Return the Brier score (lower is better).
Args:
- truth: array-like of true binary labels (0/1).
- proba: array of predicted probabilities for the positive class; can be shape (n_samples,)
+ truth (ndarray): array of true binary labels (0/1).
+ probabilities (ndarray): array of predicted probabilities for the positive class; can be shape (n_samples,)
or a (n_samples, 2) array from `predict_proba`.
Returns:
float Brier score.
"""
- proba = np.asarray(proba)
- if proba.ndim == 2:
+ if probabilities.ndim == 2:
# assume columns [P(neg), P(pos)] as returned by predict_proba
- proba = proba[:, 1]
- return brier_score_loss(truth, proba)
+ probabilities = probabilities[:, 1]
+ return brier_score_loss(truth, probabilities)
@staticmethod
def combine_data(per_frame, window):
diff --git a/src/jabs/constants.py b/src/jabs/constants.py
index 80c0e853..fa40016e 100644
--- a/src/jabs/constants.py
+++ b/src/jabs/constants.py
@@ -9,5 +9,5 @@
COMPRESSION = "gzip"
COMPRESSION_OPTS_DEFAULT = 6
-DEFAULT_CALIBRATION_METHOD = "isotonic"
+DEFAULT_CALIBRATION_METHOD = "auto" # can be 'auto', 'isotonic', or 'sigmoid'
DEFAULT_CALIBRATION_CV = 3
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index 6a884766..f5d819c3 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -19,6 +19,7 @@
QWidget,
)
+from jabs.classifier import Classifier
from jabs.constants import DEFAULT_CALIBRATION_CV, DEFAULT_CALIBRATION_METHOD
from jabs.project.settings_manager import SettingsManager
@@ -100,7 +101,7 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
)
self._method_selection = QComboBox()
self._method_selection.addItems(
- ["isotonic", "sigmoid"]
+ Classifier.CALIBRATION_METHODS
) # default will be set from settings
self._cv_selection = QSpinBox()
self._cv_selection.setRange(2, 10)
@@ -158,22 +159,32 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
help_label.setText(
"""
What do these parameters do?
- Calibrate probabilities remaps raw model scores to better probabilities, using a small
- cross-validation inside training. This improves metrics like log-loss and decision thresholds.
+ Calibrate probabilities remaps raw model scores to better probabilities using
+ cross-validation inside training. This improves log-loss, Brier score, and makes thresholding
+ (e.g., show if p ≥ 0.7) more reliable.
+
- calibration_method:
- isotonic (recommended) works best when you have at least ~2000 labeled samples
- (ideally balanced). It learns a flexible mapping and produces very accurate probabilities.
- sigmoid (Platt scaling) is safer for smaller datasets (below ~2000 labels) and is less likely
- to overfit on limited data.
+ auto (default) — automatically selects between isotonic and sigmoid calibration
+ using a simple heuristic based on data size. If each calibration fold has roughly
+ ≥ 500 labeled samples per class, isotonic is used; otherwise, sigmoid is chosen for stability.
+ Larger number of folds (calibration_cv setting) increase the data required for selecting
+ isotonic.
+ isotonic: learns a flexible mapping and produces highly accurate probabilities when enough
+ data is available, but can overfit if the calibration set is small.
+ sigmoid: (Platt scaling) is smoother and more stable for smaller datasets.
+
- calibration_cv: Number of folds used internally by the calibrator. Typical values are 3-5.
Each fold fits the base model and calibrates on held-out training data to avoid leakage.
- Larger values slow training without meaningful benefit.
+ Larger values slow training and require more data per fold for isotonic calibration to be effective.
+
- When to use isotonic: If you have thousands of labeled frames and probabilities seem too extreme
- (many near 0 or 1), isotonic will give smoother and more realistic confidence scores.
- Tip: If training is slow or your dataset is small, start with sigmoid and
- calibration_cv = 3. You can switch to isotonic later as your dataset grows.
+
+ Guidance: If your dataset is large (thousands of labeled frames and roughly balanced),
+ auto will select isotonic. If it selects sigmoid, you can collect more labels or reduce
+ calibration_cv to allow isotonic to activate.
+
+ Tip: Most users should leave calibration_method = auto.
"""
)
help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
From 64cd3be572fe0980bf4030f4323c6441d7b8fd47 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Fri, 24 Oct 2025 17:01:24 -0400
Subject: [PATCH 13/14] add option for saving reliability plots; other tweaks
to model calibration
---
.../behavior_search/behavior_search_util.py | 2 +-
src/jabs/classifier/classifier.py | 86 ++++++++++++++-
src/jabs/project/export_training.py | 6 +-
src/jabs/project/prediction_manager.py | 4 +-
src/jabs/project/project.py | 10 +-
src/jabs/project/settings_manager.py | 4 +-
src/jabs/project/video_manager.py | 2 +-
src/jabs/scripts/cli.py | 6 +-
src/jabs/scripts/initialize_project.py | 2 +-
src/jabs/ui/behavior_search_dialog.py | 4 +-
src/jabs/ui/central_widget.py | 2 +-
src/jabs/ui/main_window.py | 4 +-
src/jabs/ui/settings_dialog.py | 102 ++++++++++++++----
src/jabs/ui/training_thread.py | 19 ++++
tests/project/test_prediction_manager.py | 2 +-
tests/test_behavior_search.py | 2 +-
16 files changed, 206 insertions(+), 51 deletions(-)
diff --git a/src/jabs/behavior_search/behavior_search_util.py b/src/jabs/behavior_search/behavior_search_util.py
index b9a11eb6..324780eb 100644
--- a/src/jabs/behavior_search/behavior_search_util.py
+++ b/src/jabs/behavior_search/behavior_search_util.py
@@ -114,7 +114,7 @@ def _search_behaviors_gen(
)
case PredictionBehaviorSearchQuery() as pred_query:
- proj_settings = project.settings_manager.project_dictionary
+ proj_settings = project.settings_manager.project_info
if pred_query.behavior_label is None:
behavior_dict = proj_settings.get("behavior", {})
behaviors = list(behavior_dict.keys())
diff --git a/src/jabs/classifier/classifier.py b/src/jabs/classifier/classifier.py
index ac183f9d..5317b61c 100644
--- a/src/jabs/classifier/classifier.py
+++ b/src/jabs/classifier/classifier.py
@@ -6,9 +6,11 @@
from pathlib import Path
import joblib
+import matplotlib
+import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
-from sklearn.calibration import CalibratedClassifierCV
+from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.exceptions import InconsistentVersionWarning
from sklearn.metrics import (
@@ -24,6 +26,9 @@
from jabs.types import ClassifierType
from jabs.utils import hash_file
+matplotlib.use("Agg") # use non-GUI backend to avoid thread warnings
+
+
_VERSION = 10
_classifier_choices = [ClassifierType.RANDOM_FOREST, ClassifierType.GRADIENT_BOOSTING]
@@ -414,7 +419,7 @@ def set_project_settings(self, project: Project):
self._behavior_settings = project.settings_manager.get_behavior(self._behavior)
# grab other JABS settings from settings manager, some might be used by the classifier
- self._jabs_settings = project.settings_manager.settings
+ self._jabs_settings = project.settings_manager.jabs_settings
def set_behavior_settings(self, settings: dict):
"""assign behavior-specific settings via a dict to the classifier
@@ -707,6 +712,83 @@ def brier_score(truth: np.ndarray, probabilities: np.ndarray) -> float:
probabilities = probabilities[:, 1]
return brier_score_loss(truth, probabilities)
+ @staticmethod
+ def plot_reliability(
+ truth: np.ndarray,
+ probabilities: np.ndarray,
+ out_path: Path | str,
+ n_bins: int = 10,
+ strategy: str = "uniform",
+ title: str | None = None,
+ show_hist: bool = True,
+ ) -> dict:
+ """Create and save a reliability (calibration) plot.
+
+ Args:
+ truth: Binary ground truth labels (0 or 1).
+ probabilities: Predicted probabilities (2D array where second column is positive class).
+ out_path: File path to save the reliability plot.
+ n_bins: Number of bins for calibration curve.
+ strategy: Binning strategy ('uniform' or 'quantile').
+ title: Optional plot title.
+ show_hist: If True, adds a histogram of predicted probabilities below the curve.
+
+ Returns:
+ Dict with calibration data: 'bins', 'mean_pred', 'frac_pos', and 'counts'.
+ """
+ prob = probabilities[:, 1]
+ y = np.asarray(truth).astype(int)
+
+ pos = int(np.sum(y == 1))
+ neg = int(np.sum(y == 0))
+ if pos == 0 or neg == 0:
+ warnings.warn(
+ "plot_reliability: need both positive and negative labels.", stacklevel=2
+ )
+
+ # Compute calibration curve
+ frac_pos, mean_pred = calibration_curve(y, prob, n_bins=n_bins, strategy=strategy)
+
+ # Bin edges and counts
+ if strategy == "uniform":
+ bins = np.linspace(0.0, 1.0, n_bins + 1)
+ counts, _ = np.histogram(prob, bins=bins)
+ else:
+ q = np.linspace(0.0, 1.0, n_bins + 1)
+ bins = np.quantile(prob, q)
+ bins = np.unique(bins)
+ counts, _ = np.histogram(prob, bins=bins)
+
+ # Plot
+ fig, ax = plt.subplots(figsize=(6.5, 4.5))
+ ax.plot([0, 1], [0, 1], "--", color="gray", label="Perfect calibration")
+ ax.plot(mean_pred, frac_pos, marker="o", color="C0", label="Model")
+ ax.set_xlabel("Predicted probability")
+ ax.set_ylabel("Empirical frequency")
+ if title:
+ ax.set_title(title)
+ ax.legend(loc="best")
+
+ if show_hist:
+ ax_hist = ax.twinx()
+ ax_hist.set_ylim(0, max(counts) * 1.2 if counts.size else 1)
+ display_bins = bins if strategy == "uniform" else 10
+ ax_hist.hist(prob, bins=display_bins, alpha=0.25, color="C1")
+ ax_hist.set_yticks([])
+
+ fig.tight_layout()
+ out_path = Path(out_path)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ fig.savefig(out_path, dpi=150)
+ plt.close(fig)
+
+ return {
+ "bins": bins,
+ "mean_pred": mean_pred,
+ "frac_pos": frac_pos,
+ "counts": counts,
+ }
+
@staticmethod
def combine_data(per_frame, window):
"""combine feature sets together
diff --git a/src/jabs/project/export_training.py b/src/jabs/project/export_training.py
index 12248fae..1c4458fd 100644
--- a/src/jabs/project/export_training.py
+++ b/src/jabs/project/export_training.py
@@ -65,14 +65,14 @@ def export_training_data(
write_project_settings(out_h5, project.settings_manager.get_behavior(behavior), "settings")
out_h5.attrs["classifier_type"] = classifier_type.value
out_h5.attrs["training_seed"] = training_seed
- out_h5.attrs["calibrate_probabilities"] = project.settings_manager.settings.get(
+ out_h5.attrs["calibrate_probabilities"] = project.settings_manager.jabs_settings.get(
"calibrate_probabilities", False
)
if out_h5.attrs["calibrate_probabilities"]:
- out_h5.attrs["calibration_method"] = project.settings_manager.settings.get(
+ out_h5.attrs["calibration_method"] = project.settings_manager.jabs_settings.get(
"calibration_method", DEFAULT_CALIBRATION_METHOD
)
- out_h5.attrs["calibration_cv"] = project.settings_manager.settings.get(
+ out_h5.attrs["calibration_cv"] = project.settings_manager.jabs_settings.get(
"calibration_cv", DEFAULT_CALIBRATION_CV
)
diff --git a/src/jabs/project/prediction_manager.py b/src/jabs/project/prediction_manager.py
index d4e63caa..2dc1d776 100644
--- a/src/jabs/project/prediction_manager.py
+++ b/src/jabs/project/prediction_manager.py
@@ -126,9 +126,7 @@ def load_predictions(self, video: str, behavior: str):
file_base = Path(video).with_suffix("").name + ".h5"
path = self._project.project_paths.prediction_dir / file_base
- nident = self._project.settings_manager.project_dictionary["video_files"][video][
- "identities"
- ]
+ nident = self._project.settings_manager.project_info["video_files"][video]["identities"]
try:
with h5py.File(path, "r") as h5:
diff --git a/src/jabs/project/project.py b/src/jabs/project/project.py
index b84cc474..74c62938 100644
--- a/src/jabs/project/project.py
+++ b/src/jabs/project/project.py
@@ -69,10 +69,7 @@ def __init__(
self._session_tracker = SessionTracker(self, tracking_enabled=enable_session_tracker)
# write out the defaults to the project file
- if (
- self._settings_manager.project_dictionary.get("defaults")
- != self.get_project_defaults()
- ):
+ if self._settings_manager.project_info.get("defaults") != self.get_project_defaults():
self._settings_manager.save_project_file({"defaults": self.get_project_defaults()})
# Start a session tracker for this project.
@@ -110,11 +107,6 @@ def classifier_dir(self):
"""get the classifier directory"""
return self._paths.classifier_dir
- @property
- def settings(self):
- """get the project metadata and preferences."""
- return self._settings_manager.project_dictionary
-
@property
def settings_manager(self) -> SettingsManager:
"""get the project settings manager"""
diff --git a/src/jabs/project/settings_manager.py b/src/jabs/project/settings_manager.py
index f5ddb031..0924d3f2 100644
--- a/src/jabs/project/settings_manager.py
+++ b/src/jabs/project/settings_manager.py
@@ -57,7 +57,7 @@ def save_project_file(self, data: dict | None = None):
tmp.replace(self._paths.project_file)
@property
- def project_dictionary(self) -> dict:
+ def project_info(self) -> dict:
"""Get a copy of the current project properties and settings.
Returns:
@@ -66,7 +66,7 @@ def project_dictionary(self) -> dict:
return dict(self._project_info)
@property
- def settings(self) -> dict:
+ def jabs_settings(self) -> dict:
"""Get a copy of general JABS settings from project file
Returns:
diff --git a/src/jabs/project/video_manager.py b/src/jabs/project/video_manager.py
index 60e451e3..89f00cd2 100644
--- a/src/jabs/project/video_manager.py
+++ b/src/jabs/project/video_manager.py
@@ -142,7 +142,7 @@ def get_video_identity_count(self, video_name: str) -> int:
def _load_video_metadata(self):
"""Load metadata for each video and calculate total identities."""
- video_metadata = self._settings_manager.project_dictionary.get("video_files", {})
+ video_metadata = self._settings_manager.project_info.get("video_files", {})
flush = False
for video in self._videos:
vinfo = video_metadata.get(video, {})
diff --git a/src/jabs/scripts/cli.py b/src/jabs/scripts/cli.py
index b9fabb34..f053aa09 100644
--- a/src/jabs/scripts/cli.py
+++ b/src/jabs/scripts/cli.py
@@ -83,7 +83,7 @@ def export_training(ctx, directory: Path, behavior: str, classifier: str, outfil
jabs_project = Project(directory, enable_session_tracker=False)
# validate that the behavior exists in the project
- if behavior not in jabs_project.settings["behavior"]:
+ if behavior not in jabs_project.settings_manager.project_info["behavior"]:
raise click.ClickException(f"Behavior '{behavior}' not found in project.")
console = Console()
@@ -143,11 +143,11 @@ def rename_behavior(ctx, directory: Path, old_name: str, new_name: str) -> None:
jabs_project = Project(directory, enable_session_tracker=False)
# validate that the old behavior exists in the project
- if old_name not in jabs_project.settings["behavior"]:
+ if old_name not in jabs_project.settings_manager.project_info["behavior"]:
raise click.ClickException(f"Behavior '{old_name}' not found in project.")
# validate that the new behavior does not already exist in the project
- if new_name in jabs_project.settings["behavior"]:
+ if new_name in jabs_project.settings_manager.project_info["behavior"]:
raise click.ClickException(f"Behavior '{new_name}' already exists in project.")
console = Console()
diff --git a/src/jabs/scripts/initialize_project.py b/src/jabs/scripts/initialize_project.py
index f1f7db4e..3b5aca34 100755
--- a/src/jabs/scripts/initialize_project.py
+++ b/src/jabs/scripts/initialize_project.py
@@ -295,7 +295,7 @@ def validation_job_producer():
# save window sizes to project settings
deduped_window_sizes = set(
- project.settings_manager.project_dictionary.get("window_sizes", []) + window_sizes
+ project.settings_manager.project_info.get("window_sizes", []) + window_sizes
)
project.settings_manager.save_project_file({"window_sizes": list(deduped_window_sizes)})
diff --git a/src/jabs/ui/behavior_search_dialog.py b/src/jabs/ui/behavior_search_dialog.py
index d6f5bd35..4cbf4305 100644
--- a/src/jabs/ui/behavior_search_dialog.py
+++ b/src/jabs/ui/behavior_search_dialog.py
@@ -58,8 +58,8 @@ def __init__(self, project: Project, parent: QtWidgets.QWidget | None = None):
self.setModal(True)
self.resize(500, 320)
- proj_settings = project.settings
- self._behavior_labels = sorted(proj_settings.get("behavior", {}).keys())
+ proj_info = project.settings_manager.project_info
+ self._behavior_labels = sorted(proj_info.get("behavior", {}).keys())
# === Main Layout ===
main_layout = QtWidgets.QVBoxLayout(self)
diff --git a/src/jabs/ui/central_widget.py b/src/jabs/ui/central_widget.py
index 39015d5f..8493b00b 100644
--- a/src/jabs/ui/central_widget.py
+++ b/src/jabs/ui/central_widget.py
@@ -277,7 +277,7 @@ def set_project(self, project: Project) -> None:
self._labels = None
self._loaded_video = None
- self._controls.update_project_settings(project.settings)
+ self._controls.update_project_settings(project.settings_manager.project_info)
self._search_bar_widget.update_project(project)
self._update_timeline_search_results()
diff --git a/src/jabs/ui/main_window.py b/src/jabs/ui/main_window.py
index aaa92766..3277fad9 100644
--- a/src/jabs/ui/main_window.py
+++ b/src/jabs/ui/main_window.py
@@ -532,7 +532,7 @@ def behavior_label_add_event(self, behaviors: list[str]) -> None:
"""handle project updates required when user adds new behavior labels"""
# check for new behaviors
for behavior in behaviors:
- if behavior not in self._project.settings_manager.project_dictionary["behavior"]:
+ if behavior not in self._project.settings_manager.project_info["behavior"]:
# save new behavior with default settings
self._project.settings_manager.save_behavior(behavior, {})
@@ -1002,5 +1002,5 @@ def _view_license(self) -> None:
def _open_settings_dialog(self) -> None:
"""Open the settings dialog (JABS->Settings menu action)"""
- dialog = JabsSettingsDialog(parent=self, project_settings=self._project.settings_manager)
+ dialog = JabsSettingsDialog(parent=self, settings_manager=self._project.settings_manager)
dialog.exec_()
diff --git a/src/jabs/ui/settings_dialog.py b/src/jabs/ui/settings_dialog.py
index f5d819c3..0038249d 100644
--- a/src/jabs/ui/settings_dialog.py
+++ b/src/jabs/ui/settings_dialog.py
@@ -1,4 +1,5 @@
-from PySide6.QtCore import Qt, Signal
+from PySide6.QtCore import Qt, QTimer, Signal
+from PySide6.QtGui import QResizeEvent, QShowEvent
from PySide6.QtWidgets import (
QAbstractScrollArea,
QCheckBox,
@@ -83,14 +84,14 @@ class JabsSettingsDialog(QDialog):
Dialog for changing project settings.
Args:
- project_settings (SettingsManager): Project settings manager used to load and save settings.
+ settings_manager (SettingsManager): Project settings manager used to load and save settings.
parent (QWidget | None, optional): Parent widget for this dialog. Defaults to None.
"""
- def __init__(self, project_settings: SettingsManager, parent: QWidget | None = None) -> None:
+ def __init__(self, settings_manager: SettingsManager, parent: QWidget | None = None) -> None:
super().__init__(parent)
self.setWindowTitle("Project Settings")
- self._settings_manager = project_settings
+ self._settings_manager = settings_manager
# Allow resizing and show scrollbars if content overflows
self.setSizeGripEnabled(True)
@@ -100,24 +101,27 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
"Enable probability calibration (calibrate_probabilities)"
)
self._method_selection = QComboBox()
- self._method_selection.addItems(
- Classifier.CALIBRATION_METHODS
- ) # default will be set from settings
+ self._method_selection.addItems(Classifier.CALIBRATION_METHODS)
self._cv_selection = QSpinBox()
self._cv_selection.setRange(2, 10)
self._cv_selection.setAccelerated(True)
self._cv_selection.setToolTip("Number of CV folds used inside the calibrator")
+ self._save_reliability_checkbox = QCheckBox("Save reliability plots")
+ self._save_reliability_checkbox.setToolTip(
+ "If enabled, save reliability (calibration) plots after training/validation."
+ )
- # Load current values from project settings (keys must match classifier usage)
- current_settings = project_settings.project_dictionary.get("settings", {})
+ # Load current values from project settings
+ current_settings = settings_manager.jabs_settings
calibrate = current_settings.get("calibrate_probabilities", False)
method = current_settings.get("calibration_method", DEFAULT_CALIBRATION_METHOD)
cv = current_settings.get("calibration_cv", DEFAULT_CALIBRATION_CV)
+ save_reliability = current_settings.get("save_reliability_plots", False)
self._calibrate_checkbox.setChecked(calibrate)
- idx = max(0, self._method_selection.findText(method))
- self._method_selection.setCurrentIndex(idx)
+ self._method_selection.setCurrentIndex(max(0, self._method_selection.findText(method)))
self._cv_selection.setValue(cv)
+ self._save_reliability_checkbox.setChecked(save_reliability)
# Layout for form
form = QWidget(self)
@@ -127,7 +131,9 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
grid.setVerticalSpacing(8)
grid.setColumnStretch(0, 0) # labels column: natural size
grid.setColumnStretch(1, 0) # inputs column: natural size
- grid.setColumnStretch(2, 1) # expanding whitespace to the right
+ grid.setColumnStretch(
+ 2, 1
+ ) # consume extra width on the right (keeps content left-aligned)
# Keep inputs compact; whitespace grows in column 2
self._method_selection.setSizeAdjustPolicy(QComboBox.SizeAdjustPolicy.AdjustToContents)
@@ -138,6 +144,9 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
self._cv_selection.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
self._calibrate_checkbox.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
+ self._save_reliability_checkbox.setSizePolicy(
+ QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed
+ )
grid.addWidget(QLabel("Calibrate probabilities:"), 0, 0, Qt.AlignmentFlag.AlignRight)
grid.addWidget(self._calibrate_checkbox, 0, 1)
@@ -148,8 +157,10 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
grid.addWidget(QLabel("calibration cv (folds):"), 2, 0, Qt.AlignmentFlag.AlignRight)
grid.addWidget(self._cv_selection, 2, 1)
+ grid.addWidget(QLabel("Save reliability plots:"), 3, 0, Qt.AlignmentFlag.AlignRight)
+ grid.addWidget(self._save_reliability_checkbox, 3, 1)
grid.addItem(
- QSpacerItem(0, 0, QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum), 0, 2, 3, 1
+ QSpacerItem(0, 0, QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum), 0, 2, 4, 1
)
# Help / inline docs (rich text)
@@ -185,6 +196,9 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
calibration_cv to allow isotonic to activate.
Tip: Most users should leave calibration_method = auto.
+ Saving reliability plots: If Save reliability plots is enabled, JABS will write reliability
+ figures after training/validation to <project dir>/plots/<timestamp>/.
+ Each run creates a new timestamped folder so results are easy to compare.
"""
)
help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
@@ -196,13 +210,20 @@ def __init__(self, project_settings: SettingsManager, parent: QWidget | None = N
help_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Preferred)
_help_toggle_btn = (
calibration_help_panel._toggle_btn
- ) # internal; used to wire scrolling after scroll area is created
+ ) # used to wire scrolling after scroll area is created
- # When the collapsible expands/collapses, grow the group box to fit content and leave scrolling to the dialog
def _reflow_calibration_group():
+ """Recompute sizes so the group grows to fit all help content; scrolling handled by the dialog"""
+ help_label.adjustSize()
+ calibration_help_panel.adjustSize()
calibration_group.adjustSize()
+ page.adjustSize()
page_layout.activate()
- scroll.ensureWidgetVisible(calibration_help_panel)
+ page.adjustSize()
+ # Sync page width so group box fills viewport when scrollbar appears/disappears
+ self._sync_page_width()
+ # Defer one tick so QScrollArea can recompute its scroll range correctly
+ QTimer.singleShot(0, lambda: scroll.ensureWidgetVisible(calibration_help_panel))
calibration_help_panel.sizeChanged.connect(_reflow_calibration_group)
@@ -229,22 +250,30 @@ def _reflow_calibration_group():
scroll = QScrollArea(self)
scroll.setWidget(page)
- scroll.setWidgetResizable(True)
+ page.adjustSize()
+ scroll.setWidgetResizable(False)
scroll.setFrameShape(QFrame.Shape.NoFrame)
scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
scroll.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
scroll.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
+ # Keep references for width syncing so the group box fills the viewport width
+ self._scroll = scroll
+ self._page = page
+
# Keep content width constant: viewport gutter above matches scrollbar width
scroll.setSizeAdjustPolicy(QAbstractScrollArea.SizeAdjustPolicy.AdjustToContents)
def _on_help_toggled(checked: bool) -> None:
if checked:
- # Only scroll; don't resize caps — keeps scrollbar on the window only
- scroll.ensureWidgetVisible(calibration_help_panel)
+ # Defer one tick so QScrollArea can recompute its scroll range correctly, then scroll to the panel
+ QTimer.singleShot(0, lambda: scroll.ensureWidgetVisible(calibration_help_panel))
_help_toggle_btn.toggled.connect(_on_help_toggled)
+ # Initial width sync
+ self._sync_page_width()
+
# Buttons
btn_box = QDialogButtonBox(self)
btn_save = btn_box.addButton("Save", QDialogButtonBox.ButtonRole.AcceptRole)
@@ -263,6 +292,40 @@ def _on_help_toggled(checked: bool) -> None:
self.adjustSize()
self.resize(max(self.width(), 700), max(self.height(), 600))
+ def _sync_page_width(self) -> None:
+ """Ensure the inner page expands to the scroll viewport width to avoid right-side whitespace."""
+ try:
+ vp = self._scroll.viewport()
+ if vp is not None:
+ w = vp.width()
+ # Fill available width; when the vertical scrollbar appears, viewport width shrinks automatically
+ self._page.setMinimumWidth(w)
+ self._page.updateGeometry()
+ except Exception:
+ pass
+
+ def showEvent(self, e: QShowEvent) -> None:
+ """Handle the show event.
+
+ Ensures the settings page width is synchronized with the viewport when the dialog is first shown.
+
+ Args:
+ e (QShowEvent): The Qt show event.
+ """
+ super().showEvent(e)
+ self._sync_page_width()
+
+ def resizeEvent(self, e: QResizeEvent) -> None:
+ """Handle the resize event.
+
+ Ensures the settings page width matches the viewport width when the dialog is resized.
+
+ Args:
+ e (QResizeEvent): The Qt resize event.
+ """
+ super().resizeEvent(e)
+ self._sync_page_width()
+
def _on_save(self) -> None:
"""Save settings to project and close dialog."""
settings = {
@@ -270,6 +333,7 @@ def _on_save(self) -> None:
"calibrate_probabilities": self._calibrate_checkbox.isChecked(),
"calibration_method": self._method_selection.currentText(),
"calibration_cv": self._cv_selection.value(),
+ "save_reliability_plots": self._save_reliability_checkbox.isChecked(),
}
}
self._settings_manager.save_project_file(settings)
diff --git a/src/jabs/ui/training_thread.py b/src/jabs/ui/training_thread.py
index 20d98dd6..2308c837 100644
--- a/src/jabs/ui/training_thread.py
+++ b/src/jabs/ui/training_thread.py
@@ -1,3 +1,5 @@
+import datetime
+
import numpy as np
from PySide6.QtCore import QThread, Signal
from PySide6.QtWidgets import QWidget
@@ -75,6 +77,16 @@ def run(self) -> None:
"""
self._tasks_complete = 0
+ # Capture timestamp at start of training run
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+ # Prepare path for reliability plots if enabled
+ if self._project.settings_manager.jabs_settings.get("save_reliability_plots", False):
+ plots_dir = self._project.project_paths.project_dir / "plots" / timestamp
+ plots_dir.mkdir(parents=True, exist_ok=True)
+ else:
+ plots_dir = None
+
def check_termination_requested() -> None:
if self._should_terminate:
raise ThreadTerminatedError("Training was cancelled by the user")
@@ -179,6 +191,13 @@ def id_processed() -> None:
print("Top 10 features by importance:")
self._classifier.print_feature_importance(data["feature_names"], 10)
+ # save calibration curve plot for this iteration
+ if plots_dir is not None:
+ plot_path = plots_dir / f"calibration_curve_iter_{i + 1}.png"
+ self._classifier.plot_reliability(
+ data["test_labels"], probabilities, plot_path, n_bins=5
+ )
+
# let the parent thread know that we've finished this iteration
self._tasks_complete += 1
self.update_progress.emit(self._tasks_complete)
diff --git a/tests/project/test_prediction_manager.py b/tests/project/test_prediction_manager.py
index 899c2268..746f3ba9 100644
--- a/tests/project/test_prediction_manager.py
+++ b/tests/project/test_prediction_manager.py
@@ -25,7 +25,7 @@ class MockSettingsManager:
"""Class to simulate project settings."""
def __init__(self):
- self.project_dictionary = {"video_files": {"test_video.avi": {"identities": 2}}}
+ self.project_info = {"video_files": {"test_video.avi": {"identities": 2}}}
@pytest.fixture
diff --git a/tests/test_behavior_search.py b/tests/test_behavior_search.py
index c99902ca..d458bb24 100644
--- a/tests/test_behavior_search.py
+++ b/tests/test_behavior_search.py
@@ -40,7 +40,7 @@ def load_predictions(video, behavior):
# SettingsManager mock
settings_manager = MagicMock()
- settings_manager.project_dictionary = project_settings or {}
+ settings_manager.project_info = project_settings or {}
# Project mock
project = MagicMock()
From e318321b4d4cf49db9bb3a23d2b5f06f517502a9 Mon Sep 17 00:00:00 2001
From: Glen Beane <356266+gbeane@users.noreply.github.com>
Date: Thu, 30 Oct 2025 19:37:50 -0400
Subject: [PATCH 14/14] change default for Project enable_session_tracking
param
---
src/jabs/project/project.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/jabs/project/project.py b/src/jabs/project/project.py
index 74c62938..19eaa8af 100644
--- a/src/jabs/project/project.py
+++ b/src/jabs/project/project.py
@@ -55,7 +55,7 @@ class Project:
"""
def __init__(
- self, project_path, use_cache=True, enable_video_check=True, enable_session_tracker=True
+ self, project_path, use_cache=True, enable_video_check=True, enable_session_tracker=False
):
self._paths = ProjectPaths(Path(project_path), use_cache=use_cache)
self._paths.create_directories()