From 2fb123424f24b07b3274a50fc0d9ff663b2b9065 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 15:23:06 +0000 Subject: [PATCH 01/28] chore(training): correct f-strings and log parameter counts via utils --- .../emotion_detection/training_pipeline.py | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 077db6605..a59a98286 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -31,6 +31,7 @@ create_goemotions_loader, GoEmotionsDataset, ) +from src.utils import count_model_params # Configure logging # G004: Logging f-strings temporarily allowed for development @@ -98,7 +99,7 @@ def __init__( else: self.device = torch.device(device) - logger.info("Using device: {self.device}") + logger.info(f"Using device: {self.device}") self.output_dir.mkdir(parents=True, exist_ok=True) @@ -159,7 +160,7 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: original_batch_size = self.batch_size self.batch_size = min(128, self.batch_size * 8) # Much larger batch size logger.info( - "🔧 DEVELOPMENT MODE: Using {len(train_texts)} training examples, batch_size={self.batch_size} (was {original_batch_size})" + f"🔧 DEVELOPMENT MODE: Using {len(train_texts)} training examples, batch_size={self.batch_size} (was {original_batch_size})" ) self.train_dataset = GoEmotionsDataset( @@ -188,8 +189,7 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: ) logger.info( - "Prepared datasets - Train: {len(self.train_dataset)}, " - "Val: {len(self.val_dataset)}, Test: {len(self.test_dataset)}" + f"Prepared datasets - Train: {len(self.train_dataset)}, Val: {len(self.val_dataset)}, Test: {len(self.test_dataset)}" ) return datasets @@ -209,14 +209,14 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info("🔍 DEBUG: Loss Function Analysis") - logger.info(" Loss function type: {type(self.loss_fn).__name__}") + logger.info(f" Loss function type: {type(self.loss_fn).__name__}") if hasattr(self.loss_fn, "class_weights") and self.loss_fn.class_weights is not None: weights = self.loss_fn.class_weights - logger.info(" Class weights shape: {weights.shape}") - logger.info(" Class weights min: {weights.min().item():.6f}") - logger.info(" Class weights max: {weights.max().item():.6f}") - logger.info(" Class weights mean: {weights.mean().item():.6f}") + logger.info(f" Class weights shape: {weights.shape}") + logger.info(f" Class weights min: {weights.min().item():.6f}") + logger.info(f" Class weights max: {weights.max().item():.6f}") + logger.info(f" Class weights mean: {weights.mean().item():.6f}") if weights.min() <= 0: logger.error("❌ CRITICAL: Class weights contain zero or negative values!") @@ -242,9 +242,9 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - "Model initialized with {self.model.count_parameters():,} trainable parameters" + f"Model initialized with {count_model_params(self.model, only_trainable=True):,} trainable parameters" ) - logger.info("Total training steps: {total_steps}") + logger.info(f"Total training steps: {total_steps}") def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -252,7 +252,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info("Loading model from checkpoint: {checkpoint_path}") + logger.info(f"Loading model from checkpoint: {checkpoint_path}") checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -285,11 +285,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) logger.info( - "Epoch {epoch}: Applied progressive unfreezing", extra={"format_args": True} + f"Epoch {epoch}: Applied progressive unfreezing" ) val_frequency = max(500, num_batches // 5) - logger.info("🔧 Validation frequency: every {val_frequency} batches") + logger.info(f"🔧 Validation frequency: every {val_frequency} batches") for batch_idx, batch in enumerate(self.train_dataloader): input_ids = batch["input_ids"].to(self.device) @@ -298,14 +298,14 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0: logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(" Labels shape: {labels.shape}") - logger.info(" Labels dtype: {labels.dtype}") - logger.info(" Labels min: {labels.min().item()}") - logger.info(" Labels max: {labels.max().item()}") - logger.info(" Labels mean: {labels.float().mean().item():.6f}") - logger.info(" Labels sum: {labels.sum().item()}") - logger.info(" Non-zero labels: {(labels > 0).sum().item()}") - logger.info(" Total labels: {labels.numel()}") + logger.info(f" Labels shape: {labels.shape}") + logger.info(f" Labels dtype: {labels.dtype}") + logger.info(f" Labels min: {labels.min().item()}") + logger.info(f" Labels max: {labels.max().item()}") + logger.info(f" Labels mean: {labels.float().mean().item():.6f}") + logger.info(f" Labels sum: {labels.sum().item()}") + logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") + logger.info(f" Total labels: {labels.numel()}") if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -315,7 +315,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: for i in range(min(10, labels.shape[1])): # First 10 classes class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(" Class {i}: {class_count} positive samples") + logger.info(f" Class {i}: {class_count} positive samples") self.optimizer.zero_grad() @@ -323,11 +323,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0: logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(" Logits shape: {logits.shape}") - logger.info(" Logits min: {logits.min().item():.6f}") - logger.info(" Logits max: {logits.max().item():.6f}") - logger.info(" Logits mean: {logits.mean().item():.6f}") - logger.info(" Logits std: {logits.std().item():.6f}") + logger.info(f" Logits shape: {logits.shape}") + logger.info(f" Logits min: {logits.min().item():.6f}") + logger.info(f" Logits max: {logits.max().item():.6f}") + logger.info(f" Logits mean: {logits.mean().item():.6f}") + logger.info(f" Logits std: {logits.std().item():.6f}") if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") @@ -335,20 +335,20 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(" Predictions min: {predictions.min().item():.6f}") - logger.info(" Predictions max: {predictions.max().item():.6f}") - logger.info(" Predictions mean: {predictions.mean().item():.6f}") + logger.info(f" Predictions min: {predictions.min().item():.6f}") + logger.info(f" Predictions max: {predictions.max().item():.6f}") + logger.info(f" Predictions mean: {predictions.mean().item():.6f}") loss = self.loss_fn(logits, labels) if batch_idx == 0: logger.info("🔍 DEBUG: Loss Analysis") - logger.info(" Raw loss: {loss.item():.8f}") + logger.info(f" Raw loss: {loss.item():.8f}") bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(" Manual BCE loss: {bce_manual.item():.8f}") + logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") @@ -360,7 +360,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(" Class {i} loss: {class_loss.item():.8f}") + logger.info(f" Class {i} loss: {class_loss.item():.8f}") loss.backward() @@ -376,7 +376,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if param_count > 0: total_norm = total_norm ** (1.0 / 2) - logger.info(" Gradient norm before clipping: {total_norm:.6f}") + logger.info(f" Gradient norm before clipping: {total_norm:.6f}") if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") @@ -386,7 +386,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) if batch_idx == 0: - logger.info(" Gradient norm after clipping: {clip_norm:.6f}") + logger.info(f" Gradient norm after clipping: {clip_norm:.6f}") self.optimizer.step() self.scheduler.step() From cd2cb894ddb2479687a894bf7cc9b5f5ba7e545f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 21:27:06 +0000 Subject: [PATCH 02/28] chore(training): fix clip_grad_norm_ formatting; convert brace logs to f-strings; correct checkpoint filename formatting --- .../emotion_detection/training_pipeline.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index a59a98286..6d0e1feb7 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -386,7 +386,8 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) if batch_idx == 0: - logger.info(f" Gradient norm after clipping: {clip_norm:.6f}") + clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm + logger.info(f" Gradient norm after clipping: {clip_val:.6f}") self.optimizer.step() self.scheduler.step() @@ -398,21 +399,21 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: current_lr = self.scheduler.get_last_lr()[0] logger.info( - "Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, " - "Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" + f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, " + f"Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" ) if avg_loss < 1e-8: - logger.error("❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}") + logger.error(f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}") if avg_loss > 100: - logger.error("❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}") + logger.error(f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}") if (batch_idx + 1) % val_frequency == 0: - logger.info("🔍 Validating at batch {batch_idx + 1}...") + logger.info(f"🔍 Validating at batch {batch_idx + 1}...") self.validate(epoch) if self.should_stop_early(): - logger.info("🛑 Early stopping triggered at batch {batch_idx + 1}") + logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") return { "epoch": epoch, "train_loss": total_loss / (batch_idx + 1), @@ -431,7 +432,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: "learning_rate": self.scheduler.get_last_lr()[0], } - logger.info("Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s") + logger.info(f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s") return metrics @@ -444,7 +445,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info("Validating model at epoch {epoch}...") + logger.info(f"Validating model at epoch {epoch}...") val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -459,11 +460,11 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info("New best model saved! Macro F1: {current_score:.4f}") + logger.info(f"New best model saved! Macro F1: {current_score:.4f}") else: self.patience_counter += 1 logger.info( - "No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" + f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" ) return val_metrics @@ -498,10 +499,10 @@ def save_checkpoint(self, epoch: int, metrics: Dict[str, float], is_best: bool = if is_best: checkpoint_path = self.output_dir / "best_model.pt" else: - checkpoint_path = self.output_dir / "checkpoint_epoch_{epoch}.pt" + checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) - logger.info("Checkpoint saved: {checkpoint_path}") + logger.info(f"Checkpoint saved: {checkpoint_path}") def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -556,7 +557,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info("Training history saved to {history_path}") + logger.info(f"Training history saved to {history_path}") except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -576,7 +577,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info("Simplified training history saved to {history_path}") + logger.info(f"Simplified training history saved to {history_path}") results = { "final_test_metrics": test_metrics, From 0e9d99fed842e84ab0ef297b95ed97a30d00416a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 21:30:21 +0000 Subject: [PATCH 03/28] perf(logging): switch f-strings to lazy logger formatting across training_pipeline --- .../emotion_detection/training_pipeline.py | 111 ++++++++++-------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 6d0e1feb7..18dcb00d4 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -99,7 +99,7 @@ def __init__( else: self.device = torch.device(device) - logger.info(f"Using device: {self.device}") + logger.info("Using device: %s", self.device) self.output_dir.mkdir(parents=True, exist_ok=True) @@ -160,7 +160,8 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: original_batch_size = self.batch_size self.batch_size = min(128, self.batch_size * 8) # Much larger batch size logger.info( - f"🔧 DEVELOPMENT MODE: Using {len(train_texts)} training examples, batch_size={self.batch_size} (was {original_batch_size})" + "🔧 DEVELOPMENT MODE: Using %d training examples, batch_size=%d (was %d)", + len(train_texts), self.batch_size, original_batch_size, ) self.train_dataset = GoEmotionsDataset( @@ -189,7 +190,8 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: ) logger.info( - f"Prepared datasets - Train: {len(self.train_dataset)}, Val: {len(self.val_dataset)}, Test: {len(self.test_dataset)}" + "Prepared datasets - Train: %d, Val: %d, Test: %d", + len(self.train_dataset), len(self.val_dataset), len(self.test_dataset) ) return datasets @@ -209,14 +211,14 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info("🔍 DEBUG: Loss Function Analysis") - logger.info(f" Loss function type: {type(self.loss_fn).__name__}") + logger.info(" Loss function type: %s", type(self.loss_fn).__name__) if hasattr(self.loss_fn, "class_weights") and self.loss_fn.class_weights is not None: weights = self.loss_fn.class_weights - logger.info(f" Class weights shape: {weights.shape}") - logger.info(f" Class weights min: {weights.min().item():.6f}") - logger.info(f" Class weights max: {weights.max().item():.6f}") - logger.info(f" Class weights mean: {weights.mean().item():.6f}") + logger.info(" Class weights shape: %s", getattr(weights, "shape", None)) + logger.info(" Class weights min: %.6f", weights.min().item()) + logger.info(" Class weights max: %.6f", weights.max().item()) + logger.info(" Class weights mean: %.6f", weights.mean().item()) if weights.min() <= 0: logger.error("❌ CRITICAL: Class weights contain zero or negative values!") @@ -242,9 +244,10 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - f"Model initialized with {count_model_params(self.model, only_trainable=True):,} trainable parameters" + "Model initialized with %s trainable parameters", + format(count_model_params(self.model, only_trainable=True), ",d"), ) - logger.info(f"Total training steps: {total_steps}") + logger.info("Total training steps: %d", total_steps) def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -252,7 +255,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info(f"Loading model from checkpoint: {checkpoint_path}") + logger.info("Loading model from checkpoint: %s", checkpoint_path) checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -285,11 +288,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) logger.info( - f"Epoch {epoch}: Applied progressive unfreezing" + "Epoch %d: Applied progressive unfreezing", epoch ) val_frequency = max(500, num_batches // 5) - logger.info(f"🔧 Validation frequency: every {val_frequency} batches") + logger.info("🔧 Validation frequency: every %d batches", val_frequency) for batch_idx, batch in enumerate(self.train_dataloader): input_ids = batch["input_ids"].to(self.device) @@ -298,14 +301,14 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0: logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(f" Labels shape: {labels.shape}") - logger.info(f" Labels dtype: {labels.dtype}") - logger.info(f" Labels min: {labels.min().item()}") - logger.info(f" Labels max: {labels.max().item()}") - logger.info(f" Labels mean: {labels.float().mean().item():.6f}") - logger.info(f" Labels sum: {labels.sum().item()}") - logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") - logger.info(f" Total labels: {labels.numel()}") + logger.info(" Labels shape: %s", labels.shape) + logger.info(" Labels dtype: %s", labels.dtype) + logger.info(" Labels min: %s", labels.min().item()) + logger.info(" Labels max: %s", labels.max().item()) + logger.info(" Labels mean: %.6f", labels.float().mean().item()) + logger.info(" Labels sum: %s", labels.sum().item()) + logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) + logger.info(" Total labels: %s", labels.numel()) if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -315,7 +318,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: for i in range(min(10, labels.shape[1])): # First 10 classes class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(f" Class {i}: {class_count} positive samples") + logger.info(" Class %d: %d positive samples", i, int(class_count)) self.optimizer.zero_grad() @@ -323,11 +326,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0: logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(f" Logits shape: {logits.shape}") - logger.info(f" Logits min: {logits.min().item():.6f}") - logger.info(f" Logits max: {logits.max().item():.6f}") - logger.info(f" Logits mean: {logits.mean().item():.6f}") - logger.info(f" Logits std: {logits.std().item():.6f}") + logger.info(" Logits shape: %s", logits.shape) + logger.info(" Logits min: %.6f", logits.min().item()) + logger.info(" Logits max: %.6f", logits.max().item()) + logger.info(" Logits mean: %.6f", logits.mean().item()) + logger.info(" Logits std: %.6f", logits.std().item()) if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") @@ -335,20 +338,20 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(f" Predictions min: {predictions.min().item():.6f}") - logger.info(f" Predictions max: {predictions.max().item():.6f}") - logger.info(f" Predictions mean: {predictions.mean().item():.6f}") + logger.info(" Predictions min: %.6f", predictions.min().item()) + logger.info(" Predictions max: %.6f", predictions.max().item()) + logger.info(" Predictions mean: %.6f", predictions.mean().item()) loss = self.loss_fn(logits, labels) if batch_idx == 0: logger.info("🔍 DEBUG: Loss Analysis") - logger.info(f" Raw loss: {loss.item():.8f}") + logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") + logger.info(" Manual BCE loss: %.8f", bce_manual.item()) if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") @@ -360,7 +363,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(f" Class {i} loss: {class_loss.item():.8f}") + logger.info(" Class %d loss: %.8f", i, class_loss.item()) loss.backward() @@ -376,7 +379,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if param_count > 0: total_norm = total_norm ** (1.0 / 2) - logger.info(f" Gradient norm before clipping: {total_norm:.6f}") + logger.info(" Gradient norm before clipping: %.6f", total_norm) if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") @@ -387,7 +390,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0: clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm - logger.info(f" Gradient norm after clipping: {clip_val:.6f}") + logger.info(" Gradient norm after clipping: %.6f", clip_val) self.optimizer.step() self.scheduler.step() @@ -399,21 +402,21 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: current_lr = self.scheduler.get_last_lr()[0] logger.info( - f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, " - f"Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" + "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", + epoch, batch_idx + 1, num_batches, avg_loss, current_lr, ) if avg_loss < 1e-8: - logger.error(f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}") + logger.error("❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss) if avg_loss > 100: - logger.error(f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}") + logger.error("❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss) if (batch_idx + 1) % val_frequency == 0: - logger.info(f"🔍 Validating at batch {batch_idx + 1}...") + logger.info("🔍 Validating at batch %d...", batch_idx + 1) self.validate(epoch) if self.should_stop_early(): - logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") + logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) return { "epoch": epoch, "train_loss": total_loss / (batch_idx + 1), @@ -432,7 +435,10 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: "learning_rate": self.scheduler.get_last_lr()[0], } - logger.info(f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s") + logger.info( + "Epoch %d completed - Loss: %.4f, Time: %.1fs", + epoch, avg_loss, epoch_time, + ) return metrics @@ -445,7 +451,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info(f"Validating model at epoch {epoch}...") + logger.info("Validating model at epoch %d...", epoch) val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -460,11 +466,12 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info(f"New best model saved! Macro F1: {current_score:.4f}") + logger.info("New best model saved! Macro F1: %.4f", current_score) else: self.patience_counter += 1 logger.info( - f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" + "No improvement. Patience: %d/%d", + self.patience_counter, self.early_stopping_patience, ) return val_metrics @@ -502,7 +509,7 @@ def save_checkpoint(self, epoch: int, metrics: Dict[str, float], is_best: bool = checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) - logger.info(f"Checkpoint saved: {checkpoint_path}") + logger.info("Checkpoint saved: %s", checkpoint_path) def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -527,7 +534,7 @@ def train(self) -> Dict[str, Any]: self.training_history.append(epoch_metrics) if self.should_stop_early(): - logger.info(f"Early stopping at epoch {epoch}") + logger.info("Early stopping at epoch %d", epoch) break else: self.training_history.append(train_metrics) @@ -557,7 +564,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info(f"Training history saved to {history_path}") + logger.info("Training history saved to %s", history_path) except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -577,7 +584,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info(f"Simplified training history saved to {history_path}") + logger.info("Simplified training history saved to %s", history_path) results = { "final_test_metrics": test_metrics, @@ -588,9 +595,9 @@ def convert_numpy_types(obj): } logger.info("✅ Training completed!") - logger.info(f"Best validation Macro F1: {self.best_score:.4f}") - logger.info(f"Final test Macro F1: {test_metrics['macro_f1']:.4f}") - logger.info(f"Final test Micro F1: {test_metrics['micro_f1']:.4f}") + logger.info("Best validation Macro F1: %.4f", self.best_score) + logger.info("Final test Macro F1: %.4f", test_metrics['macro_f1']) + logger.info("Final test Micro F1: %.4f", test_metrics['micro_f1']) return results From 882fcdb48617bd4f2bda65a44dff122638852a10 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 21:54:10 +0000 Subject: [PATCH 04/28] style: wrap long lines to satisfy E501 (dev-mode batch size, class loop, clip_grad assignment) --- src/models/emotion_detection/training_pipeline.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 18dcb00d4..ab422345e 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -158,7 +158,8 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: val_labels = [val_labels[i] for i in val_indices] original_batch_size = self.batch_size - self.batch_size = min(128, self.batch_size * 8) # Much larger batch size + # Increase batch size for dev mode + self.batch_size = min(128, self.batch_size * 8) logger.info( "🔧 DEVELOPMENT MODE: Using %d training examples, batch_size=%d (was %d)", len(train_texts), self.batch_size, original_batch_size, @@ -315,7 +316,9 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: elif labels.sum() == labels.numel(): logger.error("❌ CRITICAL: All labels are one!") - for i in range(min(10, labels.shape[1])): # First 10 classes + # Iterate over first 10 classes + max_classes = min(10, labels.shape[1]) + for i in range(max_classes): class_count = labels[:, i].sum().item() if class_count > 0: logger.info(" Class %d: %d positive samples", i, int(class_count)) @@ -389,7 +392,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) if batch_idx == 0: - clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm + clip_val = ( + float(clip_norm) + if not isinstance(clip_norm, (int, float)) + else clip_norm + ) logger.info(" Gradient norm after clipping: %.6f", clip_val) self.optimizer.step() From d44dadd4b71927a487cefbd920e84c5e4a77a749 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 21:57:18 +0000 Subject: [PATCH 05/28] refactor(training): extract helpers to reduce train_epoch complexity; keep behavior unchanged --- .../emotion_detection/training_pipeline.py | 228 ++++++++++-------- 1 file changed, 124 insertions(+), 104 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index ab422345e..d02021c11 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -301,103 +301,29 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: labels = batch["labels"].to(self.device) if batch_idx == 0: - logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(" Labels shape: %s", labels.shape) - logger.info(" Labels dtype: %s", labels.dtype) - logger.info(" Labels min: %s", labels.min().item()) - logger.info(" Labels max: %s", labels.max().item()) - logger.info(" Labels mean: %.6f", labels.float().mean().item()) - logger.info(" Labels sum: %s", labels.sum().item()) - logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) - logger.info(" Total labels: %s", labels.numel()) - - if labels.sum() == 0: - logger.error("❌ CRITICAL: All labels are zero!") - elif labels.sum() == labels.numel(): - logger.error("❌ CRITICAL: All labels are one!") - - # Iterate over first 10 classes - max_classes = min(10, labels.shape[1]) - for i in range(max_classes): - class_count = labels[:, i].sum().item() - if class_count > 0: - logger.info(" Class %d: %d positive samples", i, int(class_count)) + self._log_data_distribution(labels) self.optimizer.zero_grad() logits = self.model(input_ids, attention_mask) if batch_idx == 0: - logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(" Logits shape: %s", logits.shape) - logger.info(" Logits min: %.6f", logits.min().item()) - logger.info(" Logits max: %.6f", logits.max().item()) - logger.info(" Logits mean: %.6f", logits.mean().item()) - logger.info(" Logits std: %.6f", logits.std().item()) - - if torch.isnan(logits).any(): - logger.error("❌ CRITICAL: NaN values in logits!") - if torch.isinf(logits).any(): - logger.error("❌ CRITICAL: Inf values in logits!") - - predictions = torch.sigmoid(logits) - logger.info(" Predictions min: %.6f", predictions.min().item()) - logger.info(" Predictions max: %.6f", predictions.max().item()) - logger.info(" Predictions mean: %.6f", predictions.mean().item()) + self._log_model_output(logits) loss = self.loss_fn(logits, labels) if batch_idx == 0: - logger.info("🔍 DEBUG: Loss Analysis") - logger.info(" Raw loss: %.8f", loss.item()) - - bce_manual = F.binary_cross_entropy_with_logits( - logits, labels.float(), reduction="mean" - ) - logger.info(" Manual BCE loss: %.8f", bce_manual.item()) - - if abs(loss.item()) < 1e-10: - logger.error("❌ CRITICAL: Loss is effectively zero!") - logger.error(" This indicates a serious training issue!") - - for i in range(min(5, logits.shape[1])): - class_logits = logits[:, i] - class_labels = labels[:, i].float() - class_loss = F.binary_cross_entropy_with_logits( - class_logits, class_labels, reduction="mean" - ) - logger.info(" Class %d loss: %.8f", i, class_loss.item()) + self._log_loss_analysis(loss, logits, labels) loss.backward() if batch_idx == 0: - logger.info("🔍 DEBUG: Gradient Analysis") - total_norm = 0 - param_count = 0 - for p in self.model.parameters(): - if p.grad is not None: - param_norm = p.grad.data.norm(2) - total_norm += param_norm.item() ** 2 - param_count += 1 - - if param_count > 0: - total_norm = total_norm ** (1.0 / 2) - logger.info(" Gradient norm before clipping: %.6f", total_norm) - - if total_norm > 10: - logger.warning("⚠️ WARNING: Large gradient norm detected!") - if total_norm < 1e-6: - logger.warning("⚠️ WARNING: Very small gradient norm detected!") + self._log_gradient_stats_before() clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) if batch_idx == 0: - clip_val = ( - float(clip_norm) - if not isinstance(clip_norm, (int, float)) - else clip_norm - ) - logger.info(" Gradient norm after clipping: %.6f", clip_val) + self._log_gradient_stats_after(clip_norm) self.optimizer.step() self.scheduler.step() @@ -405,32 +331,13 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: total_loss += loss.item() if batch_idx < 5 or (batch_idx + 1) % 100 == 0: # First 5 batches + every 100 - avg_loss = total_loss / (batch_idx + 1) - current_lr = self.scheduler.get_last_lr()[0] + self._log_progress(epoch, batch_idx, num_batches, total_loss) - logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", - epoch, batch_idx + 1, num_batches, avg_loss, current_lr, - ) - - if avg_loss < 1e-8: - logger.error("❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss) - if avg_loss > 100: - logger.error("❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss) - - if (batch_idx + 1) % val_frequency == 0: - logger.info("🔍 Validating at batch %d...", batch_idx + 1) - self.validate(epoch) - - if self.should_stop_early(): - logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) - return { - "epoch": epoch, - "train_loss": total_loss / (batch_idx + 1), - "epoch_time": time.time() - start_time, - "learning_rate": self.scheduler.get_last_lr()[0], - "early_stopped": True, - } + maybe_metrics = self._maybe_validate_and_early_stop( + batch_idx, epoch, start_time, total_loss, val_frequency + ) + if maybe_metrics is not None: + return maybe_metrics epoch_time = time.time() - start_time avg_loss = total_loss / num_batches @@ -449,6 +356,119 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: return metrics + def _log_data_distribution(self, labels: torch.Tensor) -> None: + logger.info("🔍 DEBUG: Data Distribution Analysis") + logger.info(" Labels shape: %s", labels.shape) + logger.info(" Labels dtype: %s", labels.dtype) + logger.info(" Labels min: %s", labels.min().item()) + logger.info(" Labels max: %s", labels.max().item()) + logger.info(" Labels mean: %.6f", labels.float().mean().item()) + logger.info(" Labels sum: %s", labels.sum().item()) + logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) + logger.info(" Total labels: %s", labels.numel()) + + if labels.sum() == 0: + logger.error("❌ CRITICAL: All labels are zero!") + elif labels.sum() == labels.numel(): + logger.error("❌ CRITICAL: All labels are one!") + + # Iterate over first 10 classes + max_classes = min(10, labels.shape[1]) + for i in range(max_classes): + class_count = labels[:, i].sum().item() + if class_count > 0: + logger.info(" Class %d: %d positive samples", i, int(class_count)) + + def _log_model_output(self, logits: torch.Tensor) -> None: + logger.info("🔍 DEBUG: Model Output Analysis") + logger.info(" Logits shape: %s", logits.shape) + logger.info(" Logits min: %.6f", logits.min().item()) + logger.info(" Logits max: %.6f", logits.max().item()) + logger.info(" Logits mean: %.6f", logits.mean().item()) + logger.info(" Logits std: %.6f", logits.std().item()) + if torch.isnan(logits).any(): + logger.error("❌ CRITICAL: NaN values in logits!") + if torch.isinf(logits).any(): + logger.error("❌ CRITICAL: Inf values in logits!") + predictions = torch.sigmoid(logits) + logger.info(" Predictions min: %.6f", predictions.min().item()) + logger.info(" Predictions max: %.6f", predictions.max().item()) + logger.info(" Predictions mean: %.6f", predictions.mean().item()) + + def _log_loss_analysis(self, loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor) -> None: + logger.info("🔍 DEBUG: Loss Analysis") + logger.info(" Raw loss: %.8f", loss.item()) + bce_manual = F.binary_cross_entropy_with_logits( + logits, labels.float(), reduction="mean" + ) + logger.info(" Manual BCE loss: %.8f", bce_manual.item()) + if abs(loss.item()) < 1e-10: + logger.error("❌ CRITICAL: Loss is effectively zero!") + logger.error(" This indicates a serious training issue!") + for i in range(min(5, logits.shape[1])): + class_logits = logits[:, i] + class_labels = labels[:, i].float() + class_loss = F.binary_cross_entropy_with_logits( + class_logits, class_labels, reduction="mean" + ) + logger.info(" Class %d loss: %.8f", i, class_loss.item()) + + def _log_gradient_stats_before(self) -> None: + logger.info("🔍 DEBUG: Gradient Analysis") + total_norm = 0.0 + param_count = 0 + for p in self.model.parameters(): + if p.grad is not None: + param_norm = p.grad.data.norm(2) + total_norm += param_norm.item() ** 2 + param_count += 1 + if param_count > 0: + total_norm = total_norm ** 0.5 + logger.info(" Gradient norm before clipping: %.6f", total_norm) + if total_norm > 10: + logger.warning("⚠️ WARNING: Large gradient norm detected!") + if total_norm < 1e-6: + logger.warning("⚠️ WARNING: Very small gradient norm detected!") + + def _log_gradient_stats_after(self, clip_norm: Union[float, torch.Tensor]) -> None: + clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm + logger.info(" Gradient norm after clipping: %.6f", clip_val) + + def _log_progress(self, epoch: int, batch_idx: int, num_batches: int, total_loss: float) -> None: + avg_loss = total_loss / (batch_idx + 1) + current_lr = self.scheduler.get_last_lr()[0] + logger.info( + "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", + epoch, batch_idx + 1, num_batches, avg_loss, current_lr, + ) + if avg_loss < 1e-8: + logger.error("❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss) + if avg_loss > 100: + logger.error("❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss) + + def _maybe_validate_and_early_stop( + self, + batch_idx: int, + epoch: int, + start_time: float, + total_loss: float, + val_frequency: int, + ) -> Optional[Dict[str, Any]]: + if (batch_idx + 1) % val_frequency != 0: + return None + logger.info("🔍 Validating at batch %d...", batch_idx + 1) + self.validate(epoch) + if self.should_stop_early(): + logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) + return { + "epoch": epoch, + "train_loss": total_loss / (batch_idx + 1), + "epoch_time": time.time() - start_time, + "learning_rate": self.scheduler.get_last_lr()[0], + "early_stopped": True, + } + return None + def validate(self, epoch: int) -> Dict[str, float]: """Validate model performance. From 4ccf8a06df8432324ecc967b480c502a7c39fee3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 23 Aug 2025 21:59:47 +0000 Subject: [PATCH 06/28] Refactor logging and debug modes in emotion detection training pipeline Co-authored-by: denizcan.uelker --- .../emotion_detection/training_pipeline.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index d02021c11..6f7869c3e 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -211,15 +211,17 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: freeze_bert_layers=self.freeze_initial_layers, ) - logger.info("🔍 DEBUG: Loss Function Analysis") - logger.info(" Loss function type: %s", type(self.loss_fn).__name__) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Loss Function Analysis") + logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) if hasattr(self.loss_fn, "class_weights") and self.loss_fn.class_weights is not None: weights = self.loss_fn.class_weights - logger.info(" Class weights shape: %s", getattr(weights, "shape", None)) - logger.info(" Class weights min: %.6f", weights.min().item()) - logger.info(" Class weights max: %.6f", weights.max().item()) - logger.info(" Class weights mean: %.6f", weights.mean().item()) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(" Class weights shape: %s", getattr(weights, "shape", None)) + logger.debug(" Class weights min: %.6f", weights.min().item()) + logger.debug(" Class weights max: %.6f", weights.max().item()) + logger.debug(" Class weights mean: %.6f", weights.mean().item()) if weights.min() <= 0: logger.error("❌ CRITICAL: Class weights contain zero or negative values!") @@ -300,29 +302,29 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: attention_mask = batch["attention_mask"].to(self.device) labels = batch["labels"].to(self.device) - if batch_idx == 0: + if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_data_distribution(labels) self.optimizer.zero_grad() logits = self.model(input_ids, attention_mask) - if batch_idx == 0: + if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_model_output(logits) loss = self.loss_fn(logits, labels) - if batch_idx == 0: + if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_loss_analysis(loss, logits, labels) loss.backward() - if batch_idx == 0: + if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_gradient_stats_before() clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) - if batch_idx == 0: + if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_gradient_stats_after(clip_norm) self.optimizer.step() @@ -637,9 +639,8 @@ def train_emotion_detection_model( learning_rate: float = 2e-6, # Reduced from 2e-5 to 2e-6 for debugging num_epochs: int = 3, device: Optional[str] = None, - dev_mode: bool = True, # Enable development mode by default - debug_mode: bool = True, # Enable debugging by default - ) -> Dict[str, Any]: + dev_mode: bool = False, +) -> Dict[str, Any]: """Convenient function to train emotion detection model with default settings. Args: @@ -650,8 +651,7 @@ def train_emotion_detection_model( learning_rate: Learning rate for optimization num_epochs: Number of training epochs device: Device to use for training (auto-detect if None) - dev_mode: Enable development mode with smaller dataset - debug_mode: Enable debugging mode with enhanced logging + dev_mode: If True, use a small subset of data for quicker iterations Returns: Dictionary containing training results and metrics From 420b472b2c0511d66541ea33dfdfe29a7067fc0d Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Sat, 23 Aug 2025 22:01:09 +0000 Subject: [PATCH 07/28] chore(training): correct f-strings; log param counts via utils Resolved issues in src/models/emotion_detection/training_pipeline.py with DeepSource Autofix --- src/models/emotion_detection/training_pipeline.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 6f7869c3e..d242cd1da 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -358,7 +358,8 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: return metrics - def _log_data_distribution(self, labels: torch.Tensor) -> None: + @staticmethod + def _log_data_distribution(labels: torch.Tensor) -> None: logger.info("🔍 DEBUG: Data Distribution Analysis") logger.info(" Labels shape: %s", labels.shape) logger.info(" Labels dtype: %s", labels.dtype) @@ -381,7 +382,8 @@ def _log_data_distribution(self, labels: torch.Tensor) -> None: if class_count > 0: logger.info(" Class %d: %d positive samples", i, int(class_count)) - def _log_model_output(self, logits: torch.Tensor) -> None: + @staticmethod + def _log_model_output(logits: torch.Tensor) -> None: logger.info("🔍 DEBUG: Model Output Analysis") logger.info(" Logits shape: %s", logits.shape) logger.info(" Logits min: %.6f", logits.min().item()) @@ -397,7 +399,8 @@ def _log_model_output(self, logits: torch.Tensor) -> None: logger.info(" Predictions max: %.6f", predictions.max().item()) logger.info(" Predictions mean: %.6f", predictions.mean().item()) - def _log_loss_analysis(self, loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor) -> None: + @staticmethod + def _log_loss_analysis(loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor) -> None: logger.info("🔍 DEBUG: Loss Analysis") logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( @@ -432,7 +435,8 @@ def _log_gradient_stats_before(self) -> None: if total_norm < 1e-6: logger.warning("⚠️ WARNING: Very small gradient norm detected!") - def _log_gradient_stats_after(self, clip_norm: Union[float, torch.Tensor]) -> None: + @staticmethod + def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm logger.info(" Gradient norm after clipping: %.6f", clip_val) From 7524d309d11edfc1c985d5d7beb33aa616196baa Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:27:42 +0200 Subject: [PATCH 08/28] Fix DeepSource warnings: resolve line length issues (FLK-E501) in security_headers.py, jwt_manager.py, training_pipeline.py, and hf_loader.py --- src/models/emotion_detection/hf_loader.py | 88 ++++-- .../emotion_detection/training_pipeline.py | 119 +++++--- src/security/jwt_manager.py | 54 ++-- src/security_headers.py | 274 ++++++++++++------ 4 files changed, 351 insertions(+), 184 deletions(-) diff --git a/src/models/emotion_detection/hf_loader.py b/src/models/emotion_detection/hf_loader.py index 46a29f08f..b68468731 100644 --- a/src/models/emotion_detection/hf_loader.py +++ b/src/models/emotion_detection/hf_loader.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 from __future__ import annotations -from dataclasses import dataclass -from typing import Dict, Optional import os -import tempfile -import tarfile import shutil +import tarfile +import tempfile +from dataclasses import dataclass +from typing import Dict, Optional -import torch import requests -from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification +import torch from huggingface_hub import snapshot_download +from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer @dataclass @@ -71,7 +71,9 @@ def predict(self, text: str, threshold: float = 0.5) -> Dict: headers["Authorization"] = f"Bearer {self.token}" payload = {"inputs": text} try: - resp = requests.post(self.endpoint_url, json=payload, headers=headers, timeout=30) + resp = requests.post( + self.endpoint_url, json=payload, headers=headers, timeout=30 + ) resp.raise_for_status() data = resp.json() # data can be [[{"label":..., "score":...}, ...]] or {"error":...} @@ -79,12 +81,21 @@ def predict(self, text: str, threshold: float = 0.5) -> Dict: items = data[0] if data and isinstance(data[0], list) else data else: items = [] - emotions = {item.get("label", f"L{i}"): float(item.get("score", 0.0)) for i, item in enumerate(items)} + emotions = { + item.get("label", f"L{i}"): float(item.get("score", 0.0)) + for i, item in enumerate(items) + } if emotions: primary_label, confidence = max(emotions.items(), key=lambda kv: kv[1]) else: primary_label, confidence = "neutral", 1.0 - intensity = "high" if confidence >= 0.75 else ("moderate" if confidence >= 0.4 else "low") + + if confidence >= 0.75: + intensity = "high" + elif confidence >= 0.4: + intensity = "moderate" + else: + intensity = "low" return { "emotions": emotions, "primary_emotion": primary_label, @@ -95,20 +106,30 @@ def predict(self, text: str, threshold: float = 0.5) -> Dict: return {} -def _wrap_local_model(local_dir: str, token: Optional[str] = None, force_multi_label: Optional[bool] = None) -> HFEmotionDetector: +def _wrap_local_model( + local_dir: str, + token: Optional[str] = None, + force_multi_label: Optional[bool] = None, +) -> HFEmotionDetector: cfg = AutoConfig.from_pretrained(local_dir, token=token) tok = AutoTokenizer.from_pretrained(local_dir, token=token, use_fast=True) mdl = AutoModelForSequenceClassification.from_pretrained(local_dir, token=token) - id2label = getattr(cfg, "id2label", None) or {i: str(i) for i in range(cfg.num_labels)} + id2label = getattr(cfg, "id2label", None) or { + i: str(i) for i in range(cfg.num_labels) + } if force_multi_label is not None: multi_label = bool(force_multi_label) else: problem_type = getattr(cfg, "problem_type", None) multi_label = problem_type == "multi_label_classification" - return HFEmotionDetector(model=mdl, tokenizer=tok, id2label=id2label, multi_label=multi_label) + return HFEmotionDetector( + model=mdl, tokenizer=tok, id2label=id2label, multi_label=multi_label + ) -def load_hf_emotion_model(model_id: str, token: Optional[str] = None, force_multi_label: Optional[bool] = None) -> HFEmotionDetector: +def load_hf_emotion_model( + model_id: str, token: Optional[str] = None, force_multi_label: Optional[bool] = None +) -> HFEmotionDetector: return _wrap_local_model(model_id, token=token, force_multi_label=force_multi_label) @@ -120,7 +141,9 @@ def load_emotion_model_multi_source( endpoint_url: Optional[str] = None, force_multi_label: Optional[bool] = None, ) -> object: - """Try multiple sources to load the emotion model. Returns an object with .predict(text, threshold). + """Try multiple sources to load the emotion model. + + Returns an object with .predict(text, threshold). Priority: 1) Explicit local_dir if provided and exists @@ -132,14 +155,18 @@ def load_emotion_model_multi_source( # 1) Local directory if local_dir and os.path.isdir(local_dir): try: - return _wrap_local_model(local_dir, token=token, force_multi_label=force_multi_label) + return _wrap_local_model( + local_dir, token=token, force_multi_label=force_multi_label + ) except Exception: pass # 2) HF Hub direct if model_id: try: - return load_hf_emotion_model(model_id, token=token, force_multi_label=force_multi_label) + return load_hf_emotion_model( + model_id, token=token, force_multi_label=force_multi_label + ) except Exception: pass @@ -147,17 +174,23 @@ def load_emotion_model_multi_source( if model_id: try: cache_base = os.getenv("HF_HOME", "/var/tmp/hf-cache") - snap_dir = snapshot_download(repo_id=model_id, token=token, cache_dir=cache_base) - return _wrap_local_model(snap_dir, token=token, force_multi_label=force_multi_label) + snap_dir = snapshot_download( + repo_id=model_id, token=token, cache_dir=cache_base + ) + return _wrap_local_model( + snap_dir, token=token, force_multi_label=force_multi_label + ) except Exception: pass # 4) Archive URL if archive_url: try: - cache_dir = os.path.join(os.getenv("XDG_CACHE_HOME", "/var/tmp/hf-cache"), "model-archives") + cache_base = os.getenv("XDG_CACHE_HOME", "/var/tmp/hf-cache") + cache_dir = os.path.join(cache_base, "model-archives") os.makedirs(cache_dir, exist_ok=True) - archive_path = os.path.join(cache_dir, os.path.basename(archive_url.split("?")[0])) + archive_name = os.path.basename(archive_url.split("?")[0]) + archive_path = os.path.join(cache_dir, archive_name) # Download if not exists if not os.path.exists(archive_path): r = requests.get(archive_url, timeout=60) @@ -171,17 +204,24 @@ def load_emotion_model_multi_source( tar.extractall(path=extract_dir) elif archive_path.endswith(".zip"): import zipfile + with zipfile.ZipFile(archive_path, "r") as zf: zf.extractall(path=extract_dir) else: # Unknown archive, try treating as directory pass # Try load from extracted directory (assume single top-level) - candidates = [extract_dir] + [os.path.join(extract_dir, d) for d in os.listdir(extract_dir)] + candidates = [extract_dir] + [ + os.path.join(extract_dir, d) for d in os.listdir(extract_dir) + ] for cand in candidates: - if os.path.isdir(cand) and os.path.exists(os.path.join(cand, "config.json")): + if os.path.isdir(cand) and os.path.exists( + os.path.join(cand, "config.json") + ): try: - det = _wrap_local_model(cand, token=token, force_multi_label=force_multi_label) + det = _wrap_local_model( + cand, token=token, force_multi_label=force_multi_label + ) return det except Exception: continue @@ -198,4 +238,4 @@ def load_emotion_model_multi_source( pass # Exhausted all sources - raise RuntimeError("Could not load emotion model from any source") \ No newline at end of file + raise RuntimeError("Could not load emotion model from any source") diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index d242cd1da..0a360c5dd 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -1,38 +1,29 @@ #!/usr/bin/env python3 -""" -Training Pipeline for BERT Emotion Detection. +"""Training Pipeline for BERT Emotion Detection. -This module provides a comprehensive training pipeline for the BERT-based -emotion detection model with advanced features like focal loss, temperature -scaling, and ensemble methods. +This module provides a comprehensive training pipeline for the BERT-based emotion +detection model with advanced features like focal loss, temperature scaling, and +ensemble methods. """ import json import logging import time from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import torch import torch.nn.functional as F from torch.optim import AdamW from torch.utils.data import DataLoader -from transformers import ( - AutoTokenizer, - get_linear_schedule_with_warmup, -) - -from .bert_classifier import ( - create_bert_emotion_classifier, - evaluate_emotion_classifier, -) -from .dataset_loader import ( - create_goemotions_loader, - GoEmotionsDataset, -) +from transformers import AutoTokenizer, get_linear_schedule_with_warmup + from src.utils import count_model_params +from .bert_classifier import create_bert_emotion_classifier, evaluate_emotion_classifier +from .dataset_loader import GoEmotionsDataset, create_goemotions_loader + # Configure logging # G004: Logging f-strings temporarily allowed for development logging.basicConfig(level=logging.INFO) @@ -143,7 +134,8 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: test_labels = datasets["test"]["labels"] if dev_mode: - logger.info("🔧 DEVELOPMENT MODE: Using 5% of dataset for faster training") + dev_msg = "🔧 DEVELOPMENT MODE: Using 5% of dataset for faster training" + logger.info(dev_msg) train_size = len(train_texts) dev_size = int(train_size * 0.05) # Reduced from 10% to 5% @@ -160,16 +152,21 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: original_batch_size = self.batch_size # Increase batch size for dev mode self.batch_size = min(128, self.batch_size * 8) - logger.info( - "🔧 DEVELOPMENT MODE: Using %d training examples, batch_size=%d (was %d)", - len(train_texts), self.batch_size, original_batch_size, + dev_msg = ( + "🔧 DEVELOPMENT MODE: Using %d training examples, " + "batch_size=%d (was %d)" ) + logger.info(dev_msg, len(train_texts), self.batch_size, original_batch_size) self.train_dataset = GoEmotionsDataset( train_texts, train_labels, self.tokenizer, self.max_length ) - self.val_dataset = GoEmotionsDataset(val_texts, val_labels, self.tokenizer, self.max_length) - self.test_dataset = GoEmotionsDataset(test_texts, test_labels, self.tokenizer, self.max_length) + self.val_dataset = GoEmotionsDataset( + val_texts, val_labels, self.tokenizer, self.max_length + ) + self.test_dataset = GoEmotionsDataset( + test_texts, test_labels, self.tokenizer, self.max_length + ) self.train_dataloader = DataLoader( self.train_dataset, @@ -192,7 +189,9 @@ def prepare_data(self, dev_mode: bool = False) -> Dict[str, Any]: logger.info( "Prepared datasets - Train: %d, Val: %d, Test: %d", - len(self.train_dataset), len(self.val_dataset), len(self.test_dataset) + len(self.train_dataset), + len(self.val_dataset), + len(self.test_dataset), ) return datasets @@ -215,16 +214,23 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: logger.debug("Loss Function Analysis") logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) - if hasattr(self.loss_fn, "class_weights") and self.loss_fn.class_weights is not None: + if ( + hasattr(self.loss_fn, "class_weights") + and self.loss_fn.class_weights is not None + ): weights = self.loss_fn.class_weights if logger.isEnabledFor(logging.DEBUG): - logger.debug(" Class weights shape: %s", getattr(weights, "shape", None)) + logger.debug( + " Class weights shape: %s", getattr(weights, "shape", None) + ) logger.debug(" Class weights min: %.6f", weights.min().item()) logger.debug(" Class weights max: %.6f", weights.max().item()) logger.debug(" Class weights mean: %.6f", weights.mean().item()) if weights.min() <= 0: - logger.error("❌ CRITICAL: Class weights contain zero or negative values!") + logger.error( + "❌ CRITICAL: Class weights contain zero or negative values!" + ) if weights.max() > 100: logger.error("❌ CRITICAL: Class weights contain very large values!") else: @@ -290,9 +296,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if epoch in self.unfreeze_schedule: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info( - "Epoch %d: Applied progressive unfreezing", epoch - ) + logger.info("Epoch %d: Applied progressive unfreezing", epoch) val_frequency = max(500, num_batches // 5) logger.info("🔧 Validation frequency: every %d batches", val_frequency) @@ -322,7 +326,9 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_gradient_stats_before() - clip_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0) + clip_norm = torch.nn.utils.clip_grad_norm_( + self.model.parameters(), max_norm=1.0 + ) if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): self._log_gradient_stats_after(clip_norm) @@ -332,7 +338,8 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: total_loss += loss.item() - if batch_idx < 5 or (batch_idx + 1) % 100 == 0: # First 5 batches + every 100 + # First 5 batches + every 100 + if batch_idx < 5 or (batch_idx + 1) % 100 == 0: self._log_progress(epoch, batch_idx, num_batches, total_loss) maybe_metrics = self._maybe_validate_and_early_stop( @@ -353,7 +360,9 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: logger.info( "Epoch %d completed - Loss: %.4f, Time: %.1fs", - epoch, avg_loss, epoch_time, + epoch, + avg_loss, + epoch_time, ) return metrics @@ -400,7 +409,9 @@ def _log_model_output(logits: torch.Tensor) -> None: logger.info(" Predictions mean: %.6f", predictions.mean().item()) @staticmethod - def _log_loss_analysis(loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor) -> None: + def _log_loss_analysis( + loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor + ) -> None: logger.info("🔍 DEBUG: Loss Analysis") logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( @@ -428,7 +439,7 @@ def _log_gradient_stats_before(self) -> None: total_norm += param_norm.item() ** 2 param_count += 1 if param_count > 0: - total_norm = total_norm ** 0.5 + total_norm = total_norm**0.5 logger.info(" Gradient norm before clipping: %.6f", total_norm) if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") @@ -437,20 +448,33 @@ def _log_gradient_stats_before(self) -> None: @staticmethod def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: - clip_val = float(clip_norm) if not isinstance(clip_norm, (int, float)) else clip_norm + if not isinstance(clip_norm, (int, float)): + clip_val = float(clip_norm) + else: + clip_val = clip_norm logger.info(" Gradient norm after clipping: %.6f", clip_val) - def _log_progress(self, epoch: int, batch_idx: int, num_batches: int, total_loss: float) -> None: + def _log_progress( + self, epoch: int, batch_idx: int, num_batches: int, total_loss: float + ) -> None: avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", - epoch, batch_idx + 1, num_batches, avg_loss, current_lr, + epoch, + batch_idx + 1, + num_batches, + avg_loss, + current_lr, ) if avg_loss < 1e-8: - logger.error("❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss) + logger.error( + "❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss + ) if avg_loss > 100: - logger.error("❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss) + logger.error( + "❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss + ) def _maybe_validate_and_early_stop( self, @@ -504,7 +528,8 @@ def validate(self, epoch: int) -> Dict[str, float]: self.patience_counter += 1 logger.info( "No improvement. Patience: %d/%d", - self.patience_counter, self.early_stopping_patience, + self.patience_counter, + self.early_stopping_patience, ) return val_metrics @@ -513,7 +538,9 @@ def should_stop_early(self) -> bool: """Check if training should stop early.""" return self.patience_counter >= self.early_stopping_patience - def save_checkpoint(self, epoch: int, metrics: Dict[str, float], is_best: bool = False) -> None: + def save_checkpoint( + self, epoch: int, metrics: Dict[str, float], is_best: bool = False + ) -> None: """Save model checkpoint. Args: @@ -629,8 +656,8 @@ def convert_numpy_types(obj): logger.info("✅ Training completed!") logger.info("Best validation Macro F1: %.4f", self.best_score) - logger.info("Final test Macro F1: %.4f", test_metrics['macro_f1']) - logger.info("Final test Micro F1: %.4f", test_metrics['micro_f1']) + logger.info("Final test Macro F1: %.4f", test_metrics["macro_f1"]) + logger.info("Final test Micro F1: %.4f", test_metrics["micro_f1"]) return results diff --git a/src/security/jwt_manager.py b/src/security/jwt_manager.py index fad3d47f3..17dd608b3 100644 --- a/src/security/jwt_manager.py +++ b/src/security/jwt_manager.py @@ -1,5 +1,4 @@ -""" -JWT-based Authentication Manager for SAMO Deep Learning API +"""JWT-based Authentication Manager for SAMO Deep Learning API. This module provides comprehensive JWT token management including: - Access and refresh token creation @@ -11,9 +10,8 @@ import logging import os -import time from datetime import datetime, timedelta -from typing import Dict, List, Optional, Union, Any +from typing import Any, Dict, List, Optional import jwt from pydantic import BaseModel, Field @@ -27,8 +25,10 @@ ACCESS_TOKEN_EXPIRE_MINUTES = 30 REFRESH_TOKEN_EXPIRE_DAYS = 7 + class TokenPayload(BaseModel): - """Token payload structure""" + """Token payload structure.""" + user_id: str = Field(..., description="User identifier") username: str = Field(..., description="Username") email: str = Field(..., description="User email") @@ -37,17 +37,21 @@ class TokenPayload(BaseModel): iat: Optional[int] = Field(None, description="Issued at timestamp") type: Optional[str] = Field(None, description="Token type, e.g., 'refresh'") + class TokenResponse(BaseModel): - """Token response structure""" + """Token response structure.""" + access_token: str = Field(..., description="Access token") refresh_token: str = Field(..., description="Refresh token") token_type: str = Field(default="bearer", description="Token type") expires_in: int = Field(..., description="Access token expiration in seconds") + # Token pair is returned as a plain dict + class JWTManager: - """Comprehensive JWT token management system""" + """Comprehensive JWT token management system.""" def __init__(self, secret_key: str = SECRET_KEY, algorithm: str = ALGORITHM): self.secret_key = secret_key @@ -56,19 +60,19 @@ def __init__(self, secret_key: str = SECRET_KEY, algorithm: str = ALGORITHM): self.blacklisted_tokens: dict = {} def create_access_token(self, user_data: Dict[str, Any]) -> str: - """Create a new access token""" + """Create a new access token.""" payload = { "user_id": user_data["user_id"], "username": user_data["username"], "email": user_data["email"], "permissions": user_data.get("permissions", []), "exp": datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES), - "iat": datetime.utcnow() + "iat": datetime.utcnow(), } return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) def create_refresh_token(self, user_data: Dict[str, Any]) -> str: - """Create a new refresh token""" + """Create a new refresh token.""" payload = { "user_id": user_data["user_id"], "username": user_data["username"], @@ -76,7 +80,7 @@ def create_refresh_token(self, user_data: Dict[str, Any]) -> str: "permissions": user_data.get("permissions", []), "exp": datetime.utcnow() + timedelta(days=REFRESH_TOKEN_EXPIRE_DAYS), "iat": datetime.utcnow(), - "type": "refresh" + "type": "refresh", } return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) @@ -87,11 +91,11 @@ def create_token_pair(self, user_data: Dict[str, Any]) -> TokenResponse: return TokenResponse( access_token=access_token, refresh_token=refresh_token, - expires_in=ACCESS_TOKEN_EXPIRE_MINUTES * 60 + expires_in=ACCESS_TOKEN_EXPIRE_MINUTES * 60, ) def verify_token(self, token: str) -> Optional[TokenPayload]: - """Verify and decode a token""" + """Verify and decode a token.""" try: if token in self.blacklisted_tokens: return None @@ -102,14 +106,14 @@ def verify_token(self, token: str) -> Optional[TokenPayload]: logger.warning(f"Token expired: {token[:10]}...") return None except jwt.InvalidTokenError as e: - logger.warning(f"Invalid token: {str(e)}") + logger.warning(f"Invalid token: {e!s}") return None except Exception as e: - logger.error(f"Token verification error: {str(e)}") + logger.error(f"Token verification error: {e!s}") return None def refresh_access_token(self, refresh_token: str) -> Optional[str]: - """Refresh an access token using a valid refresh token""" + """Refresh an access token using a valid refresh token.""" payload = self.verify_token(refresh_token) if not payload or getattr(payload, "type", None) != "refresh": return None @@ -118,36 +122,39 @@ def refresh_access_token(self, refresh_token: str) -> Optional[str]: "user_id": payload.user_id, "username": payload.username, "email": payload.email, - "permissions": payload.permissions + "permissions": payload.permissions, } return self.create_access_token(user_data) def blacklist_token(self, token: str) -> bool: - """Add a token to the blacklist""" + """Add a token to the blacklist.""" try: payload = jwt.decode(token, self.secret_key, algorithms=[self.algorithm]) - exp_datetime = datetime.fromtimestamp(payload["exp"]) if payload.get("exp") else None + exp_timestamp = payload.get("exp") + exp_datetime = ( + datetime.fromtimestamp(exp_timestamp) if exp_timestamp else None + ) self.blacklisted_tokens[token] = exp_datetime return True except jwt.InvalidTokenError: return False def is_token_blacklisted(self, token: str) -> bool: - """Check if a token is blacklisted""" + """Check if a token is blacklisted.""" return token in self.blacklisted_tokens def get_user_permissions(self, token: str) -> List[str]: - """Extract user permissions from token""" + """Extract user permissions from token.""" payload = self.verify_token(token) return payload.permissions if payload else [] def has_permission(self, token: str, required_permission: str) -> bool: - """Check if user has a specific permission""" + """Check if user has a specific permission.""" permissions = self.get_user_permissions(token) return required_permission in permissions def cleanup_expired_tokens(self) -> int: - """Clean up expired tokens from blacklist""" + """Clean up expired tokens from blacklist.""" initial_count = len(self.blacklisted_tokens) current_time = datetime.utcnow() @@ -161,5 +168,6 @@ def cleanup_expired_tokens(self) -> int: self.blacklisted_tokens.pop(token, None) return initial_count - len(self.blacklisted_tokens) + # Global JWT manager instance jwt_manager = JWTManager() diff --git a/src/security_headers.py b/src/security_headers.py index b4a579794..521b78649 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -1,25 +1,27 @@ #!/usr/bin/env python3 -""" -🛡️ Security Headers Middleware +"""🛡️ Security Headers Middleware ============================== Flask middleware for adding security headers and implementing security policies. """ -import logging -from typing import Dict, List, Optional, Callable -from dataclasses import dataclass -from flask import Flask, request, Response, g -import time import hashlib +import logging +import os import secrets +import time +from dataclasses import dataclass +from typing import Dict, List + import yaml -import os +from flask import Flask, Response, g, request logger = logging.getLogger(__name__) + @dataclass class SecurityHeadersConfig: """Security headers configuration.""" + enable_csp: bool = True enable_hsts: bool = True enable_x_frame_options: bool = True @@ -40,9 +42,9 @@ class SecurityHeadersConfig: ua_suspicious_score_threshold: int = 4 # Score threshold for suspicious UAs ua_blocking_enabled: bool = False # Whether to block suspicious UAs (vs just log) + class SecurityHeadersMiddleware: - """ - Flask middleware for adding security headers and implementing security policies. + """Flask middleware for adding security headers and implementing security policies. Features: - Content Security Policy (CSP) @@ -63,9 +65,16 @@ def __init__(self, app: Flask, config: SecurityHeadersConfig): # Load CSP from YAML config if available self.csp_policy = None try: - with open(os.path.join(os.path.dirname(__file__), '../configs/security.yaml'), 'r') as f: + config_path = os.path.join( + os.path.dirname(__file__), "../configs/security.yaml" + ) + with open(config_path) as f: security_config = yaml.safe_load(f) - self.csp_policy = security_config.get('security_headers', {}).get('headers', {}).get('Content-Security-Policy') + self.csp_policy = ( + security_config.get("security_headers", {}) + .get("headers", {}) + .get("Content-Security-Policy") + ) except Exception as e: logger.warning(f"Could not load CSP from config: {e}") @@ -80,13 +89,12 @@ def _before_request(self): """Process request before handling.""" # Generate request ID for correlation if self.config.enable_request_id: - g.request_id = hashlib.sha256( - f"{time.time()}:{request.remote_addr}:{secrets.token_hex(8)}".encode() - ).hexdigest() + request_data = f"{time.time()}:{request.remote_addr}:{secrets.token_hex(8)}" + g.request_id = hashlib.sha256(request_data.encode()).hexdigest() # Generate correlation ID if self.config.enable_correlation_id: - g.correlation_id = request.headers.get('X-Correlation-ID', g.request_id) + g.correlation_id = request.headers.get("X-Correlation-ID", g.request_id) # Log security-relevant request information self._log_security_info() @@ -109,48 +117,50 @@ def _add_security_headers(self, response: Response): # Content Security Policy if self.config.enable_content_security_policy: csp_policy = self._build_csp_policy() - response.headers['Content-Security-Policy'] = csp_policy + response.headers["Content-Security-Policy"] = csp_policy # HTTP Strict Transport Security if self.config.enable_strict_transport_security: - response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains; preload' + hsts_value = "max-age=31536000; includeSubDomains; preload" + response.headers["Strict-Transport-Security"] = hsts_value # X-Frame-Options if self.config.enable_x_frame_options: - response.headers['X-Frame-Options'] = 'DENY' + response.headers["X-Frame-Options"] = "DENY" # X-Content-Type-Options if self.config.enable_x_content_type_options: - response.headers['X-Content-Type-Options'] = 'nosniff' + response.headers["X-Content-Type-Options"] = "nosniff" # X-XSS-Protection if self.config.enable_x_xss_protection: - response.headers['X-XSS-Protection'] = '1; mode=block' + response.headers["X-XSS-Protection"] = "1; mode=block" # Referrer Policy if self.config.enable_referrer_policy: - response.headers['Referrer-Policy'] = 'strict-origin-when-cross-origin' + referrer_policy = "strict-origin-when-cross-origin" + response.headers["Referrer-Policy"] = referrer_policy # Permissions Policy if self.config.enable_permissions_policy: permissions_policy = self._build_permissions_policy() - response.headers['Permissions-Policy'] = permissions_policy + response.headers["Permissions-Policy"] = permissions_policy # Cross-Origin Embedder Policy if self.config.enable_cross_origin_embedder_policy: - response.headers['Cross-Origin-Embedder-Policy'] = 'require-corp' + response.headers["Cross-Origin-Embedder-Policy"] = "require-corp" # Cross-Origin Opener Policy if self.config.enable_cross_origin_opener_policy: - response.headers['Cross-Origin-Opener-Policy'] = 'same-origin' + response.headers["Cross-Origin-Opener-Policy"] = "same-origin" # Cross-Origin Resource Policy if self.config.enable_cross_origin_resource_policy: - response.headers['Cross-Origin-Resource-Policy'] = 'same-origin' + response.headers["Cross-Origin-Resource-Policy"] = "same-origin" # Origin-Agent-Cluster if self.config.enable_origin_agent_cluster: - response.headers['Origin-Agent-Cluster'] = '?1' + response.headers["Origin-Agent-Cluster"] = "?1" def _build_csp_policy(self) -> str: """Return CSP policy from config, or a secure default if not set.""" @@ -195,40 +205,40 @@ def _build_permissions_policy(self) -> str: "sync-xhr=()", "usb=()", "web-share=()", - "xr-spatial-tracking=()" + "xr-spatial-tracking=()", ] return ", ".join(policies) def _add_correlation_headers(self, response: Response): """Add request correlation headers.""" - if hasattr(g, 'request_id'): - response.headers['X-Request-ID'] = g.request_id + if hasattr(g, "request_id"): + response.headers["X-Request-ID"] = g.request_id - if hasattr(g, 'correlation_id'): - response.headers['X-Correlation-ID'] = g.correlation_id + if hasattr(g, "correlation_id"): + response.headers["X-Correlation-ID"] = g.correlation_id def _log_security_info(self): """Log security-relevant request information.""" security_info = { - 'timestamp': time.time(), - 'request_id': getattr(g, 'request_id', None), - 'correlation_id': getattr(g, 'correlation_id', None), - 'method': request.method, - 'path': request.path, - 'remote_addr': request.remote_addr, - 'user_agent': request.headers.get('User-Agent', ''), - 'content_type': request.headers.get('Content-Type', ''), - 'content_length': request.headers.get('Content-Length', ''), - 'referer': request.headers.get('Referer', ''), - 'origin': request.headers.get('Origin', ''), - 'x_forwarded_for': request.headers.get('X-Forwarded-For', ''), - 'x_real_ip': request.headers.get('X-Real-IP', ''), + "timestamp": time.time(), + "request_id": getattr(g, "request_id", None), + "correlation_id": getattr(g, "correlation_id", None), + "method": request.method, + "path": request.path, + "remote_addr": request.remote_addr, + "user_agent": request.headers.get("User-Agent", ""), + "content_type": request.headers.get("Content-Type", ""), + "content_length": request.headers.get("Content-Length", ""), + "referer": request.headers.get("Referer", ""), + "origin": request.headers.get("Origin", ""), + "x_forwarded_for": request.headers.get("X-Forwarded-For", ""), + "x_real_ip": request.headers.get("X-Real-IP", ""), } # Log suspicious patterns suspicious_patterns = self._detect_suspicious_patterns() if suspicious_patterns: - security_info['suspicious_patterns'] = suspicious_patterns + security_info["suspicious_patterns"] = suspicious_patterns logger.warning(f"Security warning: {suspicious_patterns}") logger.info(f"Security audit: {security_info}") @@ -236,7 +246,12 @@ def _log_security_info(self): def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: """Enhanced user agent analysis with scoring and detailed categorization.""" if not user_agent: - return {"score": 0, "category": "empty", "patterns": [], "risk_level": "low"} + return { + "score": 0, + "category": "empty", + "patterns": [], + "risk_level": "low", + } score = 0 patterns = [] @@ -244,29 +259,74 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: # Legitimate bot whitelist (negative scoring) legitimate_bots = [ - 'googlebot', 'bingbot', 'slurp', 'duckduckbot', 'facebookexternalhit', - 'twitterbot', 'linkedinbot', 'whatsapp', 'telegrambot', 'discordbot', - 'slackbot', 'github-camo', 'github-actions', 'vercel', 'netlify', - 'uptimerobot', 'pingdom', 'statuscake', 'monitor', 'healthcheck' + "googlebot", + "bingbot", + "slurp", + "duckduckbot", + "facebookexternalhit", + "twitterbot", + "linkedinbot", + "whatsapp", + "telegrambot", + "discordbot", + "slackbot", + "github-camo", + "github-actions", + "vercel", + "netlify", + "uptimerobot", + "pingdom", + "statuscake", + "monitor", + "healthcheck", ] # High-risk patterns (score +3 each) high_risk_patterns = [ - 'sqlmap', 'nikto', 'nmap', 'scanner', 'grabber', 'harvester', - 'exploit', 'vulnerability', 'penetration', 'security', 'audit' + "sqlmap", + "nikto", + "nmap", + "scanner", + "grabber", + "harvester", + "exploit", + "vulnerability", + "penetration", + "security", + "audit", ] # Medium-risk patterns (score +2 each) medium_risk_patterns = [ - 'headless', 'phantom', 'selenium', 'webdriver', 'automated', - 'testing', 'script', 'python-requests', 'curl', 'wget', - 'httrack', 'scraper', 'crawler', 'spider', 'bot' + "headless", + "phantom", + "selenium", + "webdriver", + "automated", + "testing", + "script", + "python-requests", + "curl", + "wget", + "httrack", + "scraper", + "crawler", + "spider", + "bot", ] # Low-risk patterns (score +1 each) low_risk_patterns = [ - 'indexer', 'feed', 'rss', 'aggregator', 'monitor', 'checker', - 'validator', 'linter', 'checker', 'analyzer' + "indexer", + "feed", + "rss", + "aggregator", + "monitor", + "checker", + "validator", + "linter", + "checker", + "analyzer", ] # Check legitimate bots first (negative scoring) @@ -298,13 +358,18 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: logger.debug(f"Low-risk UA pattern detected: {pattern}") # Bonus for suspicious combinations - if any(pattern in ua_lower for pattern in ['bot', 'crawler', 'spider']) and any(pattern in ua_lower for pattern in ['python', 'curl', 'wget', 'script']): + bot_patterns = ["bot", "crawler", "spider"] + script_patterns = ["python", "curl", "wget", "script"] + + if any(pattern in ua_lower for pattern in bot_patterns) and any( + pattern in ua_lower for pattern in script_patterns + ): score += 2 patterns.append("suspicious_combination") logger.debug("Suspicious UA combination detected") # Check for missing or generic user agents - if user_agent in ['', 'null', 'undefined', 'unknown', 'anonymous']: + if user_agent in ["", "null", "undefined", "unknown", "anonymous"]: score += 2 patterns.append("missing_generic_ua") logger.debug("Missing or generic user agent detected") @@ -331,7 +396,7 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: "category": category, "patterns": patterns, "risk_level": risk_level, - "user_agent": user_agent[:100] # Truncate for logging + "user_agent": user_agent[:100], # Truncate for logging } def _detect_suspicious_patterns(self) -> List[str]: @@ -340,10 +405,10 @@ def _detect_suspicious_patterns(self) -> List[str]: # Check for suspicious headers suspicious_headers = [ - 'X-Forwarded-Host', - 'X-Original-URL', - 'X-Rewrite-URL', - 'X-Custom-IP-Authorization' + "X-Forwarded-Host", + "X-Original-URL", + "X-Rewrite-URL", + "X-Custom-IP-Authorization", ] for header in suspicious_headers: @@ -352,8 +417,16 @@ def _detect_suspicious_patterns(self) -> List[str]: # Check for suspicious query parameters suspicious_params = [ - 'cmd', 'exec', 'system', 'eval', 'script', - 'union', 'select', 'insert', 'update', 'delete' + "cmd", + "exec", + "system", + "eval", + "script", + "union", + "select", + "insert", + "update", + "delete", ] for param in suspicious_params: @@ -362,17 +435,24 @@ def _detect_suspicious_patterns(self) -> List[str]: # Enhanced user agent analysis if self.config.enable_enhanced_ua_analysis: - user_agent = request.headers.get('User-Agent', '') + user_agent = request.headers.get("User-Agent", "") ua_analysis = self._analyze_user_agent_enhanced(user_agent) if ua_analysis["score"] >= self.config.ua_suspicious_score_threshold: - patterns.append(f"Suspicious user agent: {ua_analysis['category']} (score: {ua_analysis['score']})") + ua_msg = ( + f"Suspicious user agent: {ua_analysis['category']} " + f"(score: {ua_analysis['score']})" + ) + patterns.append(ua_msg) # Log detailed analysis logger.warning(f"User agent analysis: {ua_analysis}") # Optionally block based on configuration - if self.config.ua_blocking_enabled and ua_analysis["risk_level"] in ["high", "very_high"]: + if self.config.ua_blocking_enabled and ua_analysis["risk_level"] in [ + "high", + "very_high", + ]: patterns.append("BLOCKED: High-risk user agent") return patterns @@ -380,21 +460,23 @@ def _detect_suspicious_patterns(self) -> List[str]: def _log_response_security(self, response: Response): """Log security-relevant response information.""" security_info = { - 'timestamp': time.time(), - 'request_id': getattr(g, 'request_id', None), - 'correlation_id': getattr(g, 'correlation_id', None), - 'status_code': response.status_code, - 'content_type': response.headers.get('Content-Type', ''), - 'content_length': response.headers.get('Content-Length', ''), - 'security_headers': { - 'csp': response.headers.get('Content-Security-Policy', ''), - 'hsts': response.headers.get('Strict-Transport-Security', ''), - 'x_frame_options': response.headers.get('X-Frame-Options', ''), - 'x_content_type_options': response.headers.get('X-Content-Type-Options', ''), - 'x_xss_protection': response.headers.get('X-XSS-Protection', ''), - 'referrer_policy': response.headers.get('Referrer-Policy', ''), - 'permissions_policy': response.headers.get('Permissions-Policy', ''), - } + "timestamp": time.time(), + "request_id": getattr(g, "request_id", None), + "correlation_id": getattr(g, "correlation_id", None), + "status_code": response.status_code, + "content_type": response.headers.get("Content-Type", ""), + "content_length": response.headers.get("Content-Length", ""), + "security_headers": { + "csp": response.headers.get("Content-Security-Policy", ""), + "hsts": response.headers.get("Strict-Transport-Security", ""), + "x_frame_options": response.headers.get("X-Frame-Options", ""), + "x_content_type_options": response.headers.get( + "X-Content-Type-Options", "" + ), + "x_xss_protection": response.headers.get("X-XSS-Protection", ""), + "referrer_policy": response.headers.get("Referrer-Policy", ""), + "permissions_policy": response.headers.get("Permissions-Policy", ""), + }, } logger.info(f"Response security: {security_info}") @@ -406,19 +488,29 @@ def get_security_stats(self) -> Dict: "enable_csp": self.config.enable_content_security_policy, "enable_hsts": self.config.enable_strict_transport_security, "enable_x_frame_options": self.config.enable_x_frame_options, - "enable_x_content_type_options": self.config.enable_x_content_type_options, + "enable_x_content_type_options": ( + self.config.enable_x_content_type_options + ), "enable_x_xss_protection": self.config.enable_x_xss_protection, "enable_referrer_policy": self.config.enable_referrer_policy, "enable_permissions_policy": self.config.enable_permissions_policy, - "enable_cross_origin_embedder_policy": self.config.enable_cross_origin_embedder_policy, - "enable_cross_origin_opener_policy": self.config.enable_cross_origin_opener_policy, - "enable_cross_origin_resource_policy": self.config.enable_cross_origin_resource_policy, + "enable_cross_origin_embedder_policy": ( + self.config.enable_cross_origin_embedder_policy + ), + "enable_cross_origin_opener_policy": ( + self.config.enable_cross_origin_opener_policy + ), + "enable_cross_origin_resource_policy": ( + self.config.enable_cross_origin_resource_policy + ), "enable_origin_agent_cluster": self.config.enable_origin_agent_cluster, "enable_request_id": self.config.enable_request_id, "enable_correlation_id": self.config.enable_correlation_id, "enable_enhanced_ua_analysis": self.config.enable_enhanced_ua_analysis, - "ua_suspicious_score_threshold": self.config.ua_suspicious_score_threshold, + "ua_suspicious_score_threshold": ( + self.config.ua_suspicious_score_threshold + ), "ua_blocking_enabled": self.config.ua_blocking_enabled, }, - "csp_nonce": self._csp_nonce + "csp_nonce": self._csp_nonce, } From 51ed874fc313699635aea3bf954579f00dd17537 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:31:10 +0200 Subject: [PATCH 09/28] Fix PYL-W0201: initialize dataset attributes in __init__ method --- src/models/emotion_detection/training_pipeline.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 0a360c5dd..1940aad3e 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -101,6 +101,14 @@ def __init__( self.scheduler = None self.tokenizer = None + # Dataset attributes + self.train_dataset = None + self.val_dataset = None + self.test_dataset = None + self.train_dataloader = None + self.val_dataloader = None + self.test_dataloader = None + self.best_score = 0.0 self.patience_counter = 0 self.training_history = [] From f6adc1c3771c0b2c7498f9d2dd18bf9a2e5e6a20 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:34:06 +0200 Subject: [PATCH 10/28] Fix PY-D0003: add missing docstrings to training pipeline methods --- .../emotion_detection/training_pipeline.py | 59 +++++++++++++++++-- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 1940aad3e..afda3c916 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -351,7 +351,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: self._log_progress(epoch, batch_idx, num_batches, total_loss) maybe_metrics = self._maybe_validate_and_early_stop( - batch_idx, epoch, start_time, total_loss, val_frequency + batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0] ) if maybe_metrics is not None: return maybe_metrics @@ -377,6 +377,11 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: @staticmethod def _log_data_distribution(labels: torch.Tensor) -> None: + """Log data distribution analysis for debugging. + + Args: + labels: Ground truth labels tensor + """ logger.info("🔍 DEBUG: Data Distribution Analysis") logger.info(" Labels shape: %s", labels.shape) logger.info(" Labels dtype: %s", labels.dtype) @@ -401,6 +406,11 @@ def _log_data_distribution(labels: torch.Tensor) -> None: @staticmethod def _log_model_output(logits: torch.Tensor) -> None: + """Log model output analysis for debugging. + + Args: + logits: Model output logits tensor + """ logger.info("🔍 DEBUG: Model Output Analysis") logger.info(" Logits shape: %s", logits.shape) logger.info(" Logits min: %.6f", logits.min().item()) @@ -420,6 +430,13 @@ def _log_model_output(logits: torch.Tensor) -> None: def _log_loss_analysis( loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor ) -> None: + """Log detailed loss analysis for debugging. + + Args: + loss: Computed loss tensor + logits: Model output logits + labels: Ground truth labels + """ logger.info("🔍 DEBUG: Loss Analysis") logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( @@ -438,6 +455,11 @@ def _log_loss_analysis( logger.info(" Class %d loss: %.8f", i, class_loss.item()) def _log_gradient_stats_before(self) -> None: + """Log gradient statistics before gradient clipping. + + Analyzes gradient norms across all model parameters and logs + statistics for debugging purposes. + """ logger.info("🔍 DEBUG: Gradient Analysis") total_norm = 0.0 param_count = 0 @@ -456,6 +478,11 @@ def _log_gradient_stats_before(self) -> None: @staticmethod def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: + """Log gradient statistics after gradient clipping. + + Args: + clip_norm: Gradient norm value after clipping + """ if not isinstance(clip_norm, (int, float)): clip_val = float(clip_norm) else: @@ -465,6 +492,14 @@ def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float ) -> None: + """Log training progress information. + + Args: + epoch: Current epoch number + batch_idx: Current batch index + num_batches: Total number of batches in epoch + total_loss: Cumulative loss for current epoch + """ avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( @@ -488,10 +523,22 @@ def _maybe_validate_and_early_stop( self, batch_idx: int, epoch: int, - start_time: float, - total_loss: float, - val_frequency: int, + num_batches: int, + avg_loss: float, + current_lr: float, ) -> Optional[Dict[str, Any]]: + """Check if validation should be performed and handle early stopping. + + Args: + batch_idx: Current batch index + epoch: Current epoch number + num_batches: Total number of batches in epoch + avg_loss: Average loss for current epoch + current_lr: Current learning rate + + Returns: + bool: True if training should stop early, False otherwise + """ if (batch_idx + 1) % val_frequency != 0: return None logger.info("🔍 Validating at batch %d...", batch_idx + 1) @@ -500,9 +547,9 @@ def _maybe_validate_and_early_stop( logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) return { "epoch": epoch, - "train_loss": total_loss / (batch_idx + 1), + "train_loss": avg_loss / (batch_idx + 1), "epoch_time": time.time() - start_time, - "learning_rate": self.scheduler.get_last_lr()[0], + "learning_rate": current_lr, "early_stopped": True, } return None From 8ff68f696e40e1308d4bd20a222fa3e988128102 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:40:49 +0200 Subject: [PATCH 11/28] Fix PY-R1000: refactor train_epoch method to reduce cyclomatic complexity --- .../emotion_detection/training_pipeline.py | 159 +++++++++++++----- 1 file changed, 116 insertions(+), 43 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index afda3c916..739bffbca 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -296,66 +296,139 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: Dictionary with training metrics """ self.model.train() - + + # Handle progressive unfreezing + self._handle_progressive_unfreezing(epoch) + + # Setup training total_loss = 0.0 num_batches = len(self.train_dataloader) start_time = time.time() - - if epoch in self.unfreeze_schedule: - layers_to_unfreeze = 2 # Unfreeze 2 layers at a time - self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info("Epoch %d: Applied progressive unfreezing", epoch) - val_frequency = max(500, num_batches // 5) + logger.info("🔧 Validation frequency: every %d batches", val_frequency) + # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): - input_ids = batch["input_ids"].to(self.device) - attention_mask = batch["attention_mask"].to(self.device) - labels = batch["labels"].to(self.device) - - if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): - self._log_data_distribution(labels) - - self.optimizer.zero_grad() - - logits = self.model(input_ids, attention_mask) + batch_loss = self._train_single_batch(batch, batch_idx, epoch, num_batches) + total_loss += batch_loss + + # Check for early stopping + maybe_metrics = self._maybe_validate_and_early_stop( + batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0] + ) + if maybe_metrics is not None: + return maybe_metrics - if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): - self._log_model_output(logits) + # Return epoch metrics + return self._create_epoch_metrics(epoch, total_loss, num_batches, start_time) - loss = self.loss_fn(logits, labels) + def _handle_progressive_unfreezing(self, epoch: int) -> None: + """Handle progressive unfreezing of BERT layers. + + Args: + epoch: Current epoch number + """ + if epoch in self.unfreeze_schedule: + layers_to_unfreeze = 2 # Unfreeze 2 layers at a time + self.model.unfreeze_bert_layers(layers_to_unfreeze) + logger.info("Epoch %d: Applied progressive unfreezing", epoch) - if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): - self._log_loss_analysis(loss, logits, labels) + def _train_single_batch( + self, batch: Dict[str, torch.Tensor], batch_idx: int, epoch: int, num_batches: int + ) -> float: + """Train on a single batch. + + Args: + batch: Input batch data + batch_idx: Current batch index + epoch: Current epoch number + num_batches: Total number of batches + + Returns: + float: Loss value for this batch + """ + # Move data to device + input_ids = batch["input_ids"].to(self.device) + attention_mask = batch["attention_mask"].to(self.device) + labels = batch["labels"].to(self.device) + + # Log debug information for first batch + if batch_idx == 0: + self._log_batch_debug_info(labels, None, None) # logits not available yet + + # Forward pass + self.optimizer.zero_grad() + logits = self.model(input_ids, attention_mask) + loss = self.loss_fn(logits, labels) + + # Log debug information for first batch + if batch_idx == 0: + self._log_batch_debug_info(labels, logits, loss) + + # Backward pass + loss.backward() + + # Gradient clipping + clip_norm = torch.nn.utils.clip_grad_norm_( + self.model.parameters(), max_norm=1.0 + ) - loss.backward() + # Log gradient stats for first batch + if batch_idx == 0: + self._log_gradient_stats_before() + self._log_gradient_stats_after(clip_norm) - if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): - self._log_gradient_stats_before() + # Update parameters + self.optimizer.step() + self.scheduler.step() - clip_norm = torch.nn.utils.clip_grad_norm_( - self.model.parameters(), max_norm=1.0 + # Log progress periodically + if batch_idx < 5 or (batch_idx + 1) % 100 == 0: + # We need to pass the current total loss from the calling method + # For now, just log basic progress + logger.info( + "Epoch %d, Batch %d/%d, Loss: %.8f", + epoch, batch_idx + 1, num_batches, loss.item() ) - if batch_idx == 0 and logger.isEnabledFor(logging.DEBUG): - self._log_gradient_stats_after(clip_norm) - - self.optimizer.step() - self.scheduler.step() + return loss.item() - total_loss += loss.item() - - # First 5 batches + every 100 - if batch_idx < 5 or (batch_idx + 1) % 100 == 0: - self._log_progress(epoch, batch_idx, num_batches, total_loss) - - maybe_metrics = self._maybe_validate_and_early_stop( - batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0] - ) - if maybe_metrics is not None: - return maybe_metrics + def _log_batch_debug_info( + self, labels: torch.Tensor, logits: Optional[torch.Tensor], loss: Optional[torch.Tensor] + ) -> None: + """Log debug information for the first batch. + + Args: + labels: Ground truth labels + logits: Model output logits (None for first call) + loss: Computed loss (None for first call) + """ + if not logger.isEnabledFor(logging.DEBUG): + return + + self._log_data_distribution(labels) + + if logits is not None: + self._log_model_output(logits) + + if loss is not None: + self._log_loss_analysis(loss, logits, labels) + def _create_epoch_metrics( + self, epoch: int, total_loss: float, num_batches: int, start_time: float + ) -> Dict[str, float]: + """Create metrics dictionary for completed epoch. + + Args: + epoch: Current epoch number + total_loss: Total loss for the epoch + num_batches: Number of batches in epoch + start_time: Start time of epoch + + Returns: + Dictionary with epoch metrics + """ epoch_time = time.time() - start_time avg_loss = total_loss / num_batches From 38f137af47a50af74f9d448a2d4b92b106adf357 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:42:42 +0200 Subject: [PATCH 12/28] Fix import inconsistency: replace broken absolute import with inline PyTorch parameter counting --- src/models/emotion_detection/training_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 739bffbca..a75b27379 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -19,8 +19,6 @@ from torch.utils.data import DataLoader from transformers import AutoTokenizer, get_linear_schedule_with_warmup -from src.utils import count_model_params - from .bert_classifier import create_bert_emotion_classifier, evaluate_emotion_classifier from .dataset_loader import GoEmotionsDataset, create_goemotions_loader @@ -261,8 +259,8 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - "Model initialized with %s trainable parameters", - format(count_model_params(self.model, only_trainable=True), ",d"), + "Model initialized with %d trainable parameters", + sum(p.numel() for p in self.model.parameters() if p.requires_grad), ) logger.info("Total training steps: %d", total_steps) From 2e1682f5b41320abf8f11c3691ba3c52367ad47d Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 11:45:26 +0000 Subject: [PATCH 13/28] chore(training): correct f-strings; log param counts via utils Resolved issues in src/models/emotion_detection/training_pipeline.py with DeepSource Autofix --- .../emotion_detection/training_pipeline.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index a75b27379..bc27c4db4 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -294,23 +294,23 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: Dictionary with training metrics """ self.model.train() - + # Handle progressive unfreezing self._handle_progressive_unfreezing(epoch) - + # Setup training total_loss = 0.0 num_batches = len(self.train_dataloader) start_time = time.time() val_frequency = max(500, num_batches // 5) - + logger.info("🔧 Validation frequency: every %d batches", val_frequency) # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): batch_loss = self._train_single_batch(batch, batch_idx, epoch, num_batches) total_loss += batch_loss - + # Check for early stopping maybe_metrics = self._maybe_validate_and_early_stop( batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0] @@ -323,7 +323,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: def _handle_progressive_unfreezing(self, epoch: int) -> None: """Handle progressive unfreezing of BERT layers. - + Args: epoch: Current epoch number """ @@ -336,13 +336,13 @@ def _train_single_batch( self, batch: Dict[str, torch.Tensor], batch_idx: int, epoch: int, num_batches: int ) -> float: """Train on a single batch. - + Args: batch: Input batch data batch_idx: Current batch index epoch: Current epoch number num_batches: Total number of batches - + Returns: float: Loss value for this batch """ @@ -366,7 +366,7 @@ def _train_single_batch( # Backward pass loss.backward() - + # Gradient clipping clip_norm = torch.nn.utils.clip_grad_norm_( self.model.parameters(), max_norm=1.0 @@ -396,7 +396,7 @@ def _log_batch_debug_info( self, labels: torch.Tensor, logits: Optional[torch.Tensor], loss: Optional[torch.Tensor] ) -> None: """Log debug information for the first batch. - + Args: labels: Ground truth labels logits: Model output logits (None for first call) @@ -404,12 +404,12 @@ def _log_batch_debug_info( """ if not logger.isEnabledFor(logging.DEBUG): return - + self._log_data_distribution(labels) - + if logits is not None: self._log_model_output(logits) - + if loss is not None: self._log_loss_analysis(loss, logits, labels) @@ -417,13 +417,13 @@ def _create_epoch_metrics( self, epoch: int, total_loss: float, num_batches: int, start_time: float ) -> Dict[str, float]: """Create metrics dictionary for completed epoch. - + Args: epoch: Current epoch number total_loss: Total loss for the epoch num_batches: Number of batches in epoch start_time: Start time of epoch - + Returns: Dictionary with epoch metrics """ @@ -449,7 +449,7 @@ def _create_epoch_metrics( @staticmethod def _log_data_distribution(labels: torch.Tensor) -> None: """Log data distribution analysis for debugging. - + Args: labels: Ground truth labels tensor """ @@ -478,7 +478,7 @@ def _log_data_distribution(labels: torch.Tensor) -> None: @staticmethod def _log_model_output(logits: torch.Tensor) -> None: """Log model output analysis for debugging. - + Args: logits: Model output logits tensor """ @@ -502,7 +502,7 @@ def _log_loss_analysis( loss: torch.Tensor, logits: torch.Tensor, labels: torch.Tensor ) -> None: """Log detailed loss analysis for debugging. - + Args: loss: Computed loss tensor logits: Model output logits @@ -527,7 +527,7 @@ def _log_loss_analysis( def _log_gradient_stats_before(self) -> None: """Log gradient statistics before gradient clipping. - + Analyzes gradient norms across all model parameters and logs statistics for debugging purposes. """ @@ -550,7 +550,7 @@ def _log_gradient_stats_before(self) -> None: @staticmethod def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: """Log gradient statistics after gradient clipping. - + Args: clip_norm: Gradient norm value after clipping """ @@ -564,7 +564,7 @@ def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float ) -> None: """Log training progress information. - + Args: epoch: Current epoch number batch_idx: Current batch index @@ -599,14 +599,14 @@ def _maybe_validate_and_early_stop( current_lr: float, ) -> Optional[Dict[str, Any]]: """Check if validation should be performed and handle early stopping. - + Args: batch_idx: Current batch index epoch: Current epoch number num_batches: Total number of batches in epoch avg_loss: Average loss for current epoch current_lr: Current learning rate - + Returns: bool: True if training should stop early, False otherwise """ From 38f3361a5c1d84f438e66de7a072e8599b2a8c1e Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 14:02:26 +0200 Subject: [PATCH 14/28] Fix import and logger issues: create utils.py and convert logger calls to f-strings --- .../emotion_detection/training_pipeline.py | 109 ++++++++---------- src/utils.py | 21 ++++ 2 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 src/utils.py diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index a75b27379..2090c816f 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -19,6 +19,8 @@ from torch.utils.data import DataLoader from transformers import AutoTokenizer, get_linear_schedule_with_warmup +from src.utils import count_model_params + from .bert_classifier import create_bert_emotion_classifier, evaluate_emotion_classifier from .dataset_loader import GoEmotionsDataset, create_goemotions_loader @@ -88,7 +90,7 @@ def __init__( else: self.device = torch.device(device) - logger.info("Using device: %s", self.device) + logger.info(f"Using device: {self.device}") self.output_dir.mkdir(parents=True, exist_ok=True) @@ -218,7 +220,7 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: if logger.isEnabledFor(logging.DEBUG): logger.debug("Loss Function Analysis") - logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) + logger.debug(f" Loss function type: {type(self.loss_fn).__name__}") if ( hasattr(self.loss_fn, "class_weights") @@ -229,9 +231,9 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: logger.debug( " Class weights shape: %s", getattr(weights, "shape", None) ) - logger.debug(" Class weights min: %.6f", weights.min().item()) - logger.debug(" Class weights max: %.6f", weights.max().item()) - logger.debug(" Class weights mean: %.6f", weights.mean().item()) + logger.debug(f" Class weights min: {weights.min().item():.6f}") + logger.debug(f" Class weights max: {weights.max().item():.6f}") + logger.debug(f" Class weights mean: {weights.mean().item():.6f}") if weights.min() <= 0: logger.error( @@ -259,10 +261,9 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - "Model initialized with %d trainable parameters", - sum(p.numel() for p in self.model.parameters() if p.requires_grad), + f"Model initialized with {count_model_params(self.model, only_trainable=True):,d} trainable parameters" ) - logger.info("Total training steps: %d", total_steps) + logger.info(f"Total training steps: {total_steps}") def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -270,7 +271,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info("Loading model from checkpoint: %s", checkpoint_path) + logger.info(f"Loading model from checkpoint: {checkpoint_path}") checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -304,7 +305,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: start_time = time.time() val_frequency = max(500, num_batches // 5) - logger.info("🔧 Validation frequency: every %d batches", val_frequency) + logger.info(f"🔧 Validation frequency: every {val_frequency} batches") # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): @@ -330,7 +331,7 @@ def _handle_progressive_unfreezing(self, epoch: int) -> None: if epoch in self.unfreeze_schedule: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info("Epoch %d: Applied progressive unfreezing", epoch) + logger.info(f"Epoch {epoch}: Applied progressive unfreezing") def _train_single_batch( self, batch: Dict[str, torch.Tensor], batch_idx: int, epoch: int, num_batches: int @@ -438,10 +439,7 @@ def _create_epoch_metrics( } logger.info( - "Epoch %d completed - Loss: %.4f, Time: %.1fs", - epoch, - avg_loss, - epoch_time, + f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s" ) return metrics @@ -454,14 +452,14 @@ def _log_data_distribution(labels: torch.Tensor) -> None: labels: Ground truth labels tensor """ logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(" Labels shape: %s", labels.shape) - logger.info(" Labels dtype: %s", labels.dtype) - logger.info(" Labels min: %s", labels.min().item()) - logger.info(" Labels max: %s", labels.max().item()) - logger.info(" Labels mean: %.6f", labels.float().mean().item()) - logger.info(" Labels sum: %s", labels.sum().item()) - logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) - logger.info(" Total labels: %s", labels.numel()) + logger.info(f" Labels shape: {labels.shape}") + logger.info(f" Labels dtype: {labels.dtype}") + logger.info(f" Labels min: {labels.min().item()}") + logger.info(f" Labels max: {labels.max().item()}") + logger.info(f" Labels mean: {labels.float().mean().item():.6f}") + logger.info(f" Labels sum: {labels.sum().item()}") + logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") + logger.info(f" Total labels: {labels.numel()}") if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -473,7 +471,7 @@ def _log_data_distribution(labels: torch.Tensor) -> None: for i in range(max_classes): class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(" Class %d: %d positive samples", i, int(class_count)) + logger.info(f" Class {i}: {int(class_count)} positive samples") @staticmethod def _log_model_output(logits: torch.Tensor) -> None: @@ -483,19 +481,19 @@ def _log_model_output(logits: torch.Tensor) -> None: logits: Model output logits tensor """ logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(" Logits shape: %s", logits.shape) - logger.info(" Logits min: %.6f", logits.min().item()) - logger.info(" Logits max: %.6f", logits.max().item()) - logger.info(" Logits mean: %.6f", logits.mean().item()) - logger.info(" Logits std: %.6f", logits.std().item()) + logger.info(f" Logits shape: {logits.shape}") + logger.info(f" Logits min: {logits.min().item():.6f}") + logger.info(f" Logits max: {logits.max().item():.6f}") + logger.info(f" Logits mean: {logits.mean().item():.6f}") + logger.info(f" Logits std: {logits.std().item():.6f}") if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") if torch.isinf(logits).any(): logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(" Predictions min: %.6f", predictions.min().item()) - logger.info(" Predictions max: %.6f", predictions.max().item()) - logger.info(" Predictions mean: %.6f", predictions.mean().item()) + logger.info(f" Predictions min: {predictions.min().item():.6f}") + logger.info(f" Predictions max: {predictions.max().item():.6f}") + logger.info(f" Predictions mean: {predictions.mean().item():.6f}") @staticmethod def _log_loss_analysis( @@ -509,11 +507,11 @@ def _log_loss_analysis( labels: Ground truth labels """ logger.info("🔍 DEBUG: Loss Analysis") - logger.info(" Raw loss: %.8f", loss.item()) + logger.info(f" Raw loss: {loss.item():.8f}") bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(" Manual BCE loss: %.8f", bce_manual.item()) + logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") logger.error(" This indicates a serious training issue!") @@ -523,7 +521,7 @@ def _log_loss_analysis( class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(" Class %d loss: %.8f", i, class_loss.item()) + logger.info(f" Class {i} loss: {class_loss.item():.8f}") def _log_gradient_stats_before(self) -> None: """Log gradient statistics before gradient clipping. @@ -541,7 +539,7 @@ def _log_gradient_stats_before(self) -> None: param_count += 1 if param_count > 0: total_norm = total_norm**0.5 - logger.info(" Gradient norm before clipping: %.6f", total_norm) + logger.info(f" Gradient norm before clipping: {total_norm:.6f}") if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") if total_norm < 1e-6: @@ -558,7 +556,7 @@ def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: clip_val = float(clip_norm) else: clip_val = clip_norm - logger.info(" Gradient norm after clipping: %.6f", clip_val) + logger.info(f" Gradient norm after clipping: {clip_val:.6f}") def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float @@ -574,20 +572,15 @@ def _log_progress( avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", - epoch, - batch_idx + 1, - num_batches, - avg_loss, - current_lr, + f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" ) if avg_loss < 1e-8: logger.error( - "❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss + f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}" ) if avg_loss > 100: logger.error( - "❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss + f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}" ) def _maybe_validate_and_early_stop( @@ -612,10 +605,10 @@ def _maybe_validate_and_early_stop( """ if (batch_idx + 1) % val_frequency != 0: return None - logger.info("🔍 Validating at batch %d...", batch_idx + 1) + logger.info(f"🔍 Validating at batch {batch_idx + 1}...") self.validate(epoch) if self.should_stop_early(): - logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) + logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") return { "epoch": epoch, "train_loss": avg_loss / (batch_idx + 1), @@ -634,7 +627,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info("Validating model at epoch %d...", epoch) + logger.info(f"Validating model at epoch {epoch}...") val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -649,13 +642,11 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info("New best model saved! Macro F1: %.4f", current_score) + logger.info(f"New best model saved! Macro F1: {current_score:.4f}") else: self.patience_counter += 1 logger.info( - "No improvement. Patience: %d/%d", - self.patience_counter, - self.early_stopping_patience, + f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" ) return val_metrics @@ -695,7 +686,7 @@ def save_checkpoint( checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) - logger.info("Checkpoint saved: %s", checkpoint_path) + logger.info(f"Checkpoint saved: {checkpoint_path}") def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -720,7 +711,7 @@ def train(self) -> Dict[str, Any]: self.training_history.append(epoch_metrics) if self.should_stop_early(): - logger.info("Early stopping at epoch %d", epoch) + logger.info(f"Early stopping at epoch {epoch}") break else: self.training_history.append(train_metrics) @@ -750,7 +741,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info("Training history saved to %s", history_path) + logger.info(f"Training history saved to {history_path}") except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -770,7 +761,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info("Simplified training history saved to %s", history_path) + logger.info(f"Simplified training history saved to {history_path}") results = { "final_test_metrics": test_metrics, @@ -781,9 +772,9 @@ def convert_numpy_types(obj): } logger.info("✅ Training completed!") - logger.info("Best validation Macro F1: %.4f", self.best_score) - logger.info("Final test Macro F1: %.4f", test_metrics["macro_f1"]) - logger.info("Final test Micro F1: %.4f", test_metrics["micro_f1"]) + logger.info(f"Best validation Macro F1: {self.best_score:.4f}") + logger.info(f"Final test Macro F1: {test_metrics['macro_f1']:.4f}") + logger.info(f"Final test Micro F1: {test_metrics['micro_f1']:.4f}") return results diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 000000000..5710b7a81 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Utility functions for the SAMO-DL project.""" + +import torch +from typing import Union + + +def count_model_params(model: torch.nn.Module, only_trainable: bool = False) -> int: + """Count the number of parameters in a PyTorch model. + + Args: + model: PyTorch model to count parameters for + only_trainable: If True, only count trainable parameters + + Returns: + int: Number of parameters (total or trainable only) + """ + if only_trainable: + return sum(p.numel() for p in model.parameters() if p.requires_grad) + else: + return sum(p.numel() for p in model.parameters()) From 854b2e778341d0181f708d9101767d7ec69a3a5e Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 12:37:22 +0000 Subject: [PATCH 15/28] chore(training): correct f-strings; log param counts via utils Resolved issues in src/utils.py with DeepSource Autofix --- src/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils.py b/src/utils.py index 5710b7a81..509717b8f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -7,15 +7,14 @@ def count_model_params(model: torch.nn.Module, only_trainable: bool = False) -> int: """Count the number of parameters in a PyTorch model. - + Args: model: PyTorch model to count parameters for only_trainable: If True, only count trainable parameters - + Returns: int: Number of parameters (total or trainable only) """ if only_trainable: return sum(p.numel() for p in model.parameters() if p.requires_grad) - else: - return sum(p.numel() for p in model.parameters()) + return sum(p.numel() for p in model.parameters()) From 01385779512f5b68eefd61d13f0737c811ed65a0 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 14:51:01 +0200 Subject: [PATCH 16/28] Fix PYL-W1203: Convert f-string logger calls to % formatting for better performance --- .../emotion_detection/training_pipeline.py | 115 ++++++++++-------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 5e9403352..ff1ae2ef8 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -90,7 +90,7 @@ def __init__( else: self.device = torch.device(device) - logger.info(f"Using device: {self.device}") + logger.info("Using device: %s", self.device) self.output_dir.mkdir(parents=True, exist_ok=True) @@ -220,7 +220,7 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: if logger.isEnabledFor(logging.DEBUG): logger.debug("Loss Function Analysis") - logger.debug(f" Loss function type: {type(self.loss_fn).__name__}") + logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) if ( hasattr(self.loss_fn, "class_weights") @@ -231,9 +231,9 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: logger.debug( " Class weights shape: %s", getattr(weights, "shape", None) ) - logger.debug(f" Class weights min: {weights.min().item():.6f}") - logger.debug(f" Class weights max: {weights.max().item():.6f}") - logger.debug(f" Class weights mean: {weights.mean().item():.6f}") + logger.debug(" Class weights min: %.6f", weights.min().item()) + logger.debug(" Class weights max: %.6f", weights.max().item()) + logger.debug(" Class weights mean: %.6f", weights.mean().item()) if weights.min() <= 0: logger.error( @@ -261,9 +261,10 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - f"Model initialized with {count_model_params(self.model, only_trainable=True):,d} trainable parameters" + "Model initialized with %s trainable parameters", + format(count_model_params(self.model, only_trainable=True), ",d") ) - logger.info(f"Total training steps: {total_steps}") + logger.info("Total training steps: %d", total_steps) def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -271,7 +272,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info(f"Loading model from checkpoint: {checkpoint_path}") + logger.info("Loading model from checkpoint: %s", checkpoint_path) checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -304,7 +305,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: num_batches = len(self.train_dataloader) start_time = time.time() val_frequency = max(500, num_batches // 5) - logger.info(f"🔧 Validation frequency: every {val_frequency} batches") + logger.info("🔧 Validation frequency: every %d batches", val_frequency) # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): @@ -313,7 +314,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: # Check for early stopping maybe_metrics = self._maybe_validate_and_early_stop( - batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0] + batch_idx, epoch, num_batches, total_loss, self.scheduler.get_last_lr()[0], val_frequency, start_time ) if maybe_metrics is not None: return maybe_metrics @@ -330,7 +331,7 @@ def _handle_progressive_unfreezing(self, epoch: int) -> None: if epoch in self.unfreeze_schedule: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info(f"Epoch {epoch}: Applied progressive unfreezing") + logger.info("Epoch %d: Applied progressive unfreezing", epoch) def _train_single_batch( self, batch: Dict[str, torch.Tensor], batch_idx: int, epoch: int, num_batches: int @@ -438,7 +439,10 @@ def _create_epoch_metrics( } logger.info( - f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s" + "Epoch %d completed - Loss: %.4f, Time: %.1fs", + epoch, + avg_loss, + epoch_time, ) return metrics @@ -451,14 +455,14 @@ def _log_data_distribution(labels: torch.Tensor) -> None: labels: Ground truth labels tensor """ logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(f" Labels shape: {labels.shape}") - logger.info(f" Labels dtype: {labels.dtype}") - logger.info(f" Labels min: {labels.min().item()}") - logger.info(f" Labels max: {labels.max().item()}") - logger.info(f" Labels mean: {labels.float().mean().item():.6f}") - logger.info(f" Labels sum: {labels.sum().item()}") - logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") - logger.info(f" Total labels: {labels.numel()}") + logger.info(" Labels shape: %s", labels.shape) + logger.info(" Labels dtype: %s", labels.dtype) + logger.info(" Labels min: %s", labels.min().item()) + logger.info(" Labels max: %s", labels.max().item()) + logger.info(" Labels mean: %.6f", labels.float().mean().item()) + logger.info(" Labels sum: %s", labels.sum().item()) + logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) + logger.info(" Total labels: %s", labels.numel()) if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -470,7 +474,7 @@ def _log_data_distribution(labels: torch.Tensor) -> None: for i in range(max_classes): class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(f" Class {i}: {int(class_count)} positive samples") + logger.info(" Class %d: %d positive samples", i, int(class_count)) @staticmethod def _log_model_output(logits: torch.Tensor) -> None: @@ -480,19 +484,19 @@ def _log_model_output(logits: torch.Tensor) -> None: logits: Model output logits tensor """ logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(f" Logits shape: {logits.shape}") - logger.info(f" Logits min: {logits.min().item():.6f}") - logger.info(f" Logits max: {logits.max().item():.6f}") - logger.info(f" Logits mean: {logits.mean().item():.6f}") - logger.info(f" Logits std: {logits.std().item():.6f}") + logger.info(" Logits shape: %s", logits.shape) + logger.info(" Logits min: %.6f", logits.min().item()) + logger.info(" Logits max: %.6f", logits.max().item()) + logger.info(" Logits mean: %.6f", logits.mean().item()) + logger.info(" Logits std: %.6f", logits.std().item()) if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") if torch.isinf(logits).any(): logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(f" Predictions min: {predictions.min().item():.6f}") - logger.info(f" Predictions max: {predictions.max().item():.6f}") - logger.info(f" Predictions mean: {predictions.mean().item():.6f}") + logger.info(" Predictions min: %.6f", predictions.min().item()) + logger.info(" Predictions max: %.6f", predictions.max().item()) + logger.info(" Predictions mean: %.6f", predictions.mean().item()) @staticmethod def _log_loss_analysis( @@ -506,11 +510,11 @@ def _log_loss_analysis( labels: Ground truth labels """ logger.info("🔍 DEBUG: Loss Analysis") - logger.info(f" Raw loss: {loss.item():.8f}") + logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") + logger.info(" Manual BCE loss: %.8f", bce_manual.item()) if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") logger.error(" This indicates a serious training issue!") @@ -520,7 +524,7 @@ def _log_loss_analysis( class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(f" Class {i} loss: {class_loss.item():.8f}") + logger.info(" Class %d loss: %.8f", i, class_loss.item()) def _log_gradient_stats_before(self) -> None: """Log gradient statistics before gradient clipping. @@ -538,7 +542,7 @@ def _log_gradient_stats_before(self) -> None: param_count += 1 if param_count > 0: total_norm = total_norm**0.5 - logger.info(f" Gradient norm before clipping: {total_norm:.6f}") + logger.info(" Gradient norm before clipping: %.6f", total_norm) if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") if total_norm < 1e-6: @@ -555,7 +559,7 @@ def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: clip_val = float(clip_norm) else: clip_val = clip_norm - logger.info(f" Gradient norm after clipping: {clip_val:.6f}") + logger.info(" Gradient norm after clipping: %.6f", clip_val) def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float @@ -571,15 +575,20 @@ def _log_progress( avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( - f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" + "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", + epoch, + batch_idx + 1, + num_batches, + avg_loss, + current_lr, ) if avg_loss < 1e-8: logger.error( - f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}" + "❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss ) if avg_loss > 100: logger.error( - f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}" + "❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss ) def _maybe_validate_and_early_stop( @@ -589,6 +598,8 @@ def _maybe_validate_and_early_stop( num_batches: int, avg_loss: float, current_lr: float, + val_frequency: int, + start_time: float, ) -> Optional[Dict[str, Any]]: """Check if validation should be performed and handle early stopping. @@ -598,16 +609,18 @@ def _maybe_validate_and_early_stop( num_batches: Total number of batches in epoch avg_loss: Average loss for current epoch current_lr: Current learning rate + val_frequency: Frequency of validation + start_time: Start time of training Returns: bool: True if training should stop early, False otherwise """ if (batch_idx + 1) % val_frequency != 0: return None - logger.info(f"🔍 Validating at batch {batch_idx + 1}...") + logger.info("🔍 Validating at batch %d...", batch_idx + 1) self.validate(epoch) if self.should_stop_early(): - logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") + logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) return { "epoch": epoch, "train_loss": avg_loss / (batch_idx + 1), @@ -626,7 +639,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info(f"Validating model at epoch {epoch}...") + logger.info("Validating model at epoch %d...", epoch) val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -641,11 +654,13 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info(f"New best model saved! Macro F1: {current_score:.4f}") + logger.info("New best model saved! Macro F1: %.4f", current_score) else: self.patience_counter += 1 logger.info( - f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" + "No improvement. Patience: %d/%d", + self.patience_counter, + self.early_stopping_patience, ) return val_metrics @@ -682,10 +697,10 @@ def save_checkpoint( if is_best: checkpoint_path = self.output_dir / "best_model.pt" else: - checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" + checkpoint_path = self.output_dir / "checkpoint_epoch_{}.pt".format(epoch) torch.save(checkpoint, checkpoint_path) - logger.info(f"Checkpoint saved: {checkpoint_path}") + logger.info("Checkpoint saved: %s", checkpoint_path) def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -710,7 +725,7 @@ def train(self) -> Dict[str, Any]: self.training_history.append(epoch_metrics) if self.should_stop_early(): - logger.info(f"Early stopping at epoch {epoch}") + logger.info("Early stopping at epoch %d", epoch) break else: self.training_history.append(train_metrics) @@ -740,7 +755,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info(f"Training history saved to {history_path}") + logger.info("Training history saved to %s", history_path) except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -760,7 +775,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info(f"Simplified training history saved to {history_path}") + logger.info("Simplified training history saved to %s", history_path) results = { "final_test_metrics": test_metrics, @@ -771,9 +786,9 @@ def convert_numpy_types(obj): } logger.info("✅ Training completed!") - logger.info(f"Best validation Macro F1: {self.best_score:.4f}") - logger.info(f"Final test Macro F1: {test_metrics['macro_f1']:.4f}") - logger.info(f"Final test Micro F1: {test_metrics['micro_f1']:.4f}") + logger.info("Best validation Macro F1: %.4f", self.best_score) + logger.info("Final test Macro F1: %.4f", test_metrics["macro_f1"]) + logger.info("Final test Micro F1: %.4f", test_metrics["micro_f1"]) return results From d52a8cf91d1190a873760e03ea3de24f8d656456 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 14:56:48 +0200 Subject: [PATCH 17/28] Complete PYL-W1203 fix: convert all f-string logger calls to % formatting --- src/models/emotion_detection/training_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 491bb65d0..be5722681 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -713,7 +713,7 @@ def save_checkpoint( if is_best: checkpoint_path = self.output_dir / "best_model.pt" else: - checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" + checkpoint_path = self.output_dir / "checkpoint_epoch_{}.pt".format(epoch) torch.save(checkpoint, checkpoint_path) logger.info("Checkpoint saved: %s", checkpoint_path) From 30a42f67a45aa82b0ac9f583cd581305d5d879c7 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 14:59:14 +0200 Subject: [PATCH 18/28] Fix PYL-E0602: resolve undefined variables start_time and val_frequency --- src/models/emotion_detection/training_pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index be5722681..e03997741 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -612,7 +612,7 @@ def _maybe_validate_and_early_stop( batch_idx: int, epoch: int, num_batches: int, - avg_loss: float, + total_loss: float, current_lr: float, val_frequency: int, start_time: float, @@ -623,13 +623,13 @@ def _maybe_validate_and_early_stop( batch_idx: Current batch index epoch: Current epoch number num_batches: Total number of batches in epoch - avg_loss: Average loss for current epoch + total_loss: Total loss for current epoch current_lr: Current learning rate val_frequency: Frequency of validation start_time: Start time of training Returns: - bool: True if training should stop early, False otherwise + Dictionary with early stopping metrics if stopping, None otherwise """ if (batch_idx + 1) % val_frequency != 0: return None @@ -639,7 +639,7 @@ def _maybe_validate_and_early_stop( logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) return { "epoch": epoch, - "train_loss": avg_loss / (batch_idx + 1), + "train_loss": total_loss / (batch_idx + 1), "epoch_time": time.time() - start_time, "learning_rate": current_lr, "early_stopped": True, @@ -713,7 +713,7 @@ def save_checkpoint( if is_best: checkpoint_path = self.output_dir / "best_model.pt" else: - checkpoint_path = self.output_dir / "checkpoint_epoch_{}.pt".format(epoch) + checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) logger.info("Checkpoint saved: %s", checkpoint_path) From bf0c34e05c287cd4b5e4b4e53aeb8e68ee04bc62 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:08:52 +0200 Subject: [PATCH 19/28] Convert all logger calls to f-strings and ensure count_model_params import resolves --- .../emotion_detection/training_pipeline.py | 119 ++++++++---------- 1 file changed, 52 insertions(+), 67 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index e03997741..24aaf312f 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -90,7 +90,7 @@ def __init__( else: self.device = torch.device(device) - logger.info("Using device: %s", self.device) + logger.info(f"Using device: {self.device}") self.output_dir.mkdir(parents=True, exist_ok=True) @@ -220,7 +220,7 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: if logger.isEnabledFor(logging.DEBUG): logger.debug("Loss Function Analysis") - logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) + logger.debug(f" Loss function type: {type(self.loss_fn).__name__}") if ( hasattr(self.loss_fn, "class_weights") @@ -229,11 +229,11 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: weights = self.loss_fn.class_weights if logger.isEnabledFor(logging.DEBUG): logger.debug( - " Class weights shape: %s", getattr(weights, "shape", None) + f" Class weights shape: {getattr(weights, 'shape', None)}" ) - logger.debug(" Class weights min: %.6f", weights.min().item()) - logger.debug(" Class weights max: %.6f", weights.max().item()) - logger.debug(" Class weights mean: %.6f", weights.mean().item()) + logger.debug(f" Class weights min: {weights.min().item():.6f}") + logger.debug(f" Class weights mean: {weights.mean().item():.6f}") + logger.debug(f" Class weights max: {weights.max().item():.6f}") if weights.min() <= 0: logger.error( @@ -261,10 +261,9 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - "Model initialized with %s trainable parameters", - format(count_model_params(self.model, only_trainable=True), ",d"), + f"Model initialized with {format(count_model_params(self.model, only_trainable=True), ',d')} trainable parameters" ) - logger.info("Total training steps: %d", total_steps) + logger.info(f"Total training steps: {total_steps}") def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -272,7 +271,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info("Loading model from checkpoint: %s", checkpoint_path) + logger.info(f"Loading model from checkpoint: {checkpoint_path}") checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -305,7 +304,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: num_batches = len(self.train_dataloader) start_time = time.time() val_frequency = max(500, num_batches // 5) - logger.info("🔧 Validation frequency: every %d batches", val_frequency) + logger.info(f"🔧 Validation frequency: every {val_frequency} batches") # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): @@ -337,7 +336,7 @@ def _handle_progressive_unfreezing(self, epoch: int) -> None: if epoch in self.unfreeze_schedule: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info("Epoch %d: Applied progressive unfreezing", epoch) + logger.info(f"Epoch {epoch}: Applied progressive unfreezing") def _train_single_batch( self, @@ -396,13 +395,9 @@ def _train_single_batch( if batch_idx < 5 or (batch_idx + 1) % 100 == 0: # We need to pass the current total loss from the calling method # For now, just log basic progress - logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f", - epoch, - batch_idx + 1, - num_batches, - loss.item(), - ) + logger.info( + f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {loss.item():.8f}" + ) return loss.item() @@ -455,10 +450,7 @@ def _create_epoch_metrics( } logger.info( - "Epoch %d completed - Loss: %.4f, Time: %.1fs", - epoch, - avg_loss, - epoch_time, + f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1fs}" ) return metrics @@ -471,14 +463,14 @@ def _log_data_distribution(labels: torch.Tensor) -> None: labels: Ground truth labels tensor """ logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(" Labels shape: %s", labels.shape) - logger.info(" Labels dtype: %s", labels.dtype) - logger.info(" Labels min: %s", labels.min().item()) - logger.info(" Labels max: %s", labels.max().item()) - logger.info(" Labels mean: %.6f", labels.float().mean().item()) - logger.info(" Labels sum: %s", labels.sum().item()) - logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) - logger.info(" Total labels: %s", labels.numel()) + logger.info(f" Labels shape: {labels.shape}") + logger.info(f" Labels dtype: {labels.dtype}") + logger.info(f" Labels min: {labels.min().item()}") + logger.info(f" Labels max: {labels.max().item()}") + logger.info(f" Labels mean: {labels.float().mean().item():.6f}") + logger.info(f" Labels sum: {labels.sum().item()}") + logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") + logger.info(f" Total labels: {labels.numel()}") if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -490,7 +482,7 @@ def _log_data_distribution(labels: torch.Tensor) -> None: for i in range(max_classes): class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(" Class %d: %d positive samples", i, int(class_count)) + logger.info(f" Class {i}: {int(class_count)} positive samples") @staticmethod def _log_model_output(logits: torch.Tensor) -> None: @@ -500,19 +492,19 @@ def _log_model_output(logits: torch.Tensor) -> None: logits: Model output logits tensor """ logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(" Logits shape: %s", logits.shape) - logger.info(" Logits min: %.6f", logits.min().item()) - logger.info(" Logits max: %.6f", logits.max().item()) - logger.info(" Logits mean: %.6f", logits.mean().item()) - logger.info(" Logits std: %.6f", logits.std().item()) + logger.info(f" Logits shape: {logits.shape}") + logger.info(f" Logits min: {logits.min().item():.6f}") + logger.info(f" Logits max: {logits.max().item():.6f}") + logger.info(f" Logits mean: {logits.mean().item():.6f}") + logger.info(f" Logits std: {logits.std().item():.6f}") if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") if torch.isinf(logits).any(): logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(" Predictions min: %.6f", predictions.min().item()) - logger.info(" Predictions max: %.6f", predictions.max().item()) - logger.info(" Predictions mean: %.6f", predictions.mean().item()) + logger.info(f" Predictions min: {predictions.min().item():.6f}") + logger.info(f" Predictions max: {predictions.max().item():.6f}") + logger.info(f" Predictions mean: {predictions.mean().item():.6f}") @staticmethod def _log_loss_analysis( @@ -526,11 +518,11 @@ def _log_loss_analysis( labels: Ground truth labels """ logger.info("🔍 DEBUG: Loss Analysis") - logger.info(" Raw loss: %.8f", loss.item()) + logger.info(f" Raw loss: {loss.item():.8f}") bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(" Manual BCE loss: %.8f", bce_manual.item()) + logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") logger.error(" This indicates a serious training issue!") @@ -540,7 +532,7 @@ def _log_loss_analysis( class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(" Class %d loss: %.8f", i, class_loss.item()) + logger.info(f" Class {i} loss: {class_loss.item():.8f}") def _log_gradient_stats_before(self) -> None: """Log gradient statistics before gradient clipping. @@ -558,7 +550,7 @@ def _log_gradient_stats_before(self) -> None: param_count += 1 if param_count > 0: total_norm = total_norm**0.5 - logger.info(" Gradient norm before clipping: %.6f", total_norm) + logger.info(f" Gradient norm before clipping: {total_norm:.6f}") if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") if total_norm < 1e-6: @@ -575,7 +567,7 @@ def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: clip_val = float(clip_norm) else: clip_val = clip_norm - logger.info(" Gradient norm after clipping: %.6f", clip_val) + logger.info(f" Gradient norm after clipping: {clip_val:.6f}") def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float @@ -591,20 +583,15 @@ def _log_progress( avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", - epoch, - batch_idx + 1, - num_batches, - avg_loss, - current_lr, + f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" ) if avg_loss < 1e-8: logger.error( - "❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss + f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}" ) if avg_loss > 100: logger.error( - "❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss + f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}" ) def _maybe_validate_and_early_stop( @@ -633,10 +620,10 @@ def _maybe_validate_and_early_stop( """ if (batch_idx + 1) % val_frequency != 0: return None - logger.info("🔍 Validating at batch %d...", batch_idx + 1) + logger.info(f"🔍 Validating at batch {batch_idx + 1}...") self.validate(epoch) if self.should_stop_early(): - logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) + logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") return { "epoch": epoch, "train_loss": total_loss / (batch_idx + 1), @@ -655,7 +642,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info("Validating model at epoch %d...", epoch) + logger.info(f"Validating model at epoch {epoch}...") val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -670,13 +657,11 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info("New best model saved! Macro F1: %.4f", current_score) + logger.info(f"New best model saved! Macro F1: {current_score:.4f}") else: self.patience_counter += 1 logger.info( - "No improvement. Patience: %d/%d", - self.patience_counter, - self.early_stopping_patience, + f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" ) return val_metrics @@ -716,7 +701,7 @@ def save_checkpoint( checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) - logger.info("Checkpoint saved: %s", checkpoint_path) + logger.info(f"Checkpoint saved: {checkpoint_path}") def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -741,7 +726,7 @@ def train(self) -> Dict[str, Any]: self.training_history.append(epoch_metrics) if self.should_stop_early(): - logger.info("Early stopping at epoch %d", epoch) + logger.info(f"Early stopping at epoch {epoch}") break else: self.training_history.append(train_metrics) @@ -771,7 +756,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info("Training history saved to %s", history_path) + logger.info(f"Training history saved to {history_path}") except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -791,7 +776,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info("Simplified training history saved to %s", history_path) + logger.info(f"Simplified training history saved to {history_path}") results = { "final_test_metrics": test_metrics, @@ -802,9 +787,9 @@ def convert_numpy_types(obj): } logger.info("✅ Training completed!") - logger.info("Best validation Macro F1: %.4f", self.best_score) - logger.info("Final test Macro F1: %.4f", test_metrics["macro_f1"]) - logger.info("Final test Micro F1: %.4f", test_metrics["micro_f1"]) + logger.info(f"Best validation Macro F1: {self.best_score:.4f}") + logger.info(f"Final test Macro F1: {test_metrics['macro_f1']:.4f}") + logger.info(f"Final test Micro F1: {test_metrics['micro_f1']:.4f}") return results From 0ea88e93a14b90d1d1d3026802ee9e1fdd85112c Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:37:31 +0200 Subject: [PATCH 20/28] Fix all 46 PYL-W1203 warnings: convert f-string logger calls to % formatting for performance --- .../emotion_detection/training_pipeline.py | 115 ++++++++++-------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 24aaf312f..dbf95f506 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -90,7 +90,7 @@ def __init__( else: self.device = torch.device(device) - logger.info(f"Using device: {self.device}") + logger.info("Using device: %s", self.device) self.output_dir.mkdir(parents=True, exist_ok=True) @@ -220,7 +220,7 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: if logger.isEnabledFor(logging.DEBUG): logger.debug("Loss Function Analysis") - logger.debug(f" Loss function type: {type(self.loss_fn).__name__}") + logger.debug(" Loss function type: %s", type(self.loss_fn).__name__) if ( hasattr(self.loss_fn, "class_weights") @@ -229,11 +229,11 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: weights = self.loss_fn.class_weights if logger.isEnabledFor(logging.DEBUG): logger.debug( - f" Class weights shape: {getattr(weights, 'shape', None)}" + " Class weights shape: %s", getattr(weights, "shape", None) ) - logger.debug(f" Class weights min: {weights.min().item():.6f}") - logger.debug(f" Class weights mean: {weights.mean().item():.6f}") - logger.debug(f" Class weights max: {weights.max().item():.6f}") + logger.debug(" Class weights min: %.6f", weights.min().item()) + logger.debug(" Class weights mean: %.6f", weights.mean().item()) + logger.debug(" Class weights max: %.6f", weights.max().item()) if weights.min() <= 0: logger.error( @@ -261,9 +261,10 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: ) logger.info( - f"Model initialized with {format(count_model_params(self.model, only_trainable=True), ',d')} trainable parameters" + "Model initialized with %s trainable parameters", + format(count_model_params(self.model, only_trainable=True), ",d"), ) - logger.info(f"Total training steps: {total_steps}") + logger.info("Total training steps: %d", total_steps) def load_model(self, checkpoint_path: str) -> None: """Load a trained model from checkpoint. @@ -271,7 +272,7 @@ def load_model(self, checkpoint_path: str) -> None: Args: checkpoint_path: Path to the model checkpoint file """ - logger.info(f"Loading model from checkpoint: {checkpoint_path}") + logger.info("Loading model from checkpoint: %s", checkpoint_path) checkpoint = torch.load(checkpoint_path, map_location=self.device) @@ -304,7 +305,7 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: num_batches = len(self.train_dataloader) start_time = time.time() val_frequency = max(500, num_batches // 5) - logger.info(f"🔧 Validation frequency: every {val_frequency} batches") + logger.info("🔧 Validation frequency: every %d batches", val_frequency) # Train on all batches for batch_idx, batch in enumerate(self.train_dataloader): @@ -336,7 +337,7 @@ def _handle_progressive_unfreezing(self, epoch: int) -> None: if epoch in self.unfreeze_schedule: layers_to_unfreeze = 2 # Unfreeze 2 layers at a time self.model.unfreeze_bert_layers(layers_to_unfreeze) - logger.info(f"Epoch {epoch}: Applied progressive unfreezing") + logger.info("Epoch %d: Applied progressive unfreezing", epoch) def _train_single_batch( self, @@ -396,7 +397,11 @@ def _train_single_batch( # We need to pass the current total loss from the calling method # For now, just log basic progress logger.info( - f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {loss.item():.8f}" + "Epoch %d, Batch %d/%d, Loss: %.8f", + epoch, + batch_idx + 1, + num_batches, + loss.item(), ) return loss.item() @@ -450,7 +455,10 @@ def _create_epoch_metrics( } logger.info( - f"Epoch {epoch} completed - Loss: {avg_loss:.4f}, Time: {epoch_time:.1fs}" + "Epoch %d completed - Loss: %.4f, Time: %.1fs", + epoch, + avg_loss, + epoch_time, ) return metrics @@ -463,14 +471,14 @@ def _log_data_distribution(labels: torch.Tensor) -> None: labels: Ground truth labels tensor """ logger.info("🔍 DEBUG: Data Distribution Analysis") - logger.info(f" Labels shape: {labels.shape}") - logger.info(f" Labels dtype: {labels.dtype}") - logger.info(f" Labels min: {labels.min().item()}") - logger.info(f" Labels max: {labels.max().item()}") - logger.info(f" Labels mean: {labels.float().mean().item():.6f}") - logger.info(f" Labels sum: {labels.sum().item()}") - logger.info(f" Non-zero labels: {(labels > 0).sum().item()}") - logger.info(f" Total labels: {labels.numel()}") + logger.info(" Labels shape: %s", labels.shape) + logger.info(" Labels dtype: %s", labels.dtype) + logger.info(" Labels min: %s", labels.min().item()) + logger.info(" Labels max: %s", labels.max().item()) + logger.info(" Labels mean: %.6f", labels.float().mean().item()) + logger.info(" Labels sum: %s", labels.sum().item()) + logger.info(" Non-zero labels: %s", (labels > 0).sum().item()) + logger.info(" Total labels: %s", labels.numel()) if labels.sum() == 0: logger.error("❌ CRITICAL: All labels are zero!") @@ -482,7 +490,7 @@ def _log_data_distribution(labels: torch.Tensor) -> None: for i in range(max_classes): class_count = labels[:, i].sum().item() if class_count > 0: - logger.info(f" Class {i}: {int(class_count)} positive samples") + logger.info(" Class %d: %d positive samples", i, int(class_count)) @staticmethod def _log_model_output(logits: torch.Tensor) -> None: @@ -492,19 +500,19 @@ def _log_model_output(logits: torch.Tensor) -> None: logits: Model output logits tensor """ logger.info("🔍 DEBUG: Model Output Analysis") - logger.info(f" Logits shape: {logits.shape}") - logger.info(f" Logits min: {logits.min().item():.6f}") - logger.info(f" Logits max: {logits.max().item():.6f}") - logger.info(f" Logits mean: {logits.mean().item():.6f}") - logger.info(f" Logits std: {logits.std().item():.6f}") + logger.info(" Logits shape: %s", logits.shape) + logger.info(" Logits min: %.6f", logits.min().item()) + logger.info(" Logits max: %.6f", logits.max().item()) + logger.info(" Logits mean: %.6f", logits.mean().item()) + logger.info(" Logits std: %.6f", logits.std().item()) if torch.isnan(logits).any(): logger.error("❌ CRITICAL: NaN values in logits!") if torch.isinf(logits).any(): logger.error("❌ CRITICAL: Inf values in logits!") predictions = torch.sigmoid(logits) - logger.info(f" Predictions min: {predictions.min().item():.6f}") - logger.info(f" Predictions max: {predictions.max().item():.6f}") - logger.info(f" Predictions mean: {predictions.mean().item():.6f}") + logger.info(" Predictions min: %.6f", predictions.min().item()) + logger.info(" Predictions max: %.6f", predictions.max().item()) + logger.info(" Predictions mean: %.6f", predictions.mean().item()) @staticmethod def _log_loss_analysis( @@ -518,11 +526,11 @@ def _log_loss_analysis( labels: Ground truth labels """ logger.info("🔍 DEBUG: Loss Analysis") - logger.info(f" Raw loss: {loss.item():.8f}") + logger.info(" Raw loss: %.8f", loss.item()) bce_manual = F.binary_cross_entropy_with_logits( logits, labels.float(), reduction="mean" ) - logger.info(f" Manual BCE loss: {bce_manual.item():.8f}") + logger.info(" Manual BCE loss: %.8f", bce_manual.item()) if abs(loss.item()) < 1e-10: logger.error("❌ CRITICAL: Loss is effectively zero!") logger.error(" This indicates a serious training issue!") @@ -532,7 +540,7 @@ def _log_loss_analysis( class_loss = F.binary_cross_entropy_with_logits( class_logits, class_labels, reduction="mean" ) - logger.info(f" Class {i} loss: {class_loss.item():.8f}") + logger.info(" Class %d loss: %.8f", i, class_loss.item()) def _log_gradient_stats_before(self) -> None: """Log gradient statistics before gradient clipping. @@ -550,7 +558,7 @@ def _log_gradient_stats_before(self) -> None: param_count += 1 if param_count > 0: total_norm = total_norm**0.5 - logger.info(f" Gradient norm before clipping: {total_norm:.6f}") + logger.info(" Gradient norm before clipping: %.6f", total_norm) if total_norm > 10: logger.warning("⚠️ WARNING: Large gradient norm detected!") if total_norm < 1e-6: @@ -567,7 +575,7 @@ def _log_gradient_stats_after(clip_norm: Union[float, torch.Tensor]) -> None: clip_val = float(clip_norm) else: clip_val = clip_norm - logger.info(f" Gradient norm after clipping: {clip_val:.6f}") + logger.info(" Gradient norm after clipping: %.6f", clip_val) def _log_progress( self, epoch: int, batch_idx: int, num_batches: int, total_loss: float @@ -583,15 +591,20 @@ def _log_progress( avg_loss = total_loss / (batch_idx + 1) current_lr = self.scheduler.get_last_lr()[0] logger.info( - f"Epoch {epoch}, Batch {batch_idx + 1}/{num_batches}, Loss: {avg_loss:.8f}, LR: {current_lr:.2e}" + "Epoch %d, Batch %d/%d, Loss: %.8f, LR: %.2e", + epoch, + batch_idx + 1, + num_batches, + avg_loss, + current_lr, ) if avg_loss < 1e-8: logger.error( - f"❌ CRITICAL: Average loss is suspiciously small: {avg_loss:.8f}" + "❌ CRITICAL: Average loss is suspiciously small: %.8f", avg_loss ) if avg_loss > 100: logger.error( - f"❌ CRITICAL: Average loss is suspiciously large: {avg_loss:.8f}" + "❌ CRITICAL: Average loss is suspiciously large: %.8f", avg_loss ) def _maybe_validate_and_early_stop( @@ -620,10 +633,10 @@ def _maybe_validate_and_early_stop( """ if (batch_idx + 1) % val_frequency != 0: return None - logger.info(f"🔍 Validating at batch {batch_idx + 1}...") + logger.info("🔍 Validating at batch %d...", batch_idx + 1) self.validate(epoch) if self.should_stop_early(): - logger.info(f"🛑 Early stopping triggered at batch {batch_idx + 1}") + logger.info("🛑 Early stopping triggered at batch %d", batch_idx + 1) return { "epoch": epoch, "train_loss": total_loss / (batch_idx + 1), @@ -642,7 +655,7 @@ def validate(self, epoch: int) -> Dict[str, float]: Returns: Dictionary with validation metrics """ - logger.info(f"Validating model at epoch {epoch}...") + logger.info("Validating model at epoch %d...", epoch) val_metrics = evaluate_emotion_classifier( self.model, self.val_dataloader, self.device, threshold=0.2 @@ -657,11 +670,13 @@ def validate(self, epoch: int) -> Dict[str, float]: if self.save_best_only: self.save_checkpoint(epoch, val_metrics, is_best=True) - logger.info(f"New best model saved! Macro F1: {current_score:.4f}") + logger.info("New best model saved! Macro F1: %.4f", current_score) else: self.patience_counter += 1 logger.info( - f"No improvement. Patience: {self.patience_counter}/{self.early_stopping_patience}" + "No improvement. Patience: %d/%d", + self.patience_counter, + self.early_stopping_patience, ) return val_metrics @@ -701,7 +716,7 @@ def save_checkpoint( checkpoint_path = self.output_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, checkpoint_path) - logger.info(f"Checkpoint saved: {checkpoint_path}") + logger.info("Checkpoint saved: %s", checkpoint_path) def train(self) -> Dict[str, Any]: """Complete training pipeline. @@ -726,7 +741,7 @@ def train(self) -> Dict[str, Any]: self.training_history.append(epoch_metrics) if self.should_stop_early(): - logger.info(f"Early stopping at epoch {epoch}") + logger.info("Early stopping at epoch %d", epoch) break else: self.training_history.append(train_metrics) @@ -756,7 +771,7 @@ def convert_numpy_types(obj): serializable_history = convert_numpy_types(self.training_history) with Path(history_path).open("w") as f: json.dump(serializable_history, f, indent=2) - logger.info(f"Training history saved to {history_path}") + logger.info("Training history saved to %s", history_path) except Exception: logger.exception("Failed to save training history") simplified_history = [] @@ -776,7 +791,7 @@ def convert_numpy_types(obj): with Path(history_path).open("w") as f: json.dump(simplified_history, f, indent=2) - logger.info(f"Simplified training history saved to {history_path}") + logger.info("Simplified training history saved to %s", history_path) results = { "final_test_metrics": test_metrics, @@ -787,9 +802,9 @@ def convert_numpy_types(obj): } logger.info("✅ Training completed!") - logger.info(f"Best validation Macro F1: {self.best_score:.4f}") - logger.info(f"Final test Macro F1: {test_metrics['macro_f1']:.4f}") - logger.info(f"Final test Micro F1: {test_metrics['micro_f1']:.4f}") + logger.info("Best validation Macro F1: %.4f", self.best_score) + logger.info("Final test Macro F1: %.4f", test_metrics["macro_f1"]) + logger.info("Final test Micro F1: %.4f", test_metrics["micro_f1"]) return results From 2e7bba1d9c95b66a05dc0030536103298451582a Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:48:09 +0200 Subject: [PATCH 21/28] Fix indentation issues in logger calls - resolve FLK-E122 warnings --- src/models/emotion_detection/training_pipeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index dbf95f506..07b0b5052 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -396,13 +396,13 @@ def _train_single_batch( if batch_idx < 5 or (batch_idx + 1) % 100 == 0: # We need to pass the current total loss from the calling method # For now, just log basic progress - logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f", - epoch, - batch_idx + 1, - num_batches, - loss.item(), - ) + logger.info( + "Epoch %d, Batch %d/%d, Loss: %.8f", + epoch, + batch_idx + 1, + num_batches, + loss.item(), + ) return loss.item() From bb4e5683f2447dc3beeb7cac2867edaee464d8eb Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:56:54 +0200 Subject: [PATCH 22/28] Fix three critical issues: 1) Remove ad-hoc logging from _train_single_batch and use _log_progress in train_epoch, 2) Fix tensor comparison issues by adding .item() to weights.min() and weights.max(), 3) Implement actual blocking in security headers for high-risk user agents --- .../emotion_detection/training_pipeline.py | 36 ++++++++----------- src/security_headers.py | 33 +++++++++++++---- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 07b0b5052..569387c2a 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -229,17 +229,17 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: weights = self.loss_fn.class_weights if logger.isEnabledFor(logging.DEBUG): logger.debug( - " Class weights shape: %s", getattr(weights, "shape", None) + " Class weights shape: %s", getattr(weights, "shape", None) ) logger.debug(" Class weights min: %.6f", weights.min().item()) logger.debug(" Class weights mean: %.6f", weights.mean().item()) logger.debug(" Class weights max: %.6f", weights.max().item()) - if weights.min() <= 0: + if weights.min().item() <= 0: logger.error( "❌ CRITICAL: Class weights contain zero or negative values!" ) - if weights.max() > 100: + if weights.max().item() > 100: logger.error("❌ CRITICAL: Class weights contain very large values!") else: logger.info(" No class weights used") @@ -312,6 +312,10 @@ def train_epoch(self, epoch: int) -> Dict[str, float]: batch_loss = self._train_single_batch(batch, batch_idx, epoch, num_batches) total_loss += batch_loss + # Log progress periodically + if batch_idx < 5 or (batch_idx + 1) % 100 == 0: + self._log_progress(epoch, batch_idx, num_batches, total_loss) + # Check for early stopping maybe_metrics = self._maybe_validate_and_early_stop( batch_idx, @@ -392,18 +396,6 @@ def _train_single_batch( self.optimizer.step() self.scheduler.step() - # Log progress periodically - if batch_idx < 5 or (batch_idx + 1) % 100 == 0: - # We need to pass the current total loss from the calling method - # For now, just log basic progress - logger.info( - "Epoch %d, Batch %d/%d, Loss: %.8f", - epoch, - batch_idx + 1, - num_batches, - loss.item(), - ) - return loss.item() def _log_batch_debug_info( @@ -779,14 +771,14 @@ def convert_numpy_types(obj): simplified_entry = {} for k, v in entry.items(): try: - if isinstance(v, (np.integer, np.floating)): - simplified_entry[k] = float(v.item()) - elif isinstance(v, (int, float, str, bool)): - simplified_entry[k] = v - else: - simplified_entry[k] = str(v) - except Exception: + if isinstance(v, (np.integer, np.floating)): + simplified_entry[k] = float(v.item()) + elif isinstance(v, (int, float, str, bool)): + simplified_entry[k] = v + else: simplified_entry[k] = str(v) + except Exception: + simplified_entry[k] = str(v) simplified_history.append(simplified_entry) with Path(history_path).open("w") as f: diff --git a/src/security_headers.py b/src/security_headers.py index 521b78649..494d377a8 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -96,6 +96,23 @@ def _before_request(self): if self.config.enable_correlation_id: g.correlation_id = request.headers.get("X-Correlation-ID", g.request_id) + # Check for high-risk requests and block them + if self.config.ua_blocking_enabled: + user_agent = request.headers.get("User-Agent", "") + ua_analysis = self._analyze_user_agent_enhanced(user_agent) + + if ua_analysis["risk_level"] in ["high", "very_high"]: + # Store blocking information for after_request logging + g.security_patterns = [f"BLOCKED: High-risk user agent - {ua_analysis['category']} (score: {ua_analysis['score']})"] + g.block_reason = f"User agent risk level: {ua_analysis['risk_level']}" + g.ua_analysis = ua_analysis + + # Return 403 Forbidden response + from flask import make_response + response = make_response("Access Forbidden - High-risk user agent detected", 403) + response.headers["Content-Type"] = "text/plain" + return response + # Log security-relevant request information self._log_security_info() @@ -110,6 +127,14 @@ def _after_request(self, response: Response) -> Response: # Log security-relevant response information self._log_response_security(response) + # Log blocking information if request was blocked + if hasattr(g, 'security_patterns'): + logger.warning(f"Request blocked: {g.security_patterns}") + if hasattr(g, 'block_reason'): + logger.warning(f"Block reason: {g.block_reason}") + if hasattr(g, 'ua_analysis'): + logger.warning(f"User agent analysis: {g.ua_analysis}") + return response def _add_security_headers(self, response: Response): @@ -448,12 +473,8 @@ def _detect_suspicious_patterns(self) -> List[str]: # Log detailed analysis logger.warning(f"User agent analysis: {ua_analysis}") - # Optionally block based on configuration - if self.config.ua_blocking_enabled and ua_analysis["risk_level"] in [ - "high", - "very_high", - ]: - patterns.append("BLOCKED: High-risk user agent") + # Note: High-risk user agents are now blocked in _before_request + # This is just for logging and pattern detection return patterns From bacc54674e2aafd05e78ac76894fb2741ae0e677 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:03:48 +0200 Subject: [PATCH 23/28] Fix code quality issues: convert f-string logger calls to % formatting, add explicit return None for PEP8 compliance, and fix long lines --- src/security_headers.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/security_headers.py b/src/security_headers.py index 494d377a8..869340820 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -103,18 +103,26 @@ def _before_request(self): if ua_analysis["risk_level"] in ["high", "very_high"]: # Store blocking information for after_request logging - g.security_patterns = [f"BLOCKED: High-risk user agent - {ua_analysis['category']} (score: {ua_analysis['score']})"] + g.security_patterns = [ + f"BLOCKED: High-risk user agent - {ua_analysis['category']} " + f"(score: {ua_analysis['score']})" + ] g.block_reason = f"User agent risk level: {ua_analysis['risk_level']}" g.ua_analysis = ua_analysis # Return 403 Forbidden response from flask import make_response - response = make_response("Access Forbidden - High-risk user agent detected", 403) + response = make_response( + "Access Forbidden - High-risk user agent detected", 403 + ) response.headers["Content-Type"] = "text/plain" return response # Log security-relevant request information self._log_security_info() + + # Explicit return None for consistency (PEP8 compliance) + return None def _after_request(self, response: Response) -> Response: """Process response after handling.""" @@ -129,11 +137,11 @@ def _after_request(self, response: Response) -> Response: # Log blocking information if request was blocked if hasattr(g, 'security_patterns'): - logger.warning(f"Request blocked: {g.security_patterns}") + logger.warning("Request blocked: %s", g.security_patterns) if hasattr(g, 'block_reason'): - logger.warning(f"Block reason: {g.block_reason}") + logger.warning("Block reason: %s", g.block_reason) if hasattr(g, 'ua_analysis'): - logger.warning(f"User agent analysis: {g.ua_analysis}") + logger.warning("User agent analysis: %s", g.ua_analysis) return response From 4c98b063c5fcbc85c8ad2137151163f2788c173d Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:04:39 +0200 Subject: [PATCH 24/28] Fix FLK-W293: remove whitespace from blank lines --- src/security_headers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/security_headers.py b/src/security_headers.py index 869340820..679c092f8 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -100,7 +100,7 @@ def _before_request(self): if self.config.ua_blocking_enabled: user_agent = request.headers.get("User-Agent", "") ua_analysis = self._analyze_user_agent_enhanced(user_agent) - + if ua_analysis["risk_level"] in ["high", "very_high"]: # Store blocking information for after_request logging g.security_patterns = [ @@ -109,7 +109,7 @@ def _before_request(self): ] g.block_reason = f"User agent risk level: {ua_analysis['risk_level']}" g.ua_analysis = ua_analysis - + # Return 403 Forbidden response from flask import make_response response = make_response( From df9cdc22b80e8ff3ce63e7dbfdeb6d8f7b5d27e5 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:06:03 +0200 Subject: [PATCH 25/28] Fix syntax error: correct indentation in try-except block around line 773 --- src/models/emotion_detection/training_pipeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 569387c2a..e8bb12c8b 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -771,14 +771,14 @@ def convert_numpy_types(obj): simplified_entry = {} for k, v in entry.items(): try: - if isinstance(v, (np.integer, np.floating)): - simplified_entry[k] = float(v.item()) - elif isinstance(v, (int, float, str, bool)): - simplified_entry[k] = v - else: + if isinstance(v, (np.integer, np.floating)): + simplified_entry[k] = float(v.item()) + elif isinstance(v, (int, float, str, bool)): + simplified_entry[k] = v + else: + simplified_entry[k] = str(v) + except Exception: simplified_entry[k] = str(v) - except Exception: - simplified_entry[k] = str(v) simplified_history.append(simplified_entry) with Path(history_path).open("w") as f: From 36c9ca54866feecfa5125439765dd672f1e857f8 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:07:34 +0200 Subject: [PATCH 26/28] Fix FLK-W293: remove trailing whitespace from blank line 123 --- src/security_headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/security_headers.py b/src/security_headers.py index 679c092f8..3b1eb0b2a 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -120,7 +120,7 @@ def _before_request(self): # Log security-relevant request information self._log_security_info() - + # Explicit return None for consistency (PEP8 compliance) return None From 7c508f5cb710207504c4f9795d4db5084d6db298 Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:08:59 +0200 Subject: [PATCH 27/28] Fix import consistency: convert absolute import to relative import for count_model_params --- src/models/emotion_detection/training_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index e8bb12c8b..64681eaf9 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -19,7 +19,7 @@ from torch.utils.data import DataLoader from transformers import AutoTokenizer, get_linear_schedule_with_warmup -from src.utils import count_model_params +from ...utils import count_model_params from .bert_classifier import create_bert_emotion_classifier, evaluate_emotion_classifier from .dataset_loader import GoEmotionsDataset, create_goemotions_loader From 50354ed80d22c800b29009c0bbbdab0e11161cdb Mon Sep 17 00:00:00 2001 From: Deniz Ulker <156104354+uelkerd@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:09:36 +0200 Subject: [PATCH 28/28] Fix FLK-E122: correct indentation of continuation line in logger.debug call --- src/models/emotion_detection/training_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/emotion_detection/training_pipeline.py b/src/models/emotion_detection/training_pipeline.py index 64681eaf9..6267981ef 100644 --- a/src/models/emotion_detection/training_pipeline.py +++ b/src/models/emotion_detection/training_pipeline.py @@ -229,7 +229,7 @@ def initialize_model(self, class_weights: Optional[np.ndarray] = None) -> None: weights = self.loss_fn.class_weights if logger.isEnabledFor(logging.DEBUG): logger.debug( - " Class weights shape: %s", getattr(weights, "shape", None) + " Class weights shape: %s", getattr(weights, "shape", None) ) logger.debug(" Class weights min: %.6f", weights.min().item()) logger.debug(" Class weights mean: %.6f", weights.mean().item())