Merge pull request #8 from emmanueljordy/add_data_processing

Add data processing

Author: jfparie

synthpop/metrics/efficacy_metrics.py

@@ -76,15 +76,15 @@ def evaluate(self, real_df: pd.DataFrame, synthetic_df: pd.DataFrame) -> dict:
         y_real = real_df[self.target_column]
 
         # Handle categorical encoding only if it's a classification task
-        if self.task == 'classification':
-            categorical_cols = X_syn.select_dtypes(include=['object', 'category']).columns.tolist()
+        
+        categorical_cols = X_syn.select_dtypes(include=['object', 'category']).columns.tolist()
 
-            if categorical_cols:
-                X_syn = pd.get_dummies(X_syn, columns=categorical_cols, drop_first=True)
-                X_real = pd.get_dummies(X_real, columns=categorical_cols, drop_first=True)
+        if categorical_cols:
+            X_syn = pd.get_dummies(X_syn, columns=categorical_cols, drop_first=True)
+            X_real = pd.get_dummies(X_real, columns=categorical_cols, drop_first=True)
 
-                # Align columns in case of different categorical levels between real and synthetic data
-                X_syn, X_real = X_syn.align(X_real, join='left', axis=1, fill_value=0)
+            # Align columns in case of different categorical levels between real and synthetic data
+            X_syn, X_real = X_syn.align(X_real, join='left', axis=1, fill_value=0)
 
         # Model Training and Evaluation
         if self.task == 'regression':

synthpop/processor/data_processor.py

@@ -61,7 +61,7 @@ def _preprocess(self, data: pd.DataFrame) -> pd.DataFrame:
                 data = pd.concat([data, transformed_data], axis=1)
 
             elif dtype == "numerical":
-                scaler = StandardScaler()
+                scaler = StandardScaler(with_mean= False, with_std= False)
                 data[col] = scaler.fit_transform(data[[col]])
                 self.scalers[col] = scaler