ml-basic-functions.py

A developer shared a Python script for building a machine learning pipeline for real estate price prediction. The script includes exploratory data analysis, outlier filtering, feature engineering with KMeans clustering, and preprocessing steps using scikit-learn and XGBoost.

| import numpy as np | | | import pandas as pd | | | import matplotlib.pyplot as plt | | | import seaborn as sns | | | from sklearn.model selection import train test split, GridSearchCV, cross validate | | | from sklearn.pipeline import Pipeline | | | from sklearn.compose import ColumnTransformer, TransformedTargetRegressor | | | from sklearn.preprocessing import StandardScaler, OneHotEncoder, TargetEncoder | | | from sklearn.impute import SimpleImputer | | | from sklearn.feature selection import mutual info regression | | | from sklearn.cluster import KMeans | | | from sklearn.linear model import LinearRegression, RidgeCV, LassoCV | | | from xgboost import XGBRegressor | | | from sklearn.metrics import mean absolute error, root mean squared error, r2 score, median absolute error | | | from sklearn.inspection import permutation importance | | | ========================================== | | | 0. LOAD DATA Mock Example | | | ========================================== | | | Features: 'price', 'surface area', 'rooms', 'zip code', 'property type', 'latitude', 'longitude' | | | df = pd.read csv 'data.csv' | | | ========================================== | | | 1. EXPLORATORY DATA ANALYSIS EDA | | | ========================================== | | | sns.set theme style="whitegrid" | | | Target Distribution | | | plt.figure figsize= 10, 5 | | | sns.histplot data=df, x='price', kde=True, bins=50 | | | plt.title "Target Distribution: Price" | | | plt.show | | | Geographical Price Heatmap | | | plt.figure figsize= 10, 8 | | | sns.scatterplot x='longitude', y='latitude', hue='price', size='surface area', | | | sizes= 10, 200 , palette='viridis', data=df, alpha=0.7 | | | plt.title "Geographical Distribution of Prices" | | | plt.show | | | Multivariate Relationships Pairplot on subset to save time | | | sns.pairplot df 'price', 'surface area', 'rooms' , diag kind='kde' | | | plt.suptitle "Pairplot of Key Numerical Features", y=1.02 | | | plt.show | | | Correlation Matrix | | | plt.figure figsize= 8, 6 | | | sns.heatmap data=df.select dtypes include=np.number .corr , annot=True, cmap='coolwarm', fmt=".2f" | | | plt.title "Feature Correlation Matrix" | | | plt.show | | | ========================================== | | | 2. SPLIT & MANUAL OUTLIER FILTERING | | | ========================================== | | | We can also add a step here to remove rows with NAs for a subset of columns. Do before split to avoid crashing... | | | If we don't have a price, we can't train or test. | | | If we don't have a surface area, we refuse to predict. | | | columns to drop if na = 'price', 'surface area' | | | df clean = df.dropna subset=columns to drop if na .copy | | | X = df.drop 'price', axis=1 | | | y = df 'price' | | | Split BEFORE filtering to prevent leakage | | | X train, X test, y train, y test = train test split X, y, test size=0.2, random state=42 | | | Manual Outlier Filtering Logic-based from observing scatterplots | | | Example: Remove properties 500 sqm or prices that are obvious data entry errors | | | valid idx = X train 'surface area' < 500 & y train 10000 | | | X train = X train valid idx .copy | | | y train = y train valid idx .copy | | | ========================================== | | | 3. FEATURE ENGINEERING | | | ========================================== | | | Spatial Clustering Fit on train, transform both | | | kmeans = KMeans n clusters=8, random state=42, n init=10 | | | X train 'geo cluster' = kmeans.fit predict X train 'latitude', 'longitude' .astype str | | | X test 'geo cluster' = kmeans.predict X test 'latitude', 'longitude' .astype str | | | ========================================== | | | 4. PREPROCESSING | | | ========================================== | | | Group features by the exact mathematical transformation they require | | | features to scale = 'surface area', 'rooms', 'latitude', 'longitude' | | | features to ohe = 'property type', 'geo cluster' | | | features to target encode = 'zip code' | | | features to passthrough = 'has balcony' Boolean flags that need no transformation | | | Pipeline for features that need scaling with fallback median imputation | | | scale pipeline = Pipeline | | | 'imputer', SimpleImputer strategy='median' , | | | 'scaler', StandardScaler | | | | | | Pipeline for OHE with fallback mode imputation | | | ohe pipeline = Pipeline | | | 'imputer', SimpleImputer strategy='most frequent' , | | | 'ohe', OneHotEncoder handle unknown='ignore', sparse output=False | | | | | | Pipeline for Target Encoding | | | target enc pipeline = Pipeline | | | 'imputer', SimpleImputer strategy='most frequent' , | | | 'target enc', TargetEncoder smooth="auto" | | | | | | preprocessor = ColumnTransformer | | | transformers= | | | 'scale', scale pipeline, features to scale , | | | 'ohe', ohe pipeline, features to ohe , | | | 'target enc', target enc pipeline, features to target encode , | | | 'passthrough', 'passthrough', features to passthrough | | | , | | | remainder='drop' Ensures any unassigned columns are discarded | | | | | | ========================================== | | | 5. MODELING LINEAR & XGBOOST VARIANTS | | | ========================================== | | | --- 5.1 Standard Linear Regression --- | | | Model 1: No Transformation | | | pipeline lr = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', LinearRegression | | | | | | Model 2: Transformed Target log1p | | | pipeline lr log = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', TransformedTargetRegressor | | | regressor=LinearRegression , func=np.log1p, inverse func=np.expm1 | | | | | | | | | --- 5.2 Regularized Linear Models Transformed Target --- | | | Model 3: Ridge Regression log1p | | | pipeline ridge log = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', TransformedTargetRegressor | | | regressor=RidgeCV alphas=np.logspace -3, 3, 10 , func=np.log1p, inverse func=np.expm1 | | | | | | | | | Model 4: Lasso Regression log1p | | | Note: Increased max iter for Lasso as it can struggle to converge on unscaled dummy variables | | | pipeline lasso log = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', TransformedTargetRegressor | | | regressor=LassoCV alphas=np.logspace -5, 2, 10 , max iter=10000 , func=np.log1p, inverse func=np.expm1 | | | | | | | | | --- 5.3 XGBoost Models --- | | | Model 5: XGBoost No Transformation | | | pipeline xgb = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', XGBRegressor random state=42 | | | | | | Model 6: XGBoost Transformed Target log1p | | | pipeline xgb log = Pipeline | | | 'preprocessor', preprocessor , | | | 'model', TransformedTargetRegressor | | | regressor=XGBRegressor random state=42 , func=np.log1p, inverse func=np.expm1 | | | | | | | | | ========================================== | | | 6. HYPERPARAMETER TUNING & FITTING | | | ========================================== | | | 6.1 Fit all Linear Models directly Ridge/Lasso handle tuning internally via CV | | | pipeline lr.fit X train, y train | | | pipeline lr log.fit X train, y train | | | pipeline ridge log.fit X train, y train | | | pipeline lasso log.fit X train, y train | | | 6.2 Tune XGBoost No Transformation | | | Notice the param grid keys start with 'model ' because the regressor is directly in the pipeline | | | param grid standard = { | | | 'model n estimators': 100, 250 , | | | 'model learning rate': 0.05, 0.1 , | | | 'model max depth': 4, 6 | | | } | | | grid search xgb = GridSearchCV pipeline xgb, param grid standard, cv=5, scoring='neg root mean squared error', n jobs=-1 | | | grid search xgb.fit X train, y train | | | best xgb = grid search xgb.best estimator | | | 6.3 Tune XGBoost Transformed Target | | | Notice the keys start with 'model regressor ' because of the TransformedTargetRegressor wrapper | | | param grid log = { | | | 'model regressor n estimators': 100, 250 , | | | 'model regressor learning rate': 0.05, 0.1 , | | | 'model regressor max depth': 4, 6 | | | } | | | grid search xgb log = GridSearchCV pipeline xgb log, param grid log, cv=5, scoring='neg root mean squared error', n jobs=-1 | | | grid search xgb log.fit X train, y train | | | best xgb log = grid search xgb log.best estimator | | | ========================================== | | | 7. RESULTS ANALYSIS | | | ========================================== | | | def evaluate model name, model, X test, y test : | | | y pred = model.predict X test | | | r2 = r2 score y test, y pred | | | n, p = X test.shape 0 , X test.shape 1 | | | adj r2 = 1 - 1 - r2 n - 1 / n - p - 1 | | | print f"--- {name} Performance ---" | | | print f"RMSE: {root mean squared error y test, y pred :,.2f}" | | | print f"MAE: {mean absolute error y test, y pred :,.2f}" | | | print f"MedAE: {median absolute error y test, y pred :,.2f}" | | | print f"R2: {r2:.3f}" | | | print f"Adj R2: {adj r2:.3f}\n" | | | evaluate model "Linear Regression Standard ", pipeline lr, X test, y test | | | evaluate model "Linear Regression Log Target ", pipeline lr log, X test, y test | | | evaluate model "Ridge Regression Log Target ", pipeline ridge log, X test, y test | | | evaluate model "Lasso Regression Log Target ", pipeline lasso log, X test, y test | | | evaluate model "XGBoost Standard, Tuned ", best xgb, X test, y test | | | evaluate model "XGBoost Log Target, Tuned ", best xgb log, X test, y test | | | ========================================== | | | 8. FEATURE IMPORTANCE & EXPLAINABILITY | | | ========================================== | | | Extract feature names from ColumnTransformer | | | Note: Adjust 'ohe' and 'categorical features' based on the transformation-centric lists defined earlier | | | cat features out = preprocessor.named transformers 'ohe' .named steps 'ohe' .get feature names out features to ohe .tolist | | | all feature names = features to scale + cat features out + features to target encode + features to passthrough | | | --- 8.1 Linear Model Coefficients Example using Ridge Log --- | | | Because Ridge is wrapped in TransformedTargetRegressor, we must access .regressor | | | ridge estimator = pipeline ridge log.named steps 'model' .regressor | | | coefs = pd.Series ridge estimator.coef , index=all feature names .sort values key=abs, ascending=False | | | plt.figure figsize= 10, 6 | | | sns.barplot x=coefs.head 15 .values, y=coefs.head 15 .index, palette="coolwarm" | | | plt.title "Top 15 Feature Coefficients Ridge Regression - Log Target " | | | plt.xlabel "Coefficient Value Log Scale Impact " | | | plt.show | | | --- 8.2 Tree Model Permutation Importance Example using XGBoost Log --- | | | Transform test data to feed directly into the base regressor | | | X test transformed = preprocessor.transform X test | | | Since we are evaluating the base estimator of the wrapped model, we must transform the target manually | | | y test log = np.log1p y test | | | xgb estimator log = best xgb log.named steps 'model' .regressor | | | If we were evaluating the non-transformed XGBoost, it would be: | | | xgb estimator standard = best xgb.named steps 'model' | | | and we would pass y test without transforming it. | | | result = permutation importance xgb estimator log, X test transformed, y test log, | | | n repeats=5, random state=42, scoring='neg root mean squared error' | | | perm importance = pd.Series result.importances mean, index=all feature names .sort values ascending=False | | | plt.figure figsize= 10, 6 | | | sns.barplot x=perm importance.head 15 .values, y=perm importance.head 15 .index, palette="viridis" | | | plt.title "Top 15 Feature Permutation Importances XGBoost - Log Target " | | | plt.xlabel "Increase in RMSE when feature is shuffled" | | | plt.show |