{"slug": "ml-basic-functions-py", "title": "ml-basic-functions.py", "summary": "A developer shared a Python script for building a machine learning pipeline for real estate price prediction. The script includes exploratory data analysis, outlier filtering, feature engineering with KMeans clustering, and preprocessing steps using scikit-learn and XGBoost.", "body_md": "| import numpy as np | |\n| import pandas as pd | |\n| import matplotlib.pyplot as plt | |\n| import seaborn as sns | |\n| from sklearn.model_selection import train_test_split, GridSearchCV, cross_validate | |\n| from sklearn.pipeline import Pipeline | |\n| from sklearn.compose import ColumnTransformer, TransformedTargetRegressor | |\n| from sklearn.preprocessing import StandardScaler, OneHotEncoder, TargetEncoder | |\n| from sklearn.impute import SimpleImputer | |\n| from sklearn.feature_selection import mutual_info_regression | |\n| from sklearn.cluster import KMeans | |\n| from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV | |\n| from xgboost import XGBRegressor | |\n| from sklearn.metrics import mean_absolute_error, root_mean_squared_error, r2_score, median_absolute_error | |\n| from sklearn.inspection import permutation_importance | |\n| # ========================================== | |\n| # 0. LOAD DATA (Mock Example) | |\n| # ========================================== | |\n| # Features: 'price', 'surface_area', 'rooms', 'zip_code', 'property_type', 'latitude', 'longitude' | |\n| # df = pd.read_csv('data.csv') | |\n| # ========================================== | |\n| # 1. EXPLORATORY DATA ANALYSIS (EDA) | |\n| # ========================================== | |\n| sns.set_theme(style=\"whitegrid\") | |\n| # Target Distribution | |\n| plt.figure(figsize=(10, 5)) | |\n| sns.histplot(data=df, x='price', kde=True, bins=50) | |\n| plt.title(\"Target Distribution: Price\") | |\n| plt.show() | |\n| # Geographical Price Heatmap | |\n| plt.figure(figsize=(10, 8)) | |\n| sns.scatterplot(x='longitude', y='latitude', hue='price', size='surface_area', | |\n| sizes=(10, 200), palette='viridis', data=df, alpha=0.7) | |\n| plt.title(\"Geographical Distribution of Prices\") | |\n| plt.show() | |\n| # Multivariate Relationships (Pairplot on subset to save time) | |\n| sns.pairplot(df[['price', 'surface_area', 'rooms']], diag_kind='kde') | |\n| plt.suptitle(\"Pairplot of Key Numerical Features\", y=1.02) | |\n| plt.show() | |\n| # Correlation Matrix | |\n| plt.figure(figsize=(8, 6)) | |\n| sns.heatmap(data=df.select_dtypes(include=np.number).corr(), annot=True, cmap='coolwarm', fmt=\".2f\") | |\n| plt.title(\"Feature Correlation Matrix\") | |\n| plt.show() | |\n| # ========================================== | |\n| # 2. SPLIT & MANUAL OUTLIER FILTERING | |\n| # ========================================== | |\n| # We can also add a step here to remove rows with NAs for a subset of columns. Do before split to avoid crashing... | |\n| # If we don't have a price, we can't train or test. | |\n| # If we don't have a surface area, we refuse to predict. | |\n| columns_to_drop_if_na = ['price', 'surface_area'] | |\n| df_clean = df.dropna(subset=columns_to_drop_if_na).copy() | |\n| X = df.drop('price', axis=1) | |\n| y = df['price'] | |\n| # Split BEFORE filtering to prevent leakage | |\n| X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) | |\n| # Manual Outlier Filtering (Logic-based from observing scatterplots) | |\n| # Example: Remove properties > 500 sqm or prices that are obvious data entry errors | |\n| valid_idx = (X_train['surface_area'] < 500) & (y_train > 10000) | |\n| X_train = X_train[valid_idx].copy() | |\n| y_train = y_train[valid_idx].copy() | |\n| # ========================================== | |\n| # 3. FEATURE ENGINEERING | |\n| # ========================================== | |\n| # Spatial Clustering (Fit on train, transform both) | |\n| kmeans = KMeans(n_clusters=8, random_state=42, n_init=10) | |\n| X_train['geo_cluster'] = kmeans.fit_predict(X_train[['latitude', 'longitude']]).astype(str) | |\n| X_test['geo_cluster'] = kmeans.predict(X_test[['latitude', 'longitude']]).astype(str) | |\n| # ========================================== | |\n| # 4. PREPROCESSING | |\n| # ========================================== | |\n| # Group features by the exact mathematical transformation they require | |\n| features_to_scale = ['surface_area', 'rooms', 'latitude', 'longitude'] | |\n| features_to_ohe = ['property_type', 'geo_cluster'] | |\n| features_to_target_encode = ['zip_code'] | |\n| features_to_passthrough = ['has_balcony'] # Boolean flags that need no transformation | |\n| # Pipeline for features that need scaling (with fallback median imputation) | |\n| scale_pipeline = Pipeline([ | |\n| ('imputer', SimpleImputer(strategy='median')), | |\n| ('scaler', StandardScaler()) | |\n| ]) | |\n| # Pipeline for OHE (with fallback mode imputation) | |\n| ohe_pipeline = Pipeline([ | |\n| ('imputer', SimpleImputer(strategy='most_frequent')), | |\n| ('ohe', OneHotEncoder(handle_unknown='ignore', sparse_output=False)) | |\n| ]) | |\n| # Pipeline for Target Encoding | |\n| target_enc_pipeline = Pipeline([ | |\n| ('imputer', SimpleImputer(strategy='most_frequent')), | |\n| ('target_enc', TargetEncoder(smooth=\"auto\")) | |\n| ]) | |\n| preprocessor = ColumnTransformer( | |\n| transformers=[ | |\n| ('scale', scale_pipeline, features_to_scale), | |\n| ('ohe', ohe_pipeline, features_to_ohe), | |\n| ('target_enc', target_enc_pipeline, features_to_target_encode), | |\n| ('passthrough', 'passthrough', features_to_passthrough) | |\n| ], | |\n| remainder='drop' # Ensures any unassigned columns are discarded | |\n| ) | |\n| # ========================================== | |\n| # 5. MODELING (LINEAR & XGBOOST VARIANTS) | |\n| # ========================================== | |\n| # --- 5.1 Standard Linear Regression --- | |\n| # Model 1: No Transformation | |\n| pipeline_lr = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', LinearRegression()) | |\n| ]) | |\n| # Model 2: Transformed Target (log1p) | |\n| pipeline_lr_log = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', TransformedTargetRegressor( | |\n| regressor=LinearRegression(), func=np.log1p, inverse_func=np.expm1 | |\n| )) | |\n| ]) | |\n| # --- 5.2 Regularized Linear Models (Transformed Target) --- | |\n| # Model 3: Ridge Regression (log1p) | |\n| pipeline_ridge_log = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', TransformedTargetRegressor( | |\n| regressor=RidgeCV(alphas=np.logspace(-3, 3, 10)), func=np.log1p, inverse_func=np.expm1 | |\n| )) | |\n| ]) | |\n| # Model 4: Lasso Regression (log1p) | |\n| # Note: Increased max_iter for Lasso as it can struggle to converge on unscaled dummy variables | |\n| pipeline_lasso_log = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', TransformedTargetRegressor( | |\n| regressor=LassoCV(alphas=np.logspace(-5, 2, 10), max_iter=10000), func=np.log1p, inverse_func=np.expm1 | |\n| )) | |\n| ]) | |\n| # --- 5.3 XGBoost Models --- | |\n| # Model 5: XGBoost (No Transformation) | |\n| pipeline_xgb = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', XGBRegressor(random_state=42)) | |\n| ]) | |\n| # Model 6: XGBoost (Transformed Target log1p) | |\n| pipeline_xgb_log = Pipeline([ | |\n| ('preprocessor', preprocessor), | |\n| ('model', TransformedTargetRegressor( | |\n| regressor=XGBRegressor(random_state=42), func=np.log1p, inverse_func=np.expm1 | |\n| )) | |\n| ]) | |\n| # ========================================== | |\n| # 6. HYPERPARAMETER TUNING & FITTING | |\n| # ========================================== | |\n| # 6.1 Fit all Linear Models directly (Ridge/Lasso handle tuning internally via CV) | |\n| pipeline_lr.fit(X_train, y_train) | |\n| pipeline_lr_log.fit(X_train, y_train) | |\n| pipeline_ridge_log.fit(X_train, y_train) | |\n| pipeline_lasso_log.fit(X_train, y_train) | |\n| # 6.2 Tune XGBoost (No Transformation) | |\n| # Notice the param grid keys start with 'model__' because the regressor is directly in the pipeline | |\n| param_grid_standard = { | |\n| 'model__n_estimators': [100, 250], | |\n| 'model__learning_rate': [0.05, 0.1], | |\n| 'model__max_depth': [4, 6] | |\n| } | |\n| grid_search_xgb = GridSearchCV(pipeline_xgb, param_grid_standard, cv=5, scoring='neg_root_mean_squared_error', n_jobs=-1) | |\n| grid_search_xgb.fit(X_train, y_train) | |\n| best_xgb = grid_search_xgb.best_estimator_ | |\n| # 6.3 Tune XGBoost (Transformed Target) | |\n| # Notice the keys start with 'model__regressor__' because of the TransformedTargetRegressor wrapper | |\n| param_grid_log = { | |\n| 'model__regressor__n_estimators': [100, 250], | |\n| 'model__regressor__learning_rate': [0.05, 0.1], | |\n| 'model__regressor__max_depth': [4, 6] | |\n| } | |\n| grid_search_xgb_log = GridSearchCV(pipeline_xgb_log, param_grid_log, cv=5, scoring='neg_root_mean_squared_error', n_jobs=-1) | |\n| grid_search_xgb_log.fit(X_train, y_train) | |\n| best_xgb_log = grid_search_xgb_log.best_estimator_ | |\n| # ========================================== | |\n| # 7. RESULTS ANALYSIS | |\n| # ========================================== | |\n| def evaluate_model(name, model, X_test, y_test): | |\n| y_pred = model.predict(X_test) | |\n| r2 = r2_score(y_test, y_pred) | |\n| n, p = X_test.shape[0], X_test.shape[1] | |\n| adj_r2 = 1 - (1 - r2) * (n - 1) / (n - p - 1) | |\n| print(f\"--- {name} Performance ---\") | |\n| print(f\"RMSE: {root_mean_squared_error(y_test, y_pred):,.2f}\") | |\n| print(f\"MAE: {mean_absolute_error(y_test, y_pred):,.2f}\") | |\n| print(f\"MedAE: {median_absolute_error(y_test, y_pred):,.2f}\") | |\n| print(f\"R2: {r2:.3f}\") | |\n| print(f\"Adj R2: {adj_r2:.3f}\\n\") | |\n| evaluate_model(\"Linear Regression (Standard)\", pipeline_lr, X_test, y_test) | |\n| evaluate_model(\"Linear Regression (Log Target)\", pipeline_lr_log, X_test, y_test) | |\n| evaluate_model(\"Ridge Regression (Log Target)\", pipeline_ridge_log, X_test, y_test) | |\n| evaluate_model(\"Lasso Regression (Log Target)\", pipeline_lasso_log, X_test, y_test) | |\n| evaluate_model(\"XGBoost (Standard, Tuned)\", best_xgb, X_test, y_test) | |\n| evaluate_model(\"XGBoost (Log Target, Tuned)\", best_xgb_log, X_test, y_test) | |\n| # ========================================== | |\n| # 8. FEATURE IMPORTANCE & EXPLAINABILITY | |\n| # ========================================== | |\n| # Extract feature names from ColumnTransformer | |\n| # Note: Adjust 'ohe' and 'categorical_features' based on the transformation-centric lists defined earlier | |\n| cat_features_out = preprocessor.named_transformers_['ohe'].named_steps['ohe'].get_feature_names_out(features_to_ohe).tolist() | |\n| all_feature_names = features_to_scale + cat_features_out + features_to_target_encode + features_to_passthrough | |\n| # --- 8.1 Linear Model Coefficients (Example using Ridge Log) --- | |\n| # Because Ridge is wrapped in TransformedTargetRegressor, we must access .regressor_ | |\n| ridge_estimator = pipeline_ridge_log.named_steps['model'].regressor_ | |\n| coefs = pd.Series(ridge_estimator.coef_, index=all_feature_names).sort_values(key=abs, ascending=False) | |\n| plt.figure(figsize=(10, 6)) | |\n| sns.barplot(x=coefs.head(15).values, y=coefs.head(15).index, palette=\"coolwarm\") | |\n| plt.title(\"Top 15 Feature Coefficients (Ridge Regression - Log Target)\") | |\n| plt.xlabel(\"Coefficient Value (Log Scale Impact)\") | |\n| plt.show() | |\n| # --- 8.2 Tree Model Permutation Importance (Example using XGBoost Log) --- | |\n| # Transform test data to feed directly into the base regressor | |\n| X_test_transformed = preprocessor.transform(X_test) | |\n| # Since we are evaluating the base estimator of the wrapped model, we must transform the target manually | |\n| y_test_log = np.log1p(y_test) | |\n| xgb_estimator_log = best_xgb_log.named_steps['model'].regressor_ | |\n| # If we were evaluating the non-transformed XGBoost, it would be: | |\n| # xgb_estimator_standard = best_xgb.named_steps['model'] | |\n| # and we would pass y_test without transforming it. | |\n| result = permutation_importance(xgb_estimator_log, X_test_transformed, y_test_log, | |\n| n_repeats=5, random_state=42, scoring='neg_root_mean_squared_error') | |\n| perm_importance = pd.Series(result.importances_mean, index=all_feature_names).sort_values(ascending=False) | |\n| plt.figure(figsize=(10, 6)) | |\n| sns.barplot(x=perm_importance.head(15).values, y=perm_importance.head(15).index, palette=\"viridis\") | |\n| plt.title(\"Top 15 Feature Permutation Importances (XGBoost - Log Target)\") | |\n| plt.xlabel(\"Increase in RMSE when feature is shuffled\") | |\n| plt.show() |", "url": "https://wpnews.pro/news/ml-basic-functions-py", "canonical_source": "https://gist.github.com/bajam1/3503347df9ee3fb8c6caab8d8fe279c1", "published_at": "2026-08-26 15:14:14+00:00", "updated_at": "2026-08-26 15:45:10.234703+00:00", "lang": "en", "topics": ["machine-learning", "developer-tools"], "entities": ["scikit-learn", "XGBoost", "pandas", "NumPy", "matplotlib", "seaborn"], "alternates": {"html": "https://wpnews.pro/news/ml-basic-functions-py", "markdown": "https://wpnews.pro/news/ml-basic-functions-py.md", "text": "https://wpnews.pro/news/ml-basic-functions-py.txt", "jsonld": "https://wpnews.pro/news/ml-basic-functions-py.jsonld"}}