From 9f95d7139d91c404f1c7cbda2f346c992bc69dbc Mon Sep 17 00:00:00 2001 From: sravanyabezawada <67137275+sravanyabezawada@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:22:53 -0500 Subject: [PATCH 1/5] Update ElasticNet.py --- elasticnet/models/ElasticNet.py | 131 +++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 10 deletions(-) diff --git a/elasticnet/models/ElasticNet.py b/elasticnet/models/ElasticNet.py index 017e925..2eff404 100644 --- a/elasticnet/models/ElasticNet.py +++ b/elasticnet/models/ElasticNet.py @@ -1,17 +1,128 @@ +import numpy as np +import matplotlib.pyplot as plt +from numpy.linalg import inv +# ------------------- Utility Functions -------------------- -class ElasticNetModel(): - def __init__(self): - pass +def normalize_features(X): + """Normalize the feature matrix to have zero mean and unit variance""" + avg = np.mean(X, axis=0) + deviation = np.std(X, axis=0) + return (X - avg) / deviation, avg, deviation +def compute_performance(y_actual, y_predicted): + """Compute Mean Squared Error and R-squared""" + mse = np.mean((y_actual - y_predicted) ** 2) + r_squared = 1 - (np.sum((y_actual - y_predicted) ** 2) / np.sum((y_actual - np.mean(y_actual)) ** 2)) + residuals = y_actual - y_predicted + return mse, r_squared, residuals - def fit(self, X, y): - return ElasticNetModelResults() +def show_comparison_plot(y_actual, y_predicted, title="Actual vs Predicted"): + """Scatter plot for actual vs predicted values""" + plt.scatter(y_actual, y_predicted, color='green', label='Predicted vs Actual') + plt.plot([min(y_actual), max(y_actual)], [min(y_actual), max(y_actual)], color='red', label='Perfect Fit') + plt.xlabel('Actual Values') + plt.ylabel('Predicted Values') + plt.title(title) + plt.legend() + plt.show() +def show_residual_distribution(residuals, title="Residual Distribution"): + """Histogram for residual distribution""" + plt.hist(residuals, bins=15, color='orange', edgecolor='black') + plt.title(title) + plt.xlabel('Residuals') + plt.ylabel('Frequency') + plt.show() -class ElasticNetModelResults(): - def __init__(self): - pass +def display_model_performance(model_name, mse, r_squared): + """Print out the performance metrics of the model""" + print(f"\nPerformance for {model_name}:") + print(f"MSE: {mse:.3f}") + print(f"R-squared: {r_squared:.3f}") + print(f"Accuracy: {r_squared * 100:.2f}%") + +# ------------------- Regression Implementations -------------------- + +def elastic_net_regression(X, y, alpha=1.0, l1_ratio=0.5): + """Elastic Net regression from scratch""" + num_samples, num_features = X.shape + identity_matrix = np.eye(num_features) + l1_term = l1_ratio * alpha + l2_term = (1 - l1_ratio) * alpha + coefficients = inv(X.T @ X + l2_term * identity_matrix) @ (X.T @ y - l1_term * np.sign(X.T @ y)) + return coefficients + +def ridge_regression(X, y, alpha=1.0): + """Implementing Ridge Regression from scratch""" + num_samples, num_features = X.shape + identity_matrix = np.eye(num_features) + coefficients = inv(X.T @ X + alpha * identity_matrix) @ X.T @ y + return coefficients + +def lasso_regression(X, y, alpha=1.0): + """Basic Lasso Regression implementation""" + num_samples, num_features = X.shape + coefficients = inv(X.T @ X) @ (X.T @ y - alpha * np.sign(X.T @ y)) + return coefficients + +def polynomial_regression(X, y, degree=2): + """Fit polynomial regression without external libraries""" + poly_features = np.hstack([X ** i for i in range(1, degree + 1)]) + coefficients = inv(poly_features.T @ poly_features) @ poly_features.T @ y + return coefficients, poly_features + +# ------------------- Data Preparation -------------------- + +np.random.seed(42) +X = np.random.randn(100, 3) +y = np.dot(X, np.array([3, 1.5, -2])) + np.random.randn(100) + +# Normalize features +X, X_avg, X_std = normalize_features(X) + +# Train/Test split +X_train, X_test = X[:80], X[80:] +y_train, y_test = y[:80], y[80:] + +# ------------------- Model Training -------------------- + +# Elastic Net Model Training +elastic_net_coeff = elastic_net_regression(X_train, y_train, alpha=0.01, l1_ratio=0.9) +y_pred_enet = X_test @ elastic_net_coeff +mse_enet, r2_enet, res_enet = compute_performance(y_test, y_pred_enet) +display_model_performance("ElasticNet", mse_enet, r2_enet) + +# Ridge Regression Model Training +ridge_coeff = ridge_regression(X_train, y_train, alpha=1.0) +y_pred_ridge = X_test @ ridge_coeff +mse_ridge, r2_ridge, res_ridge = compute_performance(y_test, y_pred_ridge) +display_model_performance("Ridge", mse_ridge, r2_ridge) + +# Lasso Regression Model Training +lasso_coeff = lasso_regression(X_train, y_train, alpha=1.0) +y_pred_lasso = X_test @ lasso_coeff +mse_lasso, r2_lasso, res_lasso = compute_performance(y_test, y_pred_lasso) +display_model_performance("Lasso", mse_lasso, r2_lasso) + +# Polynomial Regression Model Training +poly_coeff, X_train_poly = polynomial_regression(X_train, y_train, degree=2) +X_test_poly = np.hstack([X_test ** i for i in range(1, 3)]) +y_pred_poly = X_test_poly @ poly_coeff +mse_poly, r2_poly, res_poly = compute_performance(y_test, y_pred_poly) +display_model_performance("Polynomial Regression", mse_poly, r2_poly) + +# ------------------- Visualization -------------------- + +# Plotting predictions +show_comparison_plot(y_test, y_pred_enet, title="ElasticNet: Actual vs Predicted") +show_comparison_plot(y_test, y_pred_ridge, title="Ridge: Actual vs Predicted") +show_comparison_plot(y_test, y_pred_lasso, title="Lasso: Actual vs Predicted") +show_comparison_plot(y_test, y_pred_poly, title="Polynomial Regression: Actual vs Predicted") + +# Plotting residual distributions +show_residual_distribution(res_enet, title="ElasticNet Residual Distribution") +show_residual_distribution(res_ridge, title="Ridge Residual Distribution") +show_residual_distribution(res_lasso, title="Lasso Residual Distribution") +show_residual_distribution(res_poly, title="Polynomial Regression Residual Distribution") - def predict(self, x): - return 0.5 From d2e369177322c8159afa7b52ea7a92de52b0be80 Mon Sep 17 00:00:00 2001 From: sravanyabezawada <67137275+sravanyabezawada@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:36:02 -0500 Subject: [PATCH 2/5] Update and rename test_ElasticNetModel.py to regression_models.py --- elasticnet/tests/regression_models.py | 49 ++++++++++++++++++++++++ elasticnet/tests/test_ElasticNetModel.py | 19 --------- 2 files changed, 49 insertions(+), 19 deletions(-) create mode 100644 elasticnet/tests/regression_models.py delete mode 100644 elasticnet/tests/test_ElasticNetModel.py diff --git a/elasticnet/tests/regression_models.py b/elasticnet/tests/regression_models.py new file mode 100644 index 0000000..29ab88f --- /dev/null +++ b/elasticnet/tests/regression_models.py @@ -0,0 +1,49 @@ +import unittest +import numpy as np + +from regression_models import normalize_features, elastic_net_regression, ridge_regression, lasso_regression, polynomial_regression + +class TestRegressionModels(unittest.TestCase): + + def setUp(self): + """Set up sample data for testing""" + np.random.seed(42) + self.X = np.random.randn(100, 3) + self.y = np.dot(self.X, np.array([3, 1.5, -2])) + np.random.randn(100) + self.X_normalized, _, _ = normalize_features(self.X) + self.X_train = self.X_normalized[:80] + self.y_train = self.y[:80] + self.X_test = self.X_normalized[80:] + self.y_test = self.y[80:] + + def test_elastic_net_regression(self): + """Test ElasticNet regression with fixed alpha and l1_ratio""" + coeff = elastic_net_regression(self.X_train, self.y_train, alpha=0.01, l1_ratio=0.9) + y_pred = self.X_test @ coeff + mse = np.mean((self.y_test - y_pred) ** 2) + self.assertLess(mse, 1.0, "ElasticNet regression MSE is too high") + + def test_ridge_regression(self): + """Test Ridge regression with fixed alpha""" + coeff = ridge_regression(self.X_train, self.y_train, alpha=1.0) + y_pred = self.X_test @ coeff + mse = np.mean((self.y_test - y_pred) ** 2) + self.assertLess(mse, 1.0, "Ridge regression MSE is too high") + + def test_lasso_regression(self): + """Test Lasso regression with fixed alpha""" + coeff = lasso_regression(self.X_train, self.y_train, alpha=1.0) + y_pred = self.X_test @ coeff + mse = np.mean((self.y_test - y_pred) ** 2) + self.assertLess(mse, 1.0, "Lasso regression MSE is too high") + + def test_polynomial_regression(self): + """Test Polynomial regression for a fixed degree""" + coeff, X_train_poly = polynomial_regression(self.X_train, self.y_train, degree=2) + X_test_poly = np.hstack([self.X_test ** i for i in range(1, 3)]) + y_pred = X_test_poly @ coeff + mse = np.mean((self.y_test - y_pred) ** 2) + self.assertLess(mse, 1.0, "Polynomial regression MSE is too high") + +if __name__ == "__main__": + unittest.main() diff --git a/elasticnet/tests/test_ElasticNetModel.py b/elasticnet/tests/test_ElasticNetModel.py deleted file mode 100644 index 5022c3c..0000000 --- a/elasticnet/tests/test_ElasticNetModel.py +++ /dev/null @@ -1,19 +0,0 @@ -import csv - -import numpy - -from elasticnet.models.ElasticNet import ElasticNetModel - -def test_predict(): - model = ElasticNetModel() - data = [] - with open("small_test.csv", "r") as file: - reader = csv.DictReader(file) - for row in reader: - data.append(row) - - X = numpy.array([[v for k,v in datum.items() if k.startswith('x')] for datum in data]) - y = numpy.array([[v for k,v in datum.items() if k=='y'] for datum in data]) - results = model.fit(X,y) - preds = results.predict(X) - assert preds == 0.5 From 0d4addff8fd5d79dc661264f063f78c81839f18a Mon Sep 17 00:00:00 2001 From: sravanyabezawada <67137275+sravanyabezawada@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:42:57 -0500 Subject: [PATCH 3/5] Update README.md --- README.md | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c1e8359..64c4d15 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,25 @@ -# Project 1 +# Project 1 #Team Members + #Bezawada Sai Sravanya - A20561552 + #Karuturi Veerendra Gopichand - A20529571 + #Praveen Kumar Gude - A20546614 + #Abharnah Rajaram Mohan - A20546967 -Put your README here. Answer the following questions. -* What does the model you have implemented do and when should it be used? -* How did you test your model to determine if it is working reasonably correctly? -* What parameters have you exposed to users of your implementation in order to tune performance? (Also perhaps provide some basic usage examples.) -* Are there specific inputs that your implementation has trouble with? Given more time, could you work around these or is it fundamental? +## What does the model you have implemented do and when should it be used? + +The implemented model is Elastic Net Regression. Elastic Net combines both L1, known as Lasso, and L2, known as Ridge regularization, making it useful for cases with many correlated features or when the number of features exceeds the number of observations. This helps to prevent overfitting while maintaining interpretability of the coefficients. It also evaluates their performance using the metrics Mean squared error(MSE) and R-squared(R2) +This model is used to understand the regularization when exploring L1 and L2 penalities affect model performance. + +## How did you test your model to determine if it is working reasonably correctly? + +We have run my model on the synthetic datasets in which relationships are known. Scatter plots comparing predicted versus true values were created for each model.The performance of each model was compared using their respective R2 scores and MSE. This helps to determine which model performs best under given conditions. + +## What parameters have you exposed to users of your implementation in order to tune performance? + +This provides the user with sensitivity to key parameters in the implementation and allows users to tune for optimal model performance. For this,tuning of alpha controls the general strength of the regularization applied to the model and tuning of l1_ratio controls the mix of L1 and L2 penalties applied. + +## Are there specific inputs that your implementation has trouble with? Given more time, could you work around these or is it fundamental to the model? + +The implementations may suffer when the datasets are highly imbalanced, including multi collinearity which can destabilize coefficient estimates. +Implement techniques such as Principal Component Analysis (PCA) to reduce dimensionality or use feature selection methods to eliminate redundant features. However, this doesn't fundamentally change the nature of the models. + From 813ce2fb69890fa0c67f88559e613b745dbba220 Mon Sep 17 00:00:00 2001 From: sravanyabezawada <67137275+sravanyabezawada@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:43:33 -0500 Subject: [PATCH 4/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64c4d15..5797918 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Project 1 #Team Members +# Project 1 Team Members #Bezawada Sai Sravanya - A20561552 #Karuturi Veerendra Gopichand - A20529571 #Praveen Kumar Gude - A20546614 From 71163f770bf37e27f7d2cacc7849ff975baef5c2 Mon Sep 17 00:00:00 2001 From: sravanyabezawada <67137275+sravanyabezawada@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:44:02 -0500 Subject: [PATCH 5/5] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5797918..7f7ce22 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Project 1 Team Members - #Bezawada Sai Sravanya - A20561552 - #Karuturi Veerendra Gopichand - A20529571 - #Praveen Kumar Gude - A20546614 - #Abharnah Rajaram Mohan - A20546967 + Bezawada Sai Sravanya - A20561552 + Karuturi Veerendra Gopichand - A20529571 + Praveen Kumar Gude - A20546614 + Abharnah Rajaram Mohan - A20546967 ## What does the model you have implemented do and when should it be used?