Project 1

README.md

@@ -1,8 +1,25 @@
-# Project 1 
+# Project 1 Team Members
+    Bezawada Sai Sravanya - A20561552
+    Karuturi Veerendra Gopichand - A20529571
+    Praveen Kumar Gude - A20546614
+    Abharnah Rajaram Mohan - A20546967
 
-Put your README here. Answer the following questions.
 
-* What does the model you have implemented do and when should it be used?
-* How did you test your model to determine if it is working reasonably correctly?
-* What parameters have you exposed to users of your implementation in order to tune performance? (Also perhaps provide some basic usage examples.)
-* Are there specific inputs that your implementation has trouble with? Given more time, could you work around these or is it fundamental?
+## What does the model you have implemented do and when should it be used?
+
+The implemented model is Elastic Net Regression. Elastic Net combines both L1, known as Lasso, and L2, known as Ridge regularization, making it useful for cases with many correlated features or when the number of features exceeds the number of observations. This helps to prevent overfitting while maintaining interpretability of the coefficients. It also evaluates their performance using the metrics Mean squared error(MSE) and R-squared(R2)
+This model is used to understand the regularization when exploring L1 and L2 penalities affect model performance.
+
+## How did you test your model to determine if it is working reasonably correctly?
+
+We have run my model on the synthetic datasets in which relationships are known. Scatter plots comparing predicted versus true values were created for each model.The performance of each model was compared using their respective R2 scores and MSE. This helps to determine which model performs best under given conditions.
+
+## What parameters have you exposed to users of your implementation in order to tune performance?
+
+This provides the user with sensitivity to key parameters in the implementation and allows users to tune for optimal model performance. For this,tuning of alpha controls the general strength of the regularization applied to the model and tuning of l1_ratio controls the mix of L1 and L2 penalties applied.
+
+## Are there specific inputs that your implementation has trouble with? Given more time, could you work around these or is it fundamental to the model?
+
+The implementations may suffer when the datasets are highly imbalanced, including multi collinearity which can destabilize coefficient estimates.
+Implement techniques such as Principal Component Analysis (PCA) to reduce dimensionality or use feature selection methods to eliminate redundant features. However, this doesn't fundamentally change the nature of the models.
+

elasticnet/models/ElasticNet.py

@@ -1,17 +1,128 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from numpy.linalg import inv
 
+# ------------------- Utility Functions --------------------
 
-class ElasticNetModel():
-    def __init__(self):
-        pass
+def normalize_features(X):
+    """Normalize the feature matrix to have zero mean and unit variance"""
+    avg = np.mean(X, axis=0)
+    deviation = np.std(X, axis=0)
+    return (X - avg) / deviation, avg, deviation
 
+def compute_performance(y_actual, y_predicted):
+    """Compute Mean Squared Error and R-squared"""
+    mse = np.mean((y_actual - y_predicted) ** 2)
+    r_squared = 1 - (np.sum((y_actual - y_predicted) ** 2) / np.sum((y_actual - np.mean(y_actual)) ** 2))
+    residuals = y_actual - y_predicted
+    return mse, r_squared, residuals
 
-    def fit(self, X, y):
-        return ElasticNetModelResults()
+def show_comparison_plot(y_actual, y_predicted, title="Actual vs Predicted"):
+    """Scatter plot for actual vs predicted values"""
+    plt.scatter(y_actual, y_predicted, color='green', label='Predicted vs Actual')
+    plt.plot([min(y_actual), max(y_actual)], [min(y_actual), max(y_actual)], color='red', label='Perfect Fit')
+    plt.xlabel('Actual Values')
+    plt.ylabel('Predicted Values')
+    plt.title(title)
+    plt.legend()
+    plt.show()
 
+def show_residual_distribution(residuals, title="Residual Distribution"):
+    """Histogram for residual distribution"""
+    plt.hist(residuals, bins=15, color='orange', edgecolor='black')
+    plt.title(title)
+    plt.xlabel('Residuals')
+    plt.ylabel('Frequency')
+    plt.show()
 
-class ElasticNetModelResults():
-    def __init__(self):
-        pass
+def display_model_performance(model_name, mse, r_squared):
+    """Print out the performance metrics of the model"""
+    print(f"\nPerformance for {model_name}:")
+    print(f"MSE: {mse:.3f}")
+    print(f"R-squared: {r_squared:.3f}")
+    print(f"Accuracy: {r_squared * 100:.2f}%")
+
+# ------------------- Regression Implementations --------------------
+
+def elastic_net_regression(X, y, alpha=1.0, l1_ratio=0.5):
+    """Elastic Net regression from scratch"""
+    num_samples, num_features = X.shape
+    identity_matrix = np.eye(num_features)
+    l1_term = l1_ratio * alpha
+    l2_term = (1 - l1_ratio) * alpha
+    coefficients = inv(X.T @ X + l2_term * identity_matrix) @ (X.T @ y - l1_term * np.sign(X.T @ y))
+    return coefficients
+
+def ridge_regression(X, y, alpha=1.0):
+    """Implementing Ridge Regression from scratch"""
+    num_samples, num_features = X.shape
+    identity_matrix = np.eye(num_features)
+    coefficients = inv(X.T @ X + alpha * identity_matrix) @ X.T @ y
+    return coefficients
+
+def lasso_regression(X, y, alpha=1.0):
+    """Basic Lasso Regression implementation"""
+    num_samples, num_features = X.shape
+    coefficients = inv(X.T @ X) @ (X.T @ y - alpha * np.sign(X.T @ y))
+    return coefficients
+
+def polynomial_regression(X, y, degree=2):
+    """Fit polynomial regression without external libraries"""
+    poly_features = np.hstack([X ** i for i in range(1, degree + 1)])
+    coefficients = inv(poly_features.T @ poly_features) @ poly_features.T @ y
+    return coefficients, poly_features
+
+# ------------------- Data Preparation --------------------
+
+np.random.seed(42)
+X = np.random.randn(100, 3)
+y = np.dot(X, np.array([3, 1.5, -2])) + np.random.randn(100)
+
+# Normalize features
+X, X_avg, X_std = normalize_features(X)
+
+# Train/Test split
+X_train, X_test = X[:80], X[80:]
+y_train, y_test = y[:80], y[80:]
+
+# ------------------- Model Training --------------------
+
+# Elastic Net Model Training
+elastic_net_coeff = elastic_net_regression(X_train, y_train, alpha=0.01, l1_ratio=0.9)
+y_pred_enet = X_test @ elastic_net_coeff
+mse_enet, r2_enet, res_enet = compute_performance(y_test, y_pred_enet)
+display_model_performance("ElasticNet", mse_enet, r2_enet)
+
+# Ridge Regression Model Training
+ridge_coeff = ridge_regression(X_train, y_train, alpha=1.0)
+y_pred_ridge = X_test @ ridge_coeff
+mse_ridge, r2_ridge, res_ridge = compute_performance(y_test, y_pred_ridge)
+display_model_performance("Ridge", mse_ridge, r2_ridge)
+
+# Lasso Regression Model Training
+lasso_coeff = lasso_regression(X_train, y_train, alpha=1.0)
+y_pred_lasso = X_test @ lasso_coeff
+mse_lasso, r2_lasso, res_lasso = compute_performance(y_test, y_pred_lasso)
+display_model_performance("Lasso", mse_lasso, r2_lasso)
+
+# Polynomial Regression Model Training
+poly_coeff, X_train_poly = polynomial_regression(X_train, y_train, degree=2)
+X_test_poly = np.hstack([X_test ** i for i in range(1, 3)])
+y_pred_poly = X_test_poly @ poly_coeff
+mse_poly, r2_poly, res_poly = compute_performance(y_test, y_pred_poly)
+display_model_performance("Polynomial Regression", mse_poly, r2_poly)
+
+# ------------------- Visualization --------------------
+
+# Plotting predictions
+show_comparison_plot(y_test, y_pred_enet, title="ElasticNet: Actual vs Predicted")
+show_comparison_plot(y_test, y_pred_ridge, title="Ridge: Actual vs Predicted")
+show_comparison_plot(y_test, y_pred_lasso, title="Lasso: Actual vs Predicted")
+show_comparison_plot(y_test, y_pred_poly, title="Polynomial Regression: Actual vs Predicted")
+
+# Plotting residual distributions
+show_residual_distribution(res_enet, title="ElasticNet Residual Distribution")
+show_residual_distribution(res_ridge, title="Ridge Residual Distribution")
+show_residual_distribution(res_lasso, title="Lasso Residual Distribution")
+show_residual_distribution(res_poly, title="Polynomial Regression Residual Distribution")
 
-    def predict(self, x):
-        return 0.5

elasticnet/tests/regression_models.py

@@ -0,0 +1,49 @@
+import unittest
+import numpy as np
+
+from regression_models import normalize_features, elastic_net_regression, ridge_regression, lasso_regression, polynomial_regression
+
+class TestRegressionModels(unittest.TestCase):
+
+    def setUp(self):
+        """Set up sample data for testing"""
+        np.random.seed(42)
+        self.X = np.random.randn(100, 3)
+        self.y = np.dot(self.X, np.array([3, 1.5, -2])) + np.random.randn(100)
+        self.X_normalized, _, _ = normalize_features(self.X)
+        self.X_train = self.X_normalized[:80]
+        self.y_train = self.y[:80]
+        self.X_test = self.X_normalized[80:]
+        self.y_test = self.y[80:]
+
+    def test_elastic_net_regression(self):
+        """Test ElasticNet regression with fixed alpha and l1_ratio"""
+        coeff = elastic_net_regression(self.X_train, self.y_train, alpha=0.01, l1_ratio=0.9)
+        y_pred = self.X_test @ coeff
+        mse = np.mean((self.y_test - y_pred) ** 2)
+        self.assertLess(mse, 1.0, "ElasticNet regression MSE is too high")
+
+    def test_ridge_regression(self):
+        """Test Ridge regression with fixed alpha"""
+        coeff = ridge_regression(self.X_train, self.y_train, alpha=1.0)
+        y_pred = self.X_test @ coeff
+        mse = np.mean((self.y_test - y_pred) ** 2)
+        self.assertLess(mse, 1.0, "Ridge regression MSE is too high")
+
+    def test_lasso_regression(self):
+        """Test Lasso regression with fixed alpha"""
+        coeff = lasso_regression(self.X_train, self.y_train, alpha=1.0)
+        y_pred = self.X_test @ coeff
+        mse = np.mean((self.y_test - y_pred) ** 2)
+        self.assertLess(mse, 1.0, "Lasso regression MSE is too high")
+
+    def test_polynomial_regression(self):
+        """Test Polynomial regression for a fixed degree"""
+        coeff, X_train_poly = polynomial_regression(self.X_train, self.y_train, degree=2)
+        X_test_poly = np.hstack([self.X_test ** i for i in range(1, 3)])
+        y_pred = X_test_poly @ coeff
+        mse = np.mean((self.y_test - y_pred) ** 2)
+        self.assertLess(mse, 1.0, "Polynomial regression MSE is too high")
+
+if __name__ == "__main__":
+    unittest.main()