LogisticRegression.py

import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
import pandas as pd
from PIL import Image
from scipy import ndimage
# from lr_utils import load_dataset
# %matplotlib inline
# from IPython import get_ipython
# get_ipython().run_line_magic('matplotlib', 'inline')

def load_dataset():
    train_dataset = h5py.File('datasets/Iris_train.h5', "r")
    train_set_x_orig = np.array(train_dataset["train_set_x"][0:20]) # your train set features : OBTAINS ALL THE INPUT FEATURES VALUE FOR THE M TRAINING EXAMPLES FROM TRAIN_SET_X FILE INTO A NUMPY ARRAY
    train_set_y_orig = np.array(train_dataset["train_set_y"][0:20]) # your train set labels : OBTAINS ALL THE Y VALUES FOR THE SAME
    
    test_dataset = h5py.File('datasets/iris_test.h5', "r")
    test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features
    test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels
    
    classes = np.array(test_dataset["list_classes"][:]) # the list of classes
    
    train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) # Reshaping the y array from : M X 1 to 1 X M for our algorithm : Logistic Regression
    test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0]))
    
    return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes

def load_csv() :

    x = np.genfromtxt('/users/deeps/datasets/iris_train.csv',delimiter=",")
    rqrd_train_set = np.array([row for row in x  if 0.0 == row[4] or 2.0 == row[4] ])
#    rqrd_train_set = x[20:90,:]
    np.random.shuffle(rqrd_train_set)
    train_set_x_orig = rqrd_train_set[0:60,0:4]
    train_set_y_orig = rqrd_train_set[0:60,4:6]

    x = np.genfromtxt('/users/deeps/datasets/iris_test.csv',delimiter=",")
    rqrd_test_set = np.array([row for row in x  if 0.0 == row[4] or 2.0 == row[4] ])
#    rqrd_test_set = x[20:60,:]
    np.random.shuffle(rqrd_test_set)
    test_set_x_orig = rqrd_test_set[0:40,0:4]
    test_set_y_orig = rqrd_test_set[0:40,4:6]
    
    cls=np.array(['setosa','versicolor','virginica'])
    classes=cls[0:2]
    
    train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) # Reshaping the y array from : M X 1 to 1 X M for our algorithm : Logistic Regression
    test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0]))

    return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes

# Loading the data (cat/non-cat) || (setosa/versicolor)
train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, classes = load_csv() # this function returns 5 numpy array of training and testing and classes


#printing the shapes of each of the numpy arrays :

print("train_set_x_orig : shape : " + str(train_set_x_orig.shape));
print("train_set_y(outputs) : shape : " + str(train_set_y.shape));
print("test_set_x_orig : shape : " + str(test_set_x_orig.shape));
print("test_set_y : shape : " + str(test_set_y.shape));
print("classes : shape : " + str(classes.shape));


#### FOR THE CAT NOT CAT DATASET PRINTING
#for i in range(200):
#    print(train_set_x_orig[i])
#
#for i in range(200) :
#    print(train_set_y[0][i])
#
#for i in range(10):
#    print(test_set_x_orig[i])
#
#for i in range(10) :
#    print(test_set_y[0][i])



#### FOR THE IRIS DATASET PRINTING
print("################################## PRINTING TRAIN SET ###################################")
m = train_set_x_orig.shape[0]
for i in range(m) :
    print("FOR : " + str(i) + " : " + str(train_set_x_orig[i]) + "\n" + "OUTPUT :" + str(train_set_y[0][i])+ "\n \n")

print("################################# PRINTING TEST SET  #####################################")
n = test_set_x_orig.shape[0]
for i in range(n) :
    print("FOR : " + str(i) + " : " + str(test_set_x_orig[i]) + "\n" + "OUTPUT :" + str(test_set_y[0][i])+ "\n \n")


## Extra code
#print("train_set_x_orig: " + str(train_set_x_orig.shape[0]))
#print("train_set_x_orig: " + str(train_set_x_orig.shape[1]))
#print("train_set_x_orig: " + str(train_set_x_orig.shape[2]))
#print("train_set_x_orig: " + str(train_set_x_orig.shape[3]))

### Example of a picture
#index = 30
#plt.imshow(train_set_x_orig[index])
#print ("y = " + str(train_set_y[:, index]) + ", it's a '" + classes[np.squeeze(train_set_y[:, index])].decode("utf-8") +  "' picture.")
#


#print("EXAMPLE OF A SETOSA FLOWER : ")
#index = 3
#print ("FOR INDEX = " + str(index) + ", y = " + str(train_set_y[:, index]) + ", it's a '" + classes[np.squeeze(train_set_y[:, index])].decode("utf-8") +  "' flower ")
#
#print("EXAMPLE OF A VERSICOLOR FLOWER : ")
#index = 2
#print("FOR INDEX = " + str(index) + ", y = " + str(train_set_y[:,index]) + ", it's a '" + classes[np.squeeze(train_set_y[: ,index])].decode("utf-8") + "' flower ")


#DEFINING IMPORTANT VARIABLES : M_TRAIN : NUMBER OF TRAINING EXAMPLES
#                               M_TEST : NUMBER OF TEST EXAMPLES
#                               NUM_PX : NUMBER OF INPUTS IN EACH EXAMPLE
m_train = train_set_x_orig.shape[0]
m_test = test_set_x_orig.shape[0]
num_px = train_set_x_orig.shape[1]


print ("\n Number of training examples: m_train = " + str(m_train))
print (" Number of testing examples: m_test = " + str(m_test))
#print ("Height/Width of each image: num_px = " + str(num_px)) //cat-non cat dataset
print (" Variables ( Sepal length , Petal Length )= " + str(num_px) )
#print ("Each image is of size: (" + str(num_px) + ", " + str(num_px) + ", 3)")
#print ("train_set_x shape: " + str(train_set_x_orig.shape))
#print ("train_set_y shape: " + str(train_set_y.shape))
#print ("test_set_x shape: " + str(test_set_x_orig.shape))
#print ("test_set_y shape: " + str(test_set_y.shape))
print (" Classes: " + str(classes))
#print ("(train_set_y[0,26]): " + str(train_set_y[0][26]))

## Reshape the training and test examples (cat-non cat dataset)
#### START CODE HERE ### (a 2 lines of code)
train_set_x_flatten = train_set_x_orig.reshape(m_train, -1 ).T
test_set_x_flatten = test_set_x_orig.reshape(m_test, -1 ).T
#### END CODE HERE ###


#print ("train_set_x_flatten shape: " + str(train_set_x_flatten.shape))
#print ("train_set_y shape: " + str(train_set_y.shape))
#print ("test_set_x_flatten shape: " + str(test_set_x_flatten.shape))
#print ("test_set_y shape: " + str(test_set_y.shape))
#print ("sanity check after reshaping: " + str(train_set_x_flatten[0:5,0]))

#### STANDARDIZING DATA FOR CAT NON CAT
train_set_x = train_set_x_flatten
test_set_x = test_set_x_flatten
#train_set_x = train_set_x_flatten/255.
#test_set_x = test_set_x_flatten/255.
##print(train_set_x.shape)


# GRADED FUNCTION: sigmoid

def sigmoid(z):
    """
    Compute the sigmoid of z

    Arguments:
    z -- A scalar or numpy array of any size.

    Return:
    s -- sigmoid(z)
    """

    ### START CODE HERE ### (1 line of code)
    s = 1.0/(1.0+np.exp(-z))
    ### END CODE HERE ###

    return s

#Testing Sigmoid :
#print ("sigmoid([0, 2]) = " + str(sigmoid(np.array([0,2]))))

# GRADED FUNCTION: initialize_with_zeros

def initialize_with_zeros(dim):
    """
    This function creates a vector of zeros of shape (dim, 1) for w and initializes b to 0.

    Argument:
    dim -- size of the w vector we want (or number of parameters in this case)

    Returns:
    w -- initialized vector of shape (dim, 1)
    b -- initialized scalar (corresponds to the bias)
    """

    ### START CODE HERE ### (a 1 line of code)
    w = np.zeros((dim, 1))
    b = 0.0
    ### END CODE HERE ###

    assert(w.shape == (dim, 1))
#    print("w.shape: ", w.shape)
    assert(isinstance(b, float) or isinstance(b, int))

    return w, b

##Testing Initializing Function
#dim = 2
#w, b = initialize_with_zeros(dim)
#print ("w = " + str(w))
#print ("b = " + str(b))

## Extra Cell
#a = np.array([[1, 2, 3], [4, 5, 6]])
#print("axis = 1", np.sum(a, axis = 1))
#print("axis = 0", np.sum(a, axis = 0))
#print("a.shape: ", a.shape)
#b = np.array([[2,3]]).reshape(2,1)
#print("b.shape: ", a.shape)
#print("sigmoid a "+ str(sigmoid(a)))
#c = np.array([1,2,3])
#print("c:shape ", c.shape, c.reshape(1,3).shape, c.reshape(3,1).shape)


# GRADED FUNCTION: propagate

def propagate(w, b, X, Y):
    """
    Implement the cost function and its gradient for the propagation explained above

    Arguments:
    w -- weights, a numpy array of size (num_px * num_px * 3, 1)
    b -- bias, a scalar
    X -- data of size (num_px * num_px * 3, number of examples)
    Y -- true "label" vector (containing 0 if non-cat, 1 if cat) of size (1, number of examples)

    Return:
    cost -- negative log-likelihood cost for logistic regression
    dw -- gradient of the loss with respect to w, thus same shape as w
    db -- gradient of the loss with respect to b, thus same shape as b

    Tips:
    - Write your code step by step for the propagation. np.log(), np.dot()
    """

    m = X.shape[1]

    # FORWARD PROPAGATION (FROM X TO COST)
    
    ### START CODE HERE ### (a 2 lines of code)
    A = sigmoid(np.dot(w.T, X) + b)
    cost = (-1.0/m)*np.sum((Y*np.log(A)+ (1-Y)*np.log(1-A)), axis = 1)

#    print("A.shape, Y.shape", A.shape, Y.shape)
#    print("shape of cost: " + str(cost.shape))
#    print("cost = ", cost)
    ### END CODE HERE ###
    
    # BACKWARD PROPAGATION (TO FIND GRAD)
    ### START CODE HERE ### (a 2 lines of code)
    dw = (1.0/m)*np.dot(X, (A-Y).T)
    db = (1.0/m)*np.sum(A-Y, axis = 1)
    ### END CODE HERE ###
#    print("shape of X: " + str(X.shape)) #extra code
#    print("shape of Y: " + str(Y.shape)) #extra code
#    print("shape of w: " + str(w.shape)) #extra code
#    print("shape of dz = (A-Y): " + str((A-Y).shape)) #extra code
#    print("shape of dz[0] = (A-Y)[0]: " + str(((A-Y)[0]).shape)) #extra code
#    print("Note that dz[0] selects the entire first row, hence a row vector of size m")
#    print("shape of dw: " + str(dw.shape)) #extra code
#    print("shape of db: " + str(db.shape))

    assert(dw.shape == w.shape)
    assert(db.dtype == float)
    cost = np.squeeze(cost)
    assert(cost.shape == ())

    grads = {"dw": dw,
             "db": db}

    return grads, cost

#X = np.array([[1.,2.,-1.],[3.,4.,-3.2]])
#dz = np.array([1,2,3]).reshape(3,1)
#print(str(X)+ "\n" + str(dz)+ "\n" + str(np.dot(X,dz)) )  #extra code
#
#w, b, X, Y = np.array([[1.],[2.]]), 2., np.array([[1.,2.,-1.],[3.,4.,-3.2]]), np.array([[1,0,1]])
#grads, cost = propagate(w, b, X, Y)
#print ("dw = " + str(grads["dw"]))
#print ("db = " + str(grads["db"]))
#print ("cost = " + str(cost))


## GRADED FUNCTION: optimize

def optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False):
    """
    This function optimizes w and b by running a gradient descent algorithm

    Arguments:
    w -- weights, a numpy array of size (num_px * num_px * 3, 1)
    b -- bias, a scalar
    X -- data of shape (num_px * num_px * 3, number of examples)
    Y -- true "label" vector (containing 0 if non-cat, 1 if cat), of shape (1, number of examples)
    num_iterations -- number of iterations of the optimization loop
    learning_rate -- learning rate of the gradient descent update rule
    print_cost -- True to print the loss every 100 steps

    Returns:
    params -- dictionary containing the weights w and bias b
    grads -- dictionary containing the gradients of the weights and bias with respect to the cost function
    costs -- list of all the costs computed during the optimization, this will be used to plot the learning curve.

    Tips:
    You basically need to write down two steps and iterate through them:
        1) Calculate the cost and the gradient for the current parameters. Use propagate().
        2) Update the parameters using gradient descent rule for w and b.
    """

    costs = []

    for i in range(num_iterations):


        # Cost and gradient calculation (a 1-4 lines of code)
        ### START CODE HERE ###
        grads, cost = propagate(w, b, X, Y)
        ### END CODE HERE ###

        # Retrieve derivatives from grads
        dw = grads["dw"]
        db = grads["db"]

        # update rule (a 2 lines of code)
        ### START CODE HERE ###
        w = w - learning_rate*dw
        b = b - learning_rate*db
        ### END CODE HERE ###

        # Record the costs
        if i % 100 == 0:
            costs.append(cost)

        # Print the cost every 100 training examples
        if print_cost and i % 100 == 0:
            print ("Cost after iteration %i: %f" %(i, cost))

    params = {"w": w,
              "b": b}

    grads = {"dw": dw,
             "db": db}

    return params, grads, costs

#params, grads, costs = optimize(w, b, X, Y, num_iterations= 100, learning_rate = 0.009, print_cost = False)
#
#print ("w = " + str(params["w"]))
#print ("b = " + str(params["b"]))
#print ("dw = " + str(grads["dw"]))
#print ("db = " + str(grads["db"]))

# GRADED FUNCTION: predict

def predict(w, b, X, Y):
    '''
    Predict whether the label is 0 or 1 using learned logistic regression parameters (w, b)

    Arguments:
    w -- weights, a numpy array of size (num_px * num_px * 3, 1)
    b -- bias, a scalar
    X -- data of size (num_px * num_px * 3, number of examples)

    Returns:
    Y_prediction -- a numpy array (vector) containing all predictions (0/1) for the examples in X
    '''

    m = X.shape[1]
    Y_prediction = np.zeros((1,m))
    w = w.reshape(X.shape[0], 1)

    # Compute vector "A" predicting the probabilities of a cat being present in the picture
    ### START CODE HERE ### (a 1 line of code)
    A = sigmoid(np.dot(w.T, X) + b)
#    print("A = ", A)
    ### END CODE HERE ###
    p = np.zeros(m).reshape(1,m)
    for i in range(A.shape[1]):

        # Convert probabilities A[0,i] to actual predictions p[0,i]
        ### START CODE HERE ### (a 4 lines of code)
        if A[0,i]>0.5:
            Y_prediction[0,i] = 1

        ### END CODE HERE ###

    print("\n DIFFERENCES IN THE PREDICTION AND ACTUAL OUTPUT (IF 0 -> THEN OUR ANN PREDICTS RIGHT ELSE IF -1/1 -> THEN OUR ANN PEDICTS WRONG ) : ")
    print("\n" + str(Y-Y_prediction))
    assert(Y_prediction.shape == (1, m))

    return Y_prediction

#w = np.array([[0.1124579],[0.23106775]])
#b = -0.3
#X = np.array([[1.,-1.1,-3.2],[1.2,2.,0.1]])
#Y = np.array([1,0,1])
#print ("predictions = " + str(predict(w, b, X, Y)))


# GRADED FUNCTION: model

def model(X_train, Y_train, X_test, Y_test, num_iterations = 2000, learning_rate = 0.5, print_cost = False):
    """
    Builds the logistic regression model by calling the function you've implemented previously

    Arguments:
    X_train -- training set represented by a numpy array of shape (num_px * num_px * 3, m_train)
    Y_train -- training labels represented by a numpy array (vector) of shape (1, m_train)
    X_test -- test set represented by a numpy array of shape (num_px * num_px * 3, m_test)
    Y_test -- test labels represented by a numpy array (vector) of shape (1, m_test)
    num_iterations -- hyperparameter representing the number of iterations to optimize the parameters
    learning_rate -- hyperparameter representing the learning rate used in the update rule of optimize()
    print_cost -- Set to true to print the cost every 100 iterations

    Returns:
    d -- dictionary containing information about the model.
    """

    ### START CODE HERE ###

    # initialize parameters with zeros (a 1 line of code)
    w, b = np.zeros(X_train.shape[0]).reshape(X_train.shape[0],1), 0.0

    # Gradient descent (a 1 line of code)
    #optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False)
    parameters, grads, costs = optimize(w, b, X_train, Y_train, num_iterations, learning_rate, print_cost)

    # Retrieve parameters w and b from dictionary "parameters"
    w = parameters["w"]
    b = parameters["b"]

    # Predict test/train set examples (a 2 lines of code)
    Y_prediction_test = predict(w, b, X_test, Y_test)
    Y_prediction_train = predict(w, b, X_train, Y_train)

    ### END CODE HERE ###

    # Print train/test Errors
    print("train accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))


    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test,
         "Y_prediction_train" : Y_prediction_train,
         "w" : w,
         "b" : b,
         "learning_rate" : learning_rate,
         "num_iterations": num_iterations}

    return d

d = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 2000, learning_rate = 0.005, print_cost = False)

## Example of a picture that was wrongly classified.
#index = 1
#plt.imshow(test_set_x[:,index].reshape((num_px, num_px, 3)))
#print ("y = " + str(test_set_y[0,index]) + ", you predicted that it is a \"" + classes[int(d["Y_prediction_test"][0,index])].decode("utf-8") +  "\" picture.")
#

## Plot learning curve (with costs)
#costs = np.squeeze(d['costs'])
#plt.plot(costs)
#plt.ylabel('cost')
#plt.xlabel('iterations (per hundreds)')
#plt.title("Learning rate =" + str(d["learning_rate"]))
#plt.show()
#

learning_rates = [0.01, 0.001, 0.0001, 0.00001]
models = {}
for i in learning_rates:
    print ("learning rate is: " + str(i))
    models[str(i)] = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 1500, learning_rate = i, print_cost = True)
    print ('\n' + "-------------------------------------------------------" + '\n')

for i in learning_rates:
    plt.plot(np.squeeze(models[str(i)]["costs"]), label= str(models[str(i)]["learning_rate"]))

plt.ylabel('cost')
plt.xlabel('iterations')
legend = plt.legend(loc='upper center', shadow=True)
frame = legend.get_frame()
frame.set_facecolor('0.90')
plt.show()


### START CODE HERE ## (PUT YOUR IMAGE NAME)
#my_image = "leaf.jpg"   # change this to the name of your image file
### END CODE HERE ##
#
## We preprocess the image to fit your algorithm.
#fname = my_image
#image = np.array(ndimage.imread(fname, flatten=False))
#my_image = scipy.misc.imresize(image, size=(num_px,num_px)).reshape((1, num_px*num_px*3)).T
#my_predicted_image = predict(d["w"], d["b"], my_image)
#
#plt.imshow(image)
#print("y = " + str(np.squeeze(my_predicted_image)) + ", your algorithm predicts a \"" + classes[int(np.squeeze(my_predicted_image)),].decode("utf-8") +  "\" picture.")
#


#print("\nTesting What our system predict when we enter sepal lenght and width and Petal Length and Width : ")
#print("\nEntering attributes of a setosa flower : ")
#input_arr = np.array([5.2,3.5,1.4,0.2]).reshape(1,4).T
#print(input_arr.shape)
#y=0.0
#my_predicted_flower=predict(d["w"],d["b"],input_arr,y)
#print("y = " + str(np.squeeze(my_predicted_flower)) + ", your algorithm predicts a \"" + classes[int(np.squeeze(my_predicted_flower)),].decode("utf-8") +  "\" flower.")
#