from pprint import pprint
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import sklearn.datasets
from init_utils import sigmoid, relu, compute_loss, forward_propagation, backward_propagation
from init_utils import update_parameters, predict, load_dataset, plot_decision_boundary, predict_dec

# %matplotlib inline
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()


def initialize_parameters_zeros(layers_dims):
    """
    Arguments:
    layer_dims -- python array (list) containing the size of each layer.

    Returns:
    parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                    W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                    b1 -- bias vector of shape (layers_dims[1], 1)
                    ...
                    WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                    bL -- bias vector of shape (layers_dims[L], 1)
    """

    parameters = {}
    L = len(layers_dims)  # number of layers in the network

    for l in range(1, L):
        ### START CODE HERE ### (≈ 2 lines of code)
        # if l == 1:
        #     parameters['W' + str(l)] = np.full((
        #         layers_dims[l], layers_dims[l - 1]), 0)
        # else:
        parameters['W' + str(l)] = np.full((
            layers_dims[l], layers_dims[l - 1]), np.exp(np.random.randn()))
        parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))

        parameters['W_LR' + str(l)] = np.exp(np.random.randn(
            layers_dims[l], layers_dims[l - 1]))
        parameters['b_LR' + str(l)] = np.exp(np.random.randn(
            layers_dims[l], 1))
        ### END CODE HERE ###
    pprint(parameters)
    return parameters


def initialize_parameters_random(layers_dims):
    """
    Arguments:
    layer_dims -- python array (list) containing the size of each layer.

    Returns:
    parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                    W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                    b1 -- bias vector of shape (layers_dims[1], 1)
                    ...
                    WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                    bL -- bias vector of shape (layers_dims[L], 1)
    """

    # np.random.seed(
    #     3)  # This seed makes sure your "random" numbers will be the as ours
    parameters = {}
    L = len(layers_dims)  # integer representing the number of layers

    for l in range(1, L):
        ### START CODE HERE ### (≈ 2 lines of code)
        if l == (L - 1):
            parameters['W' + str(l)] = np.zeros((
                layers_dims[l], layers_dims[l - 1]))
        else:
            parameters['W' + str(l)] = np.random.rand(
                layers_dims[l], layers_dims[l - 1])
        parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))

        parameters['W_LR' + str(l)] = np.random.rand(
            layers_dims[l], layers_dims[l - 1])
        parameters['b_LR' + str(l)] = np.random.rand(layers_dims[l], 1)
        ### END CODE HERE ###

    return parameters


def model(X, Y, learning_rate=0.01, num_iterations=15000, print_cost=True,
          initialization="he"):
    """
    Implements a three-layer neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SIGMOID.

    Arguments:
    X -- input data, of shape (2, number of examples)
    Y -- true "label" vector (containing 0 for red dots; 1 for blue dots), of shape (1, number of examples)
    learning_rate -- learning rate for gradient descent
    num_iterations -- number of iterations to run gradient descent
    print_cost -- if True, print the cost every 1000 iterations
    initialization -- flag to choose which initialization to use ("zeros","random" or "he")

    Returns:
    parameters -- parameters learnt by the model
    """

    grads = {}
    costs = []  # to keep track of the loss
    m = X.shape[1]  # number of examples
    layers_dims = [X.shape[0], 10, 5, 1]

    # Initialize parameters dictionary.
    if initialization == "zeros":
        parameters = initialize_parameters_zeros(layers_dims)
    elif initialization == "random":
        parameters = initialize_parameters_random(layers_dims)
    # elif initialization == "he":
    #     parameters = initialize_parameters_he(layers_dims)
    else:
        raise Exception('No Initialization Method Specified')

    # print(parameters)

    # Loop (gradient descent)

    for i in range(0, num_iterations):

        # Forward propagation: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID.
        a3, cache = forward_propagation(X, parameters)

        # Loss
        cost = compute_loss(a3, Y)

        # Backward propagation.
        grads = backward_propagation(X, Y, cache)

        # Update parameters.
        parameters = update_parameters(parameters, grads, learning_rate)

        # Print the loss every 1000 iterations
        if print_cost and i % 1000 == 0:
            print("Cost after iteration {}: {}".format(i, cost))
            costs.append(cost)

    # # plot the loss
    # plt.plot(costs)
    # plt.ylabel('cost')
    # plt.xlabel('iterations (per hundreds)')
    # plt.title("Learning rate =" + str(learning_rate))
    # plt.show()

    return parameters


# from nns.connection import Connection
from nns.activation import (
    Sigmoid, Relu
)
from nns.loss import CrossEntropy
from nns.layer import Layer
from nns.model import Model


if __name__ == '__main__':

    model = Model(loss=CrossEntropy(), learning_rate=0.1)
    model.append(Layer(train_X.shape[0], Relu()))
    model.append(Layer(10, Sigmoid()))
    model.append(Layer(5, Sigmoid()))
    model.append(Layer(1, Sigmoid()))
    model.compile()

    model.train(train_X, train_Y, number_iterations=15000)

    # parameters = model(train_X, train_Y, initialization="zeros", num_iterations=15000)
    # import pprint
    # pprint.pprint(parameters)

    # print ("On the train set:")
    # predictions_train = predict(train_X, train_Y, parameters)
    # print ("On the test set:")
    # predictions_test = predict(test_X, test_Y, parameters)

    plt.title("Model with large random initialization")
    axes = plt.gca()
    axes.set_xlim([-1.5, 1.5])
    axes.set_ylim([-1.5, 1.5])
    plot_decision_boundary(lambda x: model.predict(x.T) > 0.5, train_X, train_Y)

from pprint import pprint
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import sklearn.datasets
from init_utils import sigmoid, relu, compute_loss, forward_propagation, backward_propagation
from init_utils import update_parameters, predict, load_dataset, plot_decision_boundary, predict_dec

# %matplotlib inline
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()


def initialize_parameters_zeros(layers_dims):
    """
    Arguments:
    layer_dims -- python array (list) containing the size of each layer.

    Returns:
    parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                    W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                    b1 -- bias vector of shape (layers_dims[1], 1)
                    ...
                    WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                    bL -- bias vector of shape (layers_dims[L], 1)
    """

    parameters = {}
    L = len(layers_dims)  # number of layers in the network

    for l in range(1, L):
        ### START CODE HERE ### (≈ 2 lines of code)
        # if l == 1:
        #     parameters['W' + str(l)] = np.full((
        #         layers_dims[l], layers_dims[l - 1]), 0)
        # else:
        parameters['W' + str(l)] = np.full((
            layers_dims[l], layers_dims[l - 1]), np.exp(np.random.randn()))
        parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))

        parameters['W_LR' + str(l)] = np.exp(np.random.randn(
            layers_dims[l], layers_dims[l - 1]))
        parameters['b_LR' + str(l)] = np.exp(np.random.randn(
            layers_dims[l], 1))
        ### END CODE HERE ###
    pprint(parameters)
    return parameters


def initialize_parameters_random(layers_dims):
    """
    Arguments:
    layer_dims -- python array (list) containing the size of each layer.

    Returns:
    parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                    W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                    b1 -- bias vector of shape (layers_dims[1], 1)
                    ...
                    WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                    bL -- bias vector of shape (layers_dims[L], 1)
    """

    # np.random.seed(
    #     3)  # This seed makes sure your "random" numbers will be the as ours
    parameters = {}
    L = len(layers_dims)  # integer representing the number of layers

    for l in range(1, L):
        ### START CODE HERE ### (≈ 2 lines of code)
        if l == (L - 1):
            parameters['W' + str(l)] = np.zeros((
                layers_dims[l], layers_dims[l - 1]))
        else:
            parameters['W' + str(l)] = np.random.rand(
                layers_dims[l], layers_dims[l - 1])
        parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))

        parameters['W_LR' + str(l)] = np.random.rand(
            layers_dims[l], layers_dims[l - 1])
        parameters['b_LR' + str(l)] = np.random.rand(layers_dims[l], 1)
        ### END CODE HERE ###

    return parameters


def model(X, Y, learning_rate=0.01, num_iterations=15000, print_cost=True,
          initialization="he"):
    """
    Implements a three-layer neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SIGMOID.

    Arguments:
    X -- input data, of shape (2, number of examples)
    Y -- true "label" vector (containing 0 for red dots; 1 for blue dots), of shape (1, number of examples)
    learning_rate -- learning rate for gradient descent
    num_iterations -- number of iterations to run gradient descent
    print_cost -- if True, print the cost every 1000 iterations
    initialization -- flag to choose which initialization to use ("zeros","random" or "he")

    Returns:
    parameters -- parameters learnt by the model
    """

    grads = {}
    costs = []  # to keep track of the loss
    m = X.shape[1]  # number of examples
    layers_dims = [X.shape[0], 10, 5, 1]

    # Initialize parameters dictionary.
    if initialization == "zeros":
        parameters = initialize_parameters_zeros(layers_dims)
    elif initialization == "random":
        parameters = initialize_parameters_random(layers_dims)
    # elif initialization == "he":
    #     parameters = initialize_parameters_he(layers_dims)
    else:
        raise Exception('No Initialization Method Specified')

    # print(parameters)

    # Loop (gradient descent)

    for i in range(0, num_iterations):

        # Forward propagation: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID.
        a3, cache = forward_propagation(X, parameters)

        # Loss
        cost = compute_loss(a3, Y)

        # Backward propagation.
        grads = backward_propagation(X, Y, cache)

        # Update parameters.
        parameters = update_parameters(parameters, grads, learning_rate)

        # Print the loss every 1000 iterations
        if print_cost and i % 1000 == 0:
            print("Cost after iteration {}: {}".format(i, cost))
            costs.append(cost)

    # # plot the loss
    # plt.plot(costs)
    # plt.ylabel('cost')
    # plt.xlabel('iterations (per hundreds)')
    # plt.title("Learning rate =" + str(learning_rate))
    # plt.show()

    return parameters


# from nns.connection import Connection
from nns.activation import (
    Sigmoid, Relu
)
from nns.loss import CrossEntropy
from nns.layer import Layer
from nns.model import Model


if __name__ == '__main__':

    model = Model(loss=CrossEntropy(), learning_rate=0.1)
    model.append(Layer(train_X.shape[0], Relu()))
    model.append(Layer(10, Sigmoid()))
    model.append(Layer(5, Sigmoid()))
    model.append(Layer(1, Sigmoid()))
    model.compile()

    model.train(train_X, train_Y, number_iterations=15000)

    # parameters = model(train_X, train_Y, initialization="zeros", num_iterations=15000)
    # import pprint
    # pprint.pprint(parameters)

    # print ("On the train set:")
    # predictions_train = predict(train_X, train_Y, parameters)
    # print ("On the test set:")
    # predictions_test = predict(test_X, test_Y, parameters)

    plt.title("Model with large random initialization")
    axes = plt.gca()
    axes.set_xlim([-1.5, 1.5])
    axes.set_ylim([-1.5, 1.5])
    plot_decision_boundary(lambda x: model.predict(x.T) > 0.5, train_X, train_Y)

# %matplotlib inline # set default size of plots # load image dataset: blue/red dots in circles Arguments:
     layer_dims -- python array (list) containing the size of each layer.
 
     Returns:
     parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                     W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                     b1 -- bias vector of shape (layers_dims[1], 1)
                     ...
                     WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                     bL -- bias vector of shape (layers_dims[L], 1) # number of layers in the network ### START CODE HERE ### (≈ 2 lines of code) # if l == 1: #     parameters['W' + str(l)] = np.full(( #         layers_dims[l], layers_dims[l - 1]), 0) # else: ### END CODE HERE ### Arguments:
     layer_dims -- python array (list) containing the size of each layer.
 
     Returns:
     parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
                     W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
                     b1 -- bias vector of shape (layers_dims[1], 1)
                     ...
                     WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
                     bL -- bias vector of shape (layers_dims[L], 1) # np.random.seed( #     3)  # This seed makes sure your "random" numbers will be the as ours # integer representing the number of layers ### START CODE HERE ### (≈ 2 lines of code) ### END CODE HERE ### Implements a three-layer neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SIGMOID.
 
     Arguments:
     X -- input data, of shape (2, number of examples)
     Y -- true "label" vector (containing 0 for red dots; 1 for blue dots), of shape (1, number of examples)
     learning_rate -- learning rate for gradient descent
     num_iterations -- number of iterations to run gradient descent
     print_cost -- if True, print the cost every 1000 iterations
     initialization -- flag to choose which initialization to use ("zeros","random" or "he")
 
     Returns:
     parameters -- parameters learnt by the model # to keep track of the loss # number of examples # Initialize parameters dictionary. # elif initialization == "he": #     parameters = initialize_parameters_he(layers_dims) # print(parameters) # Loop (gradient descent) # Forward propagation: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID. # Loss # Backward propagation. # Update parameters. # Print the loss every 1000 iterations # # plot the loss # plt.plot(costs) # plt.ylabel('cost') # plt.xlabel('iterations (per hundreds)') # plt.title("Learning rate =" + str(learning_rate)) # plt.show() # from nns.connection import Connection # parameters = model(train_X, train_Y, initialization="zeros", num_iterations=15000) # import pprint # pprint.pprint(parameters) # print ("On the train set:") # predictions_train = predict(train_X, train_Y, parameters) # print ("On the test set:") # predictions_test = predict(test_X, test_Y, parameters)