PyNet/ImprovingNNs/OptimizationProblems.py at master · AnujCodeZ/PyNet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import numpy as np
from Utils import *

class GradProblems(DeepNet):

    def __init__(self):
        super(GradProblems, self).__init__()

        self.he_initialization()

    """He Weight Initialization
    To overcome vanishing/exploding gradients"""
    def he_initialization(self):

        for i in range(len(self.layers)):
            parameters['W'+str(i+1)] = np.random.randn(self.layers[i], self.layers[i+1]) * np.sqrt(2 / self.layers[i])
            parameters['b'+str(i+1)] = np.zeros((1, self.layers[i+1]))

    """Gradient checking
    To check whether our gradient is right or not"""
    def forward_with_theta(self, X, theta):

        params = vector_to_dictionary(theta, self.layers)
        A = X

        for i in range(len(self.layers - 1)):

            A_prev = A

            Z = np.dot(A_prev, params['W'+str(i+1)]) + params['b'+str(i+1)]
            A = self.relu(Z)
            self.caches.append(A)

        Z = np.dot(A_prev, params['W'+str(len(self.layers))]) + params['b'+str(len(self.layers))]
        AL = self.softmax(Z)

        return AL

    def grad_check(self, X, Y, EPSILON = 10e-7):

        params = dictionary_to_vector(self.parameters)
        grads = gradients_to_vector(self.gradients)

        num_parameters = len(params)
        gradsApprox = np.zeros((num_parameters, 1))

        for i in range(num_parameters):

            thetaPlus = np.copy(params)
            thetaPlus[i] = thetaPlus[i] + EPSILON

            AL = self.forward_with_theta(X, thetaPlus)
            J_plus = self.compute_cost(AL, Y)

            thetaMinus = np.copy(params)
            thetaMinus[i][0] = thetaMinus[i][0] - EPSILON

            AL = self.forward_with_theta(X, thetaMinus)
            J_minus = self.compute_cost(AL, Y)

            gradsApprox[i] = (J_plus - J_minus) / (2 * EPSILON)

        numerator = np.linalg.norm(grads - gradsApprox)
        denomenator = np.linalg.norm(grads) + np.linalg.norm(gradsApprox)

        difference = numerator / denomenator

        if difference > 10e-7:

            print("Back-propagation is not working properly.\n")

        else:

            print("Back-propagation is fine.\n")