Artificial Neural Network(ANN)-MLP Gradient Descent

MLP Gradient Descent

1.수치미분(Numerical Derivative)

1-1. gradient( ) 함수 정의

import numpy as np 

def gradient(machine, param):

    if param.ndim == 1:
        temp_param = param
        delta = 0.00005
        learned_param = np.zeros(param.shape)
        
        for index in range(len(param)):
            target_param = float(temp_param[index])
            temp_param[index] = target_param + delta            
            param_plus_delta = machine(temp_param)
            temp_param[index] = target_param - delta  
            param_minus_delta = machine(temp_param)
            learned_param[index] = (param_plus_delta - param_minus_delta ) / (2 * delta)
            temp_param[index] = target_param

        return learned_param
        

    elif param.ndim == 2:
        temp_param = param
        delta = 0.00005
        learned_param = np.zeros(param.shape)
    
        rows = param.shape[0]
        columns = param.shape[1]
    
        for row in range(rows):
            for column in range(columns):
                target_param = float(temp_param[row, column])
                temp_param[row, column] = target_param + delta            
                param_plus_delta = machine(temp_param)
                temp_param[row, column] = target_param - delta            
                param_minus_delta = machine(temp_param)
                learned_param[row, column] = (param_plus_delta - param_minus_delta) / (2 * delta)
                temp_param[row, column] = target_param

        return learned_param

---

1-2. Activation - sigmoid( )

def sigmoid(x):
    y_hat = 1 / (1 + np.exp(-x))
    return y_hat

---

2.Data Preprocessing

2-1. 실습용 데이터 설정

import seaborn as sns

DF = sns.load_dataset('iris')

DF.info()

DF.head(3)

---

2-2. Data Set Slicing

X = DF.iloc[:100, 2:4]

y = DF.iloc[:100, 4]

X_input = X.values
y_output = y.values

---

2-3. Integer Encoding

from sklearn.preprocessing import LabelEncoder

encoder =  LabelEncoder()
y_output = encoder.fit_transform(y_output)

y_output

X_input.shape, y_output.shape

((100, 2), (100,))

---

3. Model Training(Learning)

3-1. MLP_Classifier Class

class MLP_Classifier:
    
    def __init__(self, problem_Type, X_input, y_output):  

# problem_Type 문자열 지정 Member      
        self.Type = problem_Type
        
# X_input, y_output Member 초기화
        self.X_input = X_input.reshape(100, 2)
        self.y_output = y_output.reshape(100, 1)
        
# W_1, b_1 Member 초기화
        self.W_1 = np.random.rand(2, 2)  
        self.b_1 = np.random.rand(2)

# W_2, b_2 Member 초기화
        self.W_2 = np.random.rand(2, 1)  
        self.b_2 = np.random.rand(1)

# learning_rate Member 지정
        self.learning_rate = 0.01

# Cost_Function(CEE) Method
    def cost_func(self):

        z_1 = np.dot(self.X_input, self.W_1) + self.b_1     # Hidden Layer
        a_1 = sigmoid(z_1)                                
        
        z_2 = np.dot(a_1, self.W_2) + self.b_2            # Output Layer
        y_hat = sigmoid(z_2)                        

        delta = 0.00001
        return -np.sum(self.y_output * np.log(y_hat + delta) + (1 - self.y_output) * np.log((1 - y_hat) + delta))      

# Learning Method
    def fit(self):
        machine = lambda x : self.cost_func()
        print('Initial Cost = ', self.cost_func())
        
        for step in  range(10001):
            self.W_1 = self.W_1- self.learning_rate * gradient(machine, self.W_1)
            self.b_1 = self.b_1 - self.learning_rate * gradient(machine, self.b_1)

            self.W_2 = self.W_2 - self.learning_rate * gradient(machine, self.W_2)
            self.b_2 = self.b_2 - self.learning_rate * gradient(machine, self.b_2)
    
            if (step % 1000 == 0):
                print('Step = ', step, 'Cost = ', self.cost_func())
                
# Predict Method
    def predict(self, input_data):
        
        z_1 = np.dot(input_data, self.W_1) + self.b_1     # Hidden Layer
        a_1 = sigmoid(z_1)                                
        
        z_2 = np.dot(a_1, self.W_2) + self.b_2            # Output Layer
        y_prob = sigmoid(z_2)                             


        if y_prob > 0.5:
            result = 1
        else:
            result = 0
    
        return y_prob, result

---

3-2. IRIS.learn( )

%%time

IRIS = MLP_Classifier('Binary_Classification', X_input, y_output)

IRIS.fit()

---

3-3. IRIS.predict( )

---

3-4. Confusion Matrix

from sklearn.metrics import confusion_matrix

confusion_matrix(y_output, y_hat)