Pattern Recognition

The Back-Propagation Algorithm for Multi-Layer Perceptron

The Back-Propagation Algorithm for Multi-Layer Perceptron

Class Multi-Layer Perceptron

import numpy
from math import exp
from matplotlib import pyplot


class MLP:
    # properties
    int_num_epochs = 1000
    int_num_input_neurons = 0
    int_num_output_neurons = 0
    int_num_hidden_layers = 0
    int_num_hidden_neurons = 0
    dbl_mse_threshold = 0.001
    dbl_eta = 0.0001
    dbl_bias = 0.0002
    dbl_w0 = 0.0002

    arr_num_neurons_in_hidden_layers = []

    # weights
    wo = []
    wh = []

    # mse
    arr_mse = []

    ## initializations
    def __init__(self, _int_num_input_neurons, _int_num_output_neurons, _int_num_hidden_layers, _int_num_epochs,
                 _int_num_hidden_neurons, _dbl_eta):
        ## initialize properties
        self.int_num_input_neurons = _int_num_input_neurons
        self.int_num_output_neurons = _int_num_output_neurons
        self.int_num_hidden_layers = _int_num_hidden_layers
        self.int_num_epochs = _int_num_epochs
        self.int_num_hidden_neurons = _int_num_hidden_neurons
        self.dbl_eta = _dbl_eta

        ## initialize weights arrays
        self.wo = [[self.dbl_w0 for x in range(self.int_num_hidden_neurons + 1)] for y in
                   range(self.int_num_output_neurons)]  ## bias +1
        self.wh = [[self.dbl_w0 for x in range(self.int_num_input_neurons + 1)] for y in
                   range(self.int_num_hidden_neurons)]  ## bias +1
        return

    ## back-propagation-algorithm
    def train(self, training_set):
        ## loop epochs
        mse = []
        for e in range(0, self.int_num_epochs):
            ## loop training set
            errors = []
            for t in range(0, len(training_set)):
                # inputs
                x = training_set[t][0]

                # response
                d = training_set[t][1]

                # forward path
                actual_hidden_output = self.hyberb(numpy.inner(self.wh, x))

                actual_hidden_output_plus_rshp = numpy.reshape(actual_hidden_output, (self.int_num_hidden_neurons))
                actual_hidden_output_plus_bias = numpy.append(actual_hidden_output_plus_rshp, self.dbl_bias)

                actual_output = self.hyberb(numpy.inner(self.wo, actual_hidden_output_plus_bias))

                ## Question?! Why substract actual_output[3x1] Vector from scalar d = {0, 1, 2}
                error = d - actual_output

                ## erros will be used for mse
                errors = numpy.append(errors, error)

                ## backward path
                error_signal_output = error * self.derivhyberb(numpy.inner(self.wo, actual_hidden_output_plus_bias))

                error_signal_output_rshp = numpy.reshape(error_signal_output, (self.int_num_hidden_neurons))
                ####### note there is no input weights to bias node            #######
                ####### add dumpy column for bias weights to avoid numpy error #######
                error_signal_output_dump_bias = numpy.append(error_signal_output_rshp, self.dbl_bias)
                error_signal_hidden = self.derivhyberb(numpy.inner(self.wh, x)) * numpy.inner(self.wo,
                                                                                              error_signal_output_dump_bias)
                ###dimensions of error_signal_hidden = (number_of_hidden_neurons x 1)
                ## update weights hidden
                tmp_wh = numpy.transpose(self.wh)
                counter = 0
                for x_ele in x:
                    delta_wh = self.dbl_eta * error_signal_hidden * x_ele
                    tmp_wh[counter] = delta_wh + tmp_wh[counter]  ## update weight
                    counter = counter + 1
                self.wh = tmp_wh.transpose()  ## weights updated
                ## end for x

                ## update weights output
                delta_wo = self.dbl_eta * error_signal_output * actual_hidden_output
                counter = 0
                for delta_wo_ele in delta_wo:
                    self.wo[counter] = self.wo[counter] + delta_wo_ele
                    counter = counter + 1
                    ## out weights updated
            ## end loop training set
            self.arr_mse = numpy.append(self.arr_mse, numpy.mean(numpy.sum(numpy.square(errors))))
        ## end loop epochs
        return

    ## hyber-bolic function
    def hyberb(self, V):
        """
        :rtype: VECTOR SAME DIMENSIONS AS V
        """
        # PHI.arange().reshape
        return (numpy.exp(V * 2) - 1) / (numpy.exp(V * 2) + 1)

    ## derivation of hyper-bolic function
    def derivhyberb(self, V):
        """
        :rtype: VECTOR SAME DIMENSIONS AS V
        """
        # PHI.arange().reshape
        return 4 * numpy.exp(V * 2) / numpy.square(1 + numpy.exp(V * 2))

    ## plot mse
    def plotMSE(self):
        pyplot.xlabel('Number of Epochs')
        pyplot.ylabel('MSE (Mean Square Error)')
        pyplot.plot(self.arr_mse)
        pyplot.show()

Class Controller

from sklearn.datasets import load_iris
from sklearn import preprocessing
import numpy

from MLP import MLP


class Controller:
    # dataset
    data = []
    training = []
    testing = []

    # characteristics
    int_set_size = 0
    int_num_features = 0
    int_training_size = 30
    int_testing_size = 20
    int_num_per_class = 50
    int_num_classes = 0

    # composition
    obj_mlp = MLP

    # initializiation
    def __init__(self):
        iris = load_iris()
        self.prepareDataSet(iris)

    # normalize or scale data
    # divide data to training and testing
    def prepareDataSet(self, iris):
        self.int_num_classes = numpy.unique(iris.target).shape[0]

        self.int_set_size = iris.data.shape[0]
        self.int_num_features = iris.data.shape[1]

        ## normalize data
        self.data = preprocessing.normalize(iris.data)
        self.data = preprocessing.minmax_scale(self.data, (-1, 1))

        # load data in arrays
        for i in range(0, len(self.data)):
            Y = iris.target[i]
            X = numpy.append(self.data[i], 1)  ## bias input = 1
            check_i = i % self.int_num_per_class
            if check_i < self.int_training_size:
                self.training.append([X, Y])
            else:
                self.testing.append([X, Y])

    # play the MLP
    def playMLP(self):
        # allow to test all combinations of settings
        i = 1  ## number hidden layers
        step_epochs = 50  ## number of epochs
        # To calculate the number of hidden nodes we use a general rule of: (Number of inputs + outputs) x 2/3
        k = 3  ## number of hidden neurons
        l = 0.0001  ## eta learning rate
        s = 0.1  ## step
        for j in range(0, 20):
            self.obj_mlp = MLP(self.int_num_features, self.int_num_classes, i, j * step_epochs, k, l)
            self.obj_mlp.train(self.training)
            self.obj_mlp.plotMSE()

Application Entry Point

from Controller import Controller

cApp = Controller()
cApp.playMLP()

Plot Number of Epochs vs Mean-Square Error

mse vs epochs

mse vs epochs

mse vs epochs

mse vs epochs
References:

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s