Neural Network.ipynb

Kernel: Python 3 (system-wide)

import sys
np.set_printoptions(threshold=sys.maxsize)

Neural Network

import numpy as np
from numba import njit
import random
import time

@njit
def sigmoid(x:float):
    return 1 / (1 + np.e**(-x))

@njit
def d_sigmoid_d_x(x:float):
    return np.e**(-x) / (1 + np.e**(-x))**2    # sigmoid(x) * (1 - sigmoid(x))

class Network(object):
    def __init__(self, sizesOfLayers:tuple):
        self.sizesOfLayers = sizesOfLayers
        self.numLayers = len(self.sizesOfLayers)
        self.weights = [np.empty(0)] + [np.random.rand(self.sizesOfLayers[L], self.sizesOfLayers[L-1]) for L in range(1, self.numLayers)] # w
        self.baises = [np.empty(0)] + [np.random.rand(self.sizesOfLayers[L]) for L in range(1, self.numLayers)] # b
        self._zLayers = [np.zeros(sizesOfLayers[L]) for L in range(self.numLayers)] # z
        self._aLayers = [np.zeros(sizesOfLayers[L]) for L in range(self.numLayers)] # a
        self._deltaL = [np.empty(0)] + [np.zeros(self.sizesOfLayers[L]) for L in range(1, self.numLayers)] # delta
        self.trainingDevelopement = []

    # calculates the Lth layer of the nextwork given the previous layers
    def calcLayer(self, L):
        self._zLayers[L] = (self.weights[L] @ self._aLayers[L-1]) + self.baises[L]
        self._aLayers[L] = sigmoid(self._zLayers[L])

    # calculates all layers given Layer 0
    def calcAllLayers(self, layer0:np.array):
        if len(layer0) != self.sizesOfLayers[0]:
            raise ValueError('Layer 0 entered does not match the size specified in the initialization of the Network.')
        self._aLayers[0] = layer0
        for L in range(1, self.numLayers):
            self.calcLayer(L)

    # backpropogate error (deltaL) for each L, given that all aLayers have been calculated
    def backpropogateDelta(self, desiredLastLayer:np.array):
        self._deltaL[-1] = (self._aLayers[-1] - desiredLastLayer) * d_sigmoid_d_x(self._zLayers[-1])
        for L in range(self.numLayers-1-1, 0, -1):
            self._deltaL[L] = (self.weights[L+1].T @ self._deltaL[L+1]) * d_sigmoid_d_x(self._zLayers[L])

    # Updates weights and baises using the last propogated deltas
    def train(self, listOfTrainingIOPairs:list, eta:float=1):
        RunningSumForWeights = [np.empty(0)] + [np.zeros((self.sizesOfLayers[L], self.sizesOfLayers[L-1])) for L in range(1, self.numLayers)] # w
        RunningSumForBaises = [np.empty(0)] + [np.zeros(self.sizesOfLayers[L]) for L in range(1, self.numLayers)] # b
        for term in listOfTrainingIOPairs:
            self.calcAllLayers(term[0])
            self.backpropogateDelta(term[1])
            for L in range(1, self.numLayers):
                RunningSumForWeights[L] += np.outer(self._deltaL[L],self._aLayers[L-1])
                RunningSumForBaises[L] += self._deltaL[L]
        # update weights and baises
        m = len(listOfTrainingIOPairs)
        for L in range(self.numLayers):
            self.weights[L] -= (eta/m) * RunningSumForWeights[L]
            self.baises[L] -= (eta/m) * RunningSumForBaises[L]
        # update the training developement (counter)
        self.trainingDevelopement += [m]

    # calculates all layers but returns the last layer, given Layer 0
    def evaluate(self, layer0:np.array):
        self.calcAllLayers(layer0)
        return self._aLayers[-1]

    # calculates the cost of one training input (Layer 0, intended Last Layer)
    def costTerm(self, trainingInput:list):
        [layer0, lastLayer] = trainingInput
        if len(lastLayer) != self.sizesOfLayers[-1]:
            raise ValueError('Last layer entered does not match the size specified in the initialization of the Network.')
        return np.sqrt(np.sum((self.evaluate(layer0) - lastLayer)**2))

    # calculates the cost for a list of training inputs (the average of each of the costs)
    def cost(self, listOfTrainingIOPairs:list):
        return np.mean(np.array([self.costTerm(term) for term in listOfTrainingIOPairs]))


    # TODO : save weights and baises and developement in text file and load in

Example 1 -- Calculate the activation index in Layer 0 mod 2

def trainingBatch1(size):
    output = [[np.zeros(10), np.zeros(2)] for _ in range(size)]
    for i in range(size):
        a = random.randint(0,9)
        output[i][0][a] = 1
        output[i][1][a % 2] = 1
    return output

trainingBatch1(6)

[[array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0.]), array([1., 0.])],
 [array([0., 1., 0., 0., 0., 0., 0., 0., 0., 0.]), array([0., 1.])],
 [array([0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]), array([1., 0.])],
 [array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0.]), array([1., 0.])],
 [array([0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]), array([1., 0.])],
 [array([0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]), array([1., 0.])]]

mod = Network((10, 2))

mod.cost(trainingBatch1(100))

0.7657942072586827

for _ in range(500):
    mod.train(trainingBatch1(1000), eta = 10)

mod.cost(trainingBatch1(10))

0.04772877820199093

test = [np.array([0., 1., 0., 0., 0., 0., 0., 0., 0., 0.]), np.array([0,1])]

mod.evaluate(test[0])

array([0.0338679 , 0.96623541])

Example 2 -- calculate the sum (mod 2) of the layer 0 activation indeces

def trainingBatch2(size):
    output = [[np.zeros(100), np.zeros(2)] for _ in range(size)]
    for i in range(size):
        a, b = random.randint(0,99), random.randint(0,99)
        while a == b:
            b = random.randint(0,99)
        output[i][0][a] = 1
        output[i][0][b] = 1
        output[i][1][(a+b) % 2] = 1
    return output

trainingBatch2(4)

[[array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),
  array([0., 1.])],
 [array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),
  array([0., 1.])],
 [array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),
  array([0., 1.])],
 [array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),
  array([1., 0.])]]

mod2 = Network((100, 51, 26, 2))

mod2.cost(trainingBatch2(100))

0.9999991187328802

start = time.time()
while time.time() < start + 60*60*10: # train for 10 hours
    mod2.train(trainingBatch2(1000), eta = 10)

len(mod2.trainingDevelopement)

467565

mod2.cost(trainingBatch2(100))

0.5233465458241786

test = trainingBatch2(1)[0]
print(test[1])
print(mod2.evaluate(test[0]))

[0. 1.]
[9.99976797e-01 2.52256731e-04]

print(mod2.weights)
print(mod2.baises)

[array([], dtype=float64), array([[0.58494921, 0.01415033, 0.63174099, ..., 0.61526328, 0.80996793,
        0.20116062],
       [0.35047621, 0.18322772, 0.31379899, ..., 0.87003398, 0.284239  ,
        0.55240806],
       [0.10878953, 0.7391501 , 0.7495949 , ..., 0.77846843, 0.51808993,
        0.1883835 ],
       ...,
       [0.78105589, 0.67228022, 0.68269343, ..., 0.55252231, 0.79825242,
        0.24974943],
       [0.44979713, 0.28861773, 0.51300244, ..., 0.76351741, 0.40015773,
        0.98643952],
       [0.09518811, 0.79851998, 0.68866488, ..., 0.21633674, 0.40694973,
        0.64077162]]), array([[0.69428967, 0.28689698, 0.84328325, ..., 0.94044099, 0.68697946,
        0.29816575],
       [0.43916721, 0.96987074, 0.27932841, ..., 0.33369908, 0.59846927,
        0.78451966],
       [0.80738411, 0.18418786, 0.85873567, ..., 0.42755888, 0.42047416,
        0.32988068],
       ...,
       [0.5316995 , 0.76613851, 0.6331347 , ..., 0.98334076, 0.34327029,
        0.72825008],
       [0.02388077, 0.07932315, 0.93442135, ..., 0.75558994, 0.61243459,
        0.73235588],
       [0.13768392, 0.3580835 , 0.45801164, ..., 0.36120806, 0.55172172,
        0.10634281]]), array([[ 0.36268797,  0.5402054 ,  0.44215021, -0.10766891,  0.5902183 ,
         0.59362831,  0.26380136, -0.06838254,  0.46378224,  0.70098932,
         0.80908437,  0.33982019,  0.83751648,  0.1436251 ,  0.349914  ,
         0.03185011,  0.25395301,  0.47574054,  0.53360701,  0.35537477,
        -0.01797316,  0.73839915,  0.35783876,  0.30741489,  0.15961505,
         0.36481549],
       [-0.24865784,  0.0952299 ,  0.08877549, -0.81073474, -0.08069126,
        -0.58300038,  0.03147882, -0.34230858, -0.31062553, -0.75778475,
        -0.28928746, -0.79977438, -0.15696855,  0.12579846, -0.05504892,
        -0.63592004, -0.40621764, -0.04531052, -0.70873542, -0.53892584,
         0.13663748, -0.44154281, -0.61328792, -0.79716307, -0.01308149,
        -0.00481686]])]
[array([], dtype=float64), array([0.88723469, 0.56333842, 0.07539592, 0.63840257, 0.88578997,
       0.99144936, 0.39700697, 0.69220596, 0.525926  , 0.20251269,
       0.84443939, 0.97024029, 0.69134359, 0.43751027, 0.47910048,
       0.89739342, 0.9033511 , 0.15283351, 0.60388935, 0.80415795,
       0.79177389, 0.67865858, 0.67777997, 0.14574305, 0.88862566,
       0.47932232, 0.29142613, 0.66345669, 0.34841965, 0.07489499,
       0.25659221, 0.53440728, 0.13111636, 0.23365207, 0.43966209,
       0.5498237 , 0.35922241, 0.78276817, 0.27445013, 0.81162512,
       0.14898316, 0.2875323 , 0.40527198, 0.44457016, 0.10269827,
       0.94271103, 0.28699797, 0.90116573, 0.36231381, 0.04554088,
       0.36378757]), array([0.91103274, 0.89347527, 0.6077036 , 0.60211046, 0.57062663,
       0.31368028, 0.11781489, 0.2276977 , 0.5601921 , 0.22370755,
       0.00213783, 0.66088722, 0.30084279, 0.34934425, 0.74619806,
       0.76018423, 0.46057392, 0.27341154, 0.18290353, 0.24518643,
       0.54189546, 0.64683471, 0.59889279, 0.10577189, 0.08926806,
       0.55741148]), array([ 0.84921829, -0.12284706])]

dict = {"weights" : mod2.weights, "baises" : mod2.baises}

f = open( 'mod2.py', 'w' )
f.write( 'dict = ' + repr(dict) + '\n' )
f.close()