Probability gets reverted to 0 in each epoch for same number
I'm having issue with the code below. I'll try to sum up my problem, so please read it before diving into the code. So, I customly created Network/Levels simulation in python. The code which doesn't do what I want it to do is
for i in range(1):
for i in get_indices(labels, 0)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 2)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 1)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 3)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 5)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 4)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 6)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 7)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 8)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 9)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
Basically, I try to teach my neural network number per number. I have 10 output neurons for numbers between 0-9. I iterate each time through 1000 different examples of each number. If I try to teach it what 0 is, it does kinda learn it and the first output neuron gets close to 1. Now, if I switch and try to teach it any other number, it lowers the probability of the number being 0 to a very small number (first output neuron, which represents number 0, gets back to a value close to 0 when showing it any other number).
If I run it multiple times ( I increase the range ) and circle goes back to 0, it didn't learn anything and it starts again with probability of 0. If anyone can find where I am going wrong conceptually, please let me know.
Full code:
from asyncio.windows_events import NULL
from json import JSONEncoder
import math
import random
import sys
from matplotlib import image
from matplotlib.font_manager import json_dump
import numpy as np
import idx2numpy
import matplotlib.pyplot as plt
def sig(x):
return 1/(1 + np.exp(-x))
class Level:
def __init__(self, inputCount, outputCount) -> None:
self.inputCount = inputCount
self.outputCount = outputCount
self.input = np.zeros(inputCount)
self.output = np.zeros(outputCount)
self.weights = np.zeros(inputCount * outputCount)
self.bias = np.zeros(outputCount)
self.randomizeWeights(inputCount, outputCount)
self.randomizeBias(outputCount)
pass
def randomizeWeights(self, inputCount, outputCount):
for i in range(inputCount * outputCount):
num = (random.random() * 2) - 1
self.weights[i] = num
self.weights = self.weights.reshape(outputCount, inputCount)
#print(self.weights)
def randomizeBias(self, outputCount):
for i in range(outputCount):
self.bias[i] = (random.random() * 2) - 1
self.bias = self.bias.reshape(outputCount, 1)
def setInput(self, img, dontLoop=False):
self.input = img.astype(dtype=float)
self.input = self.input.reshape(self.inputCount, 1)
if not dontLoop:
for i, element in enumerate(self.input):
val = self.input[i][0]
newVal = float(val) / 255.0
self.input[i][0] = newVal
def feedForward(self, useSigmoid=False):
self.output = np.dot(self.weights, self.input) + self.bias
if useSigmoid:
for i, element in enumerate(self.output):
val = self.output[i][0]
newVal = sig(val)
self.output[i][0] = newVal
for i, element in enumerate(self.bias):
val = self.output[i][0]
newVal = sig(val)
self.bias[i][0] = newVal
return self.output
class Network:
def __init__(self) -> None:
self.level1 = Level(784, 128)
self.level2 = Level(128, 64)
self.level3 = Level(64,16)
self.level4 = Level(16, 16)
self.level5 = Level(16,10)
def getResult(self, img, predicted):
self.level1.setInput(img)
output = self.level1.feedForward(useSigmoid=True)
self.level2.setInput(output, True)
output = self.level2.feedForward(useSigmoid=True)
self.level3.setInput(output, True)
output = self.level3.feedForward(useSigmoid=True)
self.level4.setInput(output, True)
output = self.level4.feedForward(useSigmoid=True)
self.level5.setInput(output, True)
output = self.level5.feedForward(useSigmoid=True)
value = np.argmax(self.level5.output)
predictedMatrix = np.array([[0,0,0,0,0,0,0,0,0,0]])
predictedMatrix[0][predicted] = 1
delta_o = self.level5.output.T - predictedMatrix
w_h_o = -0.001* np.dot(delta_o.T, self.level5.input.T)
self.level5.weights = self.level5.weights + w_h_o
self.level5.bias = self.level5.bias - (-0.001 * delta_o.T)
h = self.level4.output * (1 - self.level4.output)
s = np.dot(w_h_o.T, delta_o.T)
delta_h = s * h
w_i_h = -0.001*np.dot(delta_h, self.level4.input.T)
self.level4.weights = self.level4.weights + w_i_h
self.level4.bias = self.level4.bias - (-0.001 * delta_h)
h = self.level3.output * (1 - self.level3.output)
s = np.dot(w_i_h, delta_h)
delta_g = s * h
#print(delta_g.shape)
w_i_h_1 = -0.001*np.dot(delta_g, self.level3.input.T)
#print(w_i_h_1.shape)
#print(self.level3.weights.shape)
self.level3.weights = self.level3.weights + w_i_h_1
self.level3.bias = self.level3.bias - (-0.001 * delta_g)
h = self.level2.output * (1 - self.level2.output)
s = np.dot(w_i_h_1.T, delta_g)
delta_d = s * h
w_i_h_2 = -0.001*np.dot(delta_d, self.level2.input.T)
self.level2.weights = self.level2.weights + w_i_h_2
self.level2.bias = self.level2.bias - (-0.001 * delta_d)
h = self.level1.output * (1 - self.level1.output)
s = np.dot(w_i_h_2.T, delta_d)
delta_b = s * h
w_i_h_3 = -0.001*np.dot(delta_b, self.level1.input.T)
self.level1.weights = self.level1.weights + w_i_h_3
self.level1.bias = self.level1.bias - (-0.001 * delta_b)
#print(self.level3.weights)
#print(self.level1.output)
print(self.level5.output)
#print(self.level2.weights)
print(value, predicted)
return value
imagefile = 'train-images.idx3-ubyte'
imagearray = idx2numpy.convert_from_file(imagefile)
labels = idx2numpy.convert_from_file("train-labels.idx1-ubyte")
def get_indices(x: list, value: int) -> list:
indices = list()
for i in range(len(x)):
if x[i] == value:
indices.append(i)
return indices
network = Network()
file = open('demofile3.txt', 'r')
text = file.read()
array = text.split('\n')
zeros = get_indices(labels, 9)
for i in range(1):
for i in get_indices(labels, 0)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 2)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 1)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 3)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 5)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 4)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 6)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 7)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 8)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
for i in get_indices(labels, 9)[0:1000]:
output = network.getResult(imagearray[i], labels[i])
#network.level1.weights = weights1
#network.level2.weights = weights2
#network.level3.weights = weights3
correct = 0
for img, label in zip(imagearray,labels):
output = network.getResult(img, label)
for img, label in zip(imagearray,labels):
output = network.getResult(img, label)
if output == label:
correct += 1
print(correct)
f = open("demofile3.txt", "w")
f.write(str(network.level1.weights.tobytes()))
f.write('\n')
f.write(str(network.level2.weights.tobytes()))
f.write('\n')
f.write(str(network.level3.weights.tobytes()))
f.close()
#for i in get_indices(labels, 9):
#img = plt.imshow(imagearray[i])
#plt.show()
#output = network.getResult(imagearray[i], labels[i])
#if(output == labels[i]):
#correct+=1
#print('correct')
# print(correct)
# print(len(get_indices(labels,9)))
#print(arr1.T, arr2.T)
#print(np.dot(arr1.T, arr2.T))
🔴 No definitive solution yet