I'm trying to make neural network with 2 layers(1st have 2 neurons, 2nd have 1 neuron) that looks through the list of 4 numbers, and if the first AND third numbers are equal to 1 then answer should be 1, in other cases = 0.

I dont use any AI framework libraries at all, just random to make random weights and numpy yo make mean square loss. I know, that if I don't set seed to randomizer it will be always random, but my answers are way too random.

I have test dataset, where first and last list should be 1's, and 2nd and 3rd are 0:

x2 = [
    [1, 0, 1, 0], # answer: 1
    [1, 0, 0, 0], # answer: 0
    [0, 1, 1, 0], # answer: 0
    [1, 0, 1, 1] # answer: 1
]

But every time I run the code, the answer to 1st and 2nd list are more random, than 3rd and 4th. Sometimes the answer to 2nd is bigger than first.

I tried to change the weights error adjustment from:

err3 = y1[i] - neuron3.activate(pY3[i])
err1 = y1[i] - pY1[i]
err2 = y1[i] - pY2[i]
neuron1.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * err1 * lr
neuron2.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * err2 * lr
#(x1 is training dataset, [i] is list index and [k] is list element index)

to:

neuron1.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * numpy.mean(err1 ** 2) * lr
neuron2.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * numpy.mean(err2 ** 2) * lr

but I have the oveflow errors.

All the code I currently have:

import random
import math
import numpy

class Neuron:
    def __init__(self, weightsC):
        self.w = []
        for i in range(weightsC):
            self.w.append(random.random())

    def activate(self, x):
        return max(0, x)

    def predict(self, px):
        value = 0
        for i in range(len(self.w)):
            value += px[i] * self.w[i]
        return value

    '''def train(self, count, x, y, lr=0.1):
        print("\tTraining Start:")
        for i in range(count):
            print("EPOCH:", i)
            for j in range(len(x)):
                py = self.predict(x[j])
                print("pY:", py)
                err = y[j] - self.activate(py)
                print("err:", err)
                for k in range(len(self.w)):
                    self.w[k] += err * x[j][k] * lr
                    print("w[k]", self.w[k])'''

x1 = [
    [0, 0, 0, 0],
    [1, 0, 1, 0],
    [1, 1, 0, 1],
    [0, 1, 1, 0],
    [1, 1, 0, 0],
    [1, 0, 1, 1],
    [1, 1, 1, 0]
]

y1 = [0, 1, 0, 0, 0, 1, 1]

x2 = [
    [1, 0, 1, 0],
    [1, 0, 0, 0],
    [0, 1, 1, 0],
    [1, 0, 1, 1]
]

pY1 = []
pY2 = []
pY3 = []

if __name__ == "__main__":
    lr = 0.001

    neuron1 = Neuron(4)
    neuron2 = Neuron(4)
    neuron3 = Neuron(2)
    print("w1 = ", neuron1.w)
    print("w2 = ", neuron2.w)
    print("w3 = ", neuron3.w)

    for l in range(1000):
        #print("EPOCH = ", l)
        for i in range(len(x1)):
            pY1.append(
                neuron1.activate(
                    neuron1.predict(x1[i])
                )
            )
            #print("layer1_pred1 = ", pY1[i])

            pY2.append(
                neuron2.activate(
                    neuron2.predict(x1[i])
                )
            )
            #print("layer1_pred2 = ", pY2[i])

            layer1_pred = [pY1[i], pY2[i]]
            pY3.append(
                neuron3.predict(
                    layer1_pred
                )
            )
            #print("layer2_pred1 = ", pY3[i])

            err3 = y1[i] - neuron3.activate(pY3[i])
            #print("err3 = ", err3)

            for j in range(len(neuron3.w)):
                neuron3.w[j] += layer1_pred[j] * numpy.mean(err3 ** 2) * lr

            err1 = y1[i] - pY1[i]
            #print("err1 = ", err1)
            err2 = y1[i] - pY2[i]
            #print("err2 = ", err2)

            for k in range(len(neuron1.w)):
                neuron1.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * err1 * lr
            for k in range(len(neuron2.w)):
                neuron2.w[k] += x1[i][k] * numpy.mean(err3 ** 2) * err2 * lr

        pY1 = pY2 = pY3 = []

    print("testing")
    for item in x2:
        pred1 = neuron1.activate(neuron1.predict(item))
        pred2 = neuron2.activate(neuron2.predict(item))
        print(neuron3.predict([pred1, pred2]))