diff --git a/src/NeuralNet.py b/src/NeuralNet.py index 2c98569..f6c55a1 100644 --- a/src/NeuralNet.py +++ b/src/NeuralNet.py @@ -23,13 +23,16 @@ def __init__(self, layer_sizes, num_inputs): self.layers[i] = new_layer # Set step size for numerical integration - self.h_param = 0.000001 + self.h_param = 0.0000001 # Set learning rate value - self.learning_rate = 0.4 + self.learning_rate = 3.0 def sim(self, inputs): """Simulates the feedforward network - inputs is a list of inputs which corresponds to the number of inputs in the input layer""" + inputs is a 2darray of inputs which corresponds to the number of inputs in the input layer + Columns in the inputs are samples + Rows are elements + Ie a single input should be a single column""" tempOutputs = inputs for i in self.layers: @@ -39,16 +42,17 @@ def sim(self, inputs): def train(self, inputs, outputs, num_epochs): """Trains the network - inputs is a 2darray of inputs, rows are samples, columns are elements - outputs is a 2darray of corresponding network outputs, rows are samples, columns are elements + inputs is a 2darray of inputs, rows are elements, columns are samples + outputs is a 2darray of corresponding network outputs, rows are elements, columns are samples num_epochs is the number of epochs to run """ # Loop through the epochs for i in range(num_epochs): # Find current cost - current_outputs = [self.sim(inputs[:,n]) for n in range(np.size(inputs, 1))] - current_outputs = np.concatenate(current_outputs,1) + #current_outputs = [self.sim(inputs[:,n]) for n in range(np.shape(inputs)[1])] + #current_outputs = np.concatenate(current_outputs,1) + current_outputs = self.sim(inputs) current_cost = cost(current_outputs, outputs) print "Cost for epoch %d: %f" % (i,current_cost) # Find the partial derivative of Cost w.r.t. each parameter @@ -59,8 +63,9 @@ def train(self, inputs, outputs, num_epochs): # Change weight by self.h_param j.w[k,m] += self.h_param # Simulate network - perturbed_outputs = [self.sim(inputs[:,n]) for n in range(np.size(inputs, 1))] - perturbed_outputs = np.concatenate(perturbed_outputs, 1) + #perturbed_outputs = [self.sim(inputs[:,n]) for n in range(np.size(inputs, 1))] + #perturbed_outputs = np.concatenate(perturbed_outputs, 1) + perturbed_outputs = self.sim(inputs) # Find cost perturbed_cost = cost(perturbed_outputs, outputs) # Calculate partial cost @@ -74,8 +79,9 @@ def train(self, inputs, outputs, num_epochs): # Change bias by self.h_param j.b[k] += self.h_param # Simulate network - perturbed_outputs = [self.sim(inputs[:,n]) for n in range(np.size(inputs, 1))] - perturbed_outputs = np.concatenate(perturbed_outputs, 1) + #perturbed_outputs = [self.sim(inputs[:,n]) for n in range(np.size(inputs, 1))] + #perturbed_outputs = np.concatenate(perturbed_outputs, 1) + perturbed_outputs = self.sim(inputs) # Find cost perturbed_cost = cost(perturbed_outputs, outputs) # Calculate partial cost @@ -92,16 +98,30 @@ def train(self, inputs, outputs, num_epochs): for k in j.nodes: k.update_parameters() ''' + def __str__(self): + """Print the network""" + + # Loop through the layers + a = "" + for i in range(len(self.layers)): + a+=str(self.layers[i])+"\n" + #print(a) + return a class Layer: def __init__(self, num_nodes, num_inputs): """Initialises the Layer object num_nodes is the number of Nodes to be in the layer - num_inputs is the number of outputs from the previous layer""" + num_inputs is the number of outputs from the previous layer - self.w = np.matrix(np.random.rand(num_nodes, num_inputs)) + The weights w will be of size [num_nodes x num_inputs] + The biases b will be of size [num_nodes x 1]""" + a = 10 + self.num_nodes = num_nodes + self.num_inputs = num_inputs + self.w = np.matrix(a*(np.random.normal(0,1,size = [num_nodes, num_inputs]))) self.w_to_update = np.zeros_like(self.w) - self.b = np.matrix.transpose(np.matrix(np.random.rand(num_nodes))) + self.b = np.matrix.transpose(a*(np.matrix(np.random.normal(0,1,num_nodes)))) self.b_to_update = np.zeros_like(self.b) ''' self.nodes = [0 for i in range(num_nodes)] @@ -115,7 +135,8 @@ def sim(self, inputs): z = self.w*inputs+self.b - outputs = sigmoid(z) + outputs = np.tanh(z) + # outputs = z ''' for i in self.nodes: outputs.append(i.sim(inputs)) @@ -124,7 +145,12 @@ def sim(self, inputs): def update_parameters(self): """Updates the weights and biases of the Layer""" self.w = np.matrix(self.w_to_update) - self.b = np.matrix(self.b_to_update) + self.b = np.matrix(self.b_to_update) + + def __str__(self): + """Returns the details of this layer""" + + return "Layer has %s inputs and %s nodes, b: %s, w: %s" % (self.num_inputs, self.num_nodes, str(self.b), str(self.w)) def sigmoid(z): #output = sp.logistic.cdf(z) @@ -136,29 +162,5 @@ def cost(simulated_outputs, actual_outputs): if len(simulated_outputs.shape) == 1: cost_value = 1.0/(2.0*n) * np.sum(np.subtract(simulated_outputs,actual_outputs)**2) else: - cost_value = 1.0/(2.0*n) * np.sum(np.linalg.norm(np.subtract(simulated_outputs,actual_outputs), axis=0)**2) + cost_value = 1.0/(2.0*n) * np.sum(np.linalg.norm(np.subtract(simulated_outputs,actual_outputs), axis=0)) return cost_value - - - -# Code to test the cost function -#print cost(np.array([[3, 8, 9, 12]]).T, np.array([[4, 9, 7, 20]]).T) - - -# Code to test the network creation -# Fairly simple network - -net = Network([5,2],4) -training_input = np.matrix([[1, 1, 3], [1, 2, 6], [4, 6, 2], [2, 8, 4]]) -training_output = np.matrix([[1, 0, 0.5], [0, 1, 0.8]]) - -# Simple 1-node network -''' -net = Network([1],1) -training_input = np.array([1]) -training_output = np.array([0.6]) -''' - - -net.train(training_input, training_output, 1000) -print np.concatenate([net.sim(training_input[:,n]) for n in range(np.size(training_input, 1))], 1) diff --git a/src/basic_test.py b/src/basic_test.py new file mode 100644 index 0000000..1ce500e --- /dev/null +++ b/src/basic_test.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Apr 04 15:09:05 2018 + +@author: richardf +""" +import cPickle, gzip, numpy as np, matplotlib.pyplot as plt, time + + +# Load the MNIST data +f = gzip.open('../mnist.pkl.gz', 'rb') +train_set, valid_set, test_set = cPickle.load(f) +f.close() + +plt.ion() +fig = plt.figure() + +#ax = fig.add_subplot(111) + +# Display an image +for k in range(0,10): + pixels = train_set[0][k].reshape(28,28) + # pixels = np.array(pixels, dtype='uint8').reshape(28,28) + plt.imshow(pixels,cmap='gray') + plt.show() + fig.canvas.draw() + print("Updated") + time.sleep(0.25) + \ No newline at end of file diff --git a/src/mnist_utils.py b/src/mnist_utils.py new file mode 100644 index 0000000..86768f1 --- /dev/null +++ b/src/mnist_utils.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Apr 04 15:45:08 2018 + +@author: richardf +""" + +def load_mnist(): + # Load the MNIST data + f = gzip.open('../mnist.pkl.gz', 'rb') + train_set, valid_set, test_set = cPickle.load(f) + f.close() + return train_set, valid_set, test_set \ No newline at end of file diff --git a/src/test_script.py b/src/test_script.py new file mode 100644 index 0000000..acdde6b --- /dev/null +++ b/src/test_script.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Sep 04 13:09:52 2017 + +@author: richardf +""" + +import NeuralNet as nn +import numpy as np +import matplotlib.pyplot as plt + +# Code to test the cost function +#print cost(np.array([[3, 8, 9, 12]]).T, np.array([[4, 9, 7, 20]]).T) + + +# Code to test the network creation +# Fairly simple network + +net = nn.Network([5,2],4) +training_input = np.matrix([[1, 1, 3], [1, 2, 6], [4, 6, 2], [2, 8, 4]]) +training_output = np.matrix([[1, 0, 0.5], [0, 1, 0.8]]) + +# Simple 1-node network +''' +net = Network([1],1) +training_input = np.array([1]) +training_output = np.array([0.6]) +''' + + +net.train(training_input, training_output, 1000) +print np.concatenate([net.sim(training_input[:,n]) for n in range(np.size(training_input, 1))], 1) + +#%% +''' +Create a function-learning NN +''' + +reload(nn) + +# function: y = sin(x)*cos(x)+x**3 +x = np.arange(0,1,0.01) +y = 0.25*(np.sin(7*x)*np.cos(10*x)+(x)**3+1) + +# fig, ax = plt.subplots() +# ax.cla() +# plt.figure() +plt.ion() +fig,ax = plt.subplots(1,1) +ax.plot(x,y) + +training_input = np.matrix(x) +training_output = np.matrix(y) + +net = nn.Network([20,10,1],1) + + + +plt.hold(True) + +for i in range(2): + net.train(training_input, training_output, 10) + + # Plot the current approximation + out = net.sim(training_input) + + ax.plot(x,out.T) + plt.draw() + +#%% +net = nn.Network([20,10, 6, 30, 3,1],1) + +reload(nn) + +#a = 50 +#net.layers[0].w[0] = a*(np.random.rand(1)-0.5) +#net.layers[0].w[1] = a*(np.random.rand(1)-0.5) +#net.layers[0].w[2] = a*(np.random.rand(1)-0.5) +#net.layers[0].w[3] = a*(np.random.rand(1)-0.5) +#net.layers[0].w[4] = a*(np.random.rand(1)-0.5) + +#net.layers[0].b[0] = a*(np.random.rand(1)-0.5) +#net.layers[0].b[1] = a*(np.random.rand(1)-0.5) +#net.layers[0].b[2] = a*(np.random.rand(1)-0.5) +#net.layers[0].b[3] = a*(np.random.rand(1)-0.5) +#net.layers[0].b[4] = a*(np.random.rand(1)-0.5) + +#net.layers[1].w = a*(np.random.rand(1,5)-0.5) +#net.layers[1].b = a*(np.random.rand(1)-0.5) +plt.cla() +plt.plot(x,net.sim(x).T) diff --git a/src/test_sin_function.py b/src/test_sin_function.py new file mode 100644 index 0000000..da5837d --- /dev/null +++ b/src/test_sin_function.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Apr 04 15:47:23 2018 + +@author: richardf +""" +import math, numpy as np, NeuralNet as nn, matplotlib.pyplot as plt + +# Test whether the NN can learn a sin function +train_inputs = np.random.rand(1,1000) +train_outputs = 0.5*np.sin(train_inputs*2*math.pi)+0.5 + +test_inputs = np.random.rand(1,10) +test_outputs = 0.5*np.sin(test_inputs*2*math.pi)+0.5 + +new_inputs = [np.arange(0,1,0.01)] + +# Create the NN +net = nn.Network([20,5,1],1) + +print(net.sim([[0]])) + + +print(net) + +net.train(train_inputs, train_outputs, 500) + +new_outputs = net.sim(new_inputs) + +plt.plot(np.transpose(new_inputs), np.transpose(new_outputs)) +plt.plot(np.transpose(train_inputs), np.transpose(train_outputs),'x') \ No newline at end of file diff --git a/src/test_single_target_function.py b/src/test_single_target_function.py new file mode 100644 index 0000000..835946e --- /dev/null +++ b/src/test_single_target_function.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Apr 13 14:38:55 2018 + +@author: richardf +""" + +import math, numpy as np, NeuralNet as nn, matplotlib.pyplot as plt + +# Test whether the NN can learn a single target function +train_inputs = np.arange(0,1.001,0.5) +#train_outputs = -(train_inputs-0.5)**(2)/10.0+0.5 +#train_outputs = train_inputs*(1.0) +train_outputs = np.abs(train_inputs -0.5) + +new_inputs = [np.arange(0,1,0.01)] + +# Create the NN +net = nn.Network([5,5,1],1) + +print(net.sim([[0]])) + + +print(net) + +net.train(train_inputs, train_outputs, 500) + +new_outputs = net.sim(new_inputs) + +plt.plot(np.transpose(new_inputs), np.transpose(new_outputs)) +plt.plot(train_inputs, train_outputs) +plt.legend(('Predictions','Training data')) + +#%% Try create a network + +net = nn.Network([2,1],1) + +#net.layers[0].b[0] = -0.8 +#net.layers[0].b[1] = 0 +#net.layers[0].w[0] = -10 +#net.layers[0].w[1] = 1 + +#net.layers[1].b[0] = 0 +#net.layers[1].w[0,0] = 1 +#net.layers[1].w[0,1] = 1 + +plt.plot(np.transpose(new_inputs), np.transpose(net.sim(new_inputs))) +#plt.plot(train_inputs, train_outputs) +plt.legend(('Predictions','Training data')) \ No newline at end of file