一个神经网络的基础实现

    今天状态不太好,写了一个基础神经网络,可能有写错和写的不好的地方

    用异或网络训练后,发现只有tanh函数成功了,sigmoid函数和relu失败了,tanh建立三层以上网络也预测失败了

    可能是tanh的收敛速度比较快的缘故

    只考虑了全连接层,没有使用任何优化训练的方法,卷积神经网络应该只是连接方式不同(更加稀疏),目前还不想写也写不出来,试验正确性也挺麻烦的,以后可能理解深刻了再写

import numpy as np
import pandas as pd
import copy
def tanh(x):
    return np.tanh(x)
def tanh_derivative(x):
    return 1.0 - np.tanh(x) * np.tanh(x)
def sigmoid(x):
    return 1 / (1 + np.exp(-x))
def sigmoid_derivative(x):
    return sigmoid(x) * (1 - sigmoid(x))
def relu(x):
	t = copy.copy(x)
	for i in range(len(t)):
		if t[i] < 0:
			t[i] = 0
	return t

def relu_derivative(x):
	t = copy.copy(x)
	for i in range(len(t)):
		if t[i] < 0:
			t[i] = 0
		else:
			t[i] = 1
	return t

class ActivationFunc:
	def __init__(self):
		self.tdict = dict()
		self.tdict['tanh'] = np.tanh
		self.tdict['sigmoid'] = lambda x: 1 / (1 + np.exp(-x))
		self.tdict['relu'] = relu
		self.tdict['softmax'] = np.exp
		self.ddict = dict()
		self.ddict['tanh'] = tanh_derivative
		self.ddict['sigmoid'] = sigmoid_derivative
		self.ddict['relu'] = relu_derivative
		self.ddict['softmax'] = np.exp

	def getActivation(self, activation):
		if activation in self.tdict:
			return self.tdict[activation]
		else:
			return self.tdict['relu']

	def getDActivation(self, activation):
		if activation in self.ddict:
			return self.ddict[activation]
		else:
			return self.ddict['relu']
	

#print(ActivationFunc().getActivation('logistic')(1.0))
#print(logistic_derivative(1.0))
class NNetwork:
	def __init__(self, inputsize, lr = 0.01, withbias = True) :
		self.para = []
		self.layerout = []
		self.grad = []
		self.backout = []
		
		self.activationclass = ActivationFunc()
		self.inputsize = inputsize
		self.lastsize = inputsize
		self.lr = lr
		self.layerlen = 0
		self.activation = []
		self.deactivation = []
		self.wbias = withbias
		self.outputfunc = 'softmax'
		self.maxnum = 0
		#self.activation = ActivationFunc().getActivation(mactivation)
		
	
	def add(self, densesize, actstr):
		tsize = self.lastsize
		if self.wbias:
			tsize += 1

		self.para.append(np.random.rand(densesize, tsize) - 0.5)
		self.grad.append(np.zeros((densesize, tsize)))
		
		self.lastsize = densesize
		self.activation.append(self.activationclass.tdict[actstr])
		self.deactivation.append(self.activationclass.ddict[actstr])
		self.layerlen += 1
		self.outputfunc = actstr

	def forward(self,input):
		self.layerout = []
		if self.wbias:
			self.layerout.append(np.append(np.array(input),1))
		else:
			self.layerout.append(np.array(input))
		for i in range(self.layerlen):
			#print(self.layerout[-1].shape, self.para[i].shape)
			
			if self.wbias and i != self.layerlen - 1:
				self.layerout.append(np.append(self.activation[i](np.dot(self.para[i], self.layerout[-1].T)), 1))
			else:
				self.layerout.append(self.activation[i](np.dot(self.para[i], self.layerout[-1].T)))
		return self.layerout[-1]

		
	def backward(self, y, y_label):
		self.maxnum = 0.001

		tsumy = sum(y)
		y[y_label] -= tsumy
		
		self.maxnum = max(self.maxnum, max(y))
		self.backout = []
		self.backout.append(np.matrix(y).T)

		for i in range(self.layerlen, 0, -1):
			#print(self.backout[-1].shape, np.matrix(self.layerout[i - 1]).shape)
			self.grad[i - 1] += np.dot(self.backout[-1], np.matrix(self.layerout[i - 1]))
			self.maxnum = max(self.grad[i - 1].max().max(), self.maxnum)
			if i > 1:
				if self.wbias:
					self.backout.append(np.multiply(self.deactivation[i - 2](self.layerout[i - 1]), np.dot(self.backout[-1].T, self.para[i - 1])).T[:-1,:])
				else:
					self.backout.append(np.multiply(self.deactivation[i - 2](self.layerout[i - 1]), np.dot(self.backout[-1].T, self.para[i - 1])).T)

	def zero_grad(self):
		for obj in self.grad:
			obj.fill(0)
	
	def step(self):
		for obj1, obj2 in zip(self.para, self.grad):
			obj1 -= self.lr /self.maxnum * obj2
	
	def predict(self, input):
		y = self.forward(input)
		y /= np.sum(y)
		return y



#2*x + y - 3
model = NNetwork(2, withbias = True)
#model.add(16, 'relu')
model.add(16, 'tanh')
model.add(4, 'tanh')
model.add(2, 'softmax')
X = [[0,0],[0,1],[1,0],[1,1]]
y = [0, 1, 1, 0]

for i in range(1000000):
	tid = i % 4
	model.zero_grad()
	output = model.forward(X[tid])
	model.backward(output, y[tid])
	model.step()

print(model.predict([1,1]))
print(model.predict([0,1]))
print(model.predict([0,0]))
print(model.predict([1,0]))

发现写错了,激活函数对输入求导写成对输出求导了,突然发现sigmoid,relu,tanh这三个函数求导都可以写成输出的函数,而且更加简单,比如y = tanh(x) , dy = (1 - y * y)dx,比dy = (1 - tanh(x) * tanh(x)) dx更加简洁

import numpy as np
import pandas as pd
import copy
def tanh(x):
    return np.tanh(x)
def tanh_derivative(x):
    return 1.0 - x * x
def sigmoid(x):
    return 1 / (1 + np.exp(-x))
def sigmoid_derivative(x):
    return x * (1 - x)
def relu(x):
	return np.maximum(x, 0)
	#t = copy.copy(x)
	#for i in range(len(t)):
	#	if t[i] < 0:
	#		t[i] = 0
	#return t

def relu_derivative(x):
	t = copy.copy(x)
	for i in range(len(t)):
		if t[i] <= (1e-12):
			t[i] = 0
		else:
			t[i] = 1
	return t

class ActivationFunc:
	def __init__(self):
		self.tdict = dict()
		self.tdict['tanh'] = np.tanh
		self.tdict['sigmoid'] = lambda x: 1 / (1 + np.exp(-x))
		self.tdict['relu'] = relu
		self.tdict['softmax'] = np.exp
		self.ddict = dict()
		self.ddict['tanh'] = tanh_derivative
		self.ddict['sigmoid'] = sigmoid_derivative
		self.ddict['relu'] = relu_derivative
		self.ddict['softmax'] = np.exp

	def getActivation(self, activation):
		if activation in self.tdict:
			return self.tdict[activation]
		else:
			return self.tdict['relu']

	def getDActivation(self, activation):
		if activation in self.ddict:
			return self.ddict[activation]
		else:
			return self.ddict['relu']
	

#print(ActivationFunc().getActivation('logistic')(1.0))
#print(logistic_derivative(1.0))
class NNetwork:
	def __init__(self, inputsize, lr = 0.01, withbias = True) :
		self.para = []
		self.layerout = []
		self.grad = []
		self.backout = []
		
		self.activationclass = ActivationFunc()
		self.inputsize = inputsize
		self.lastsize = inputsize
		self.lr = lr
		self.layerlen = 0
		self.activation = []
		self.deactivation = []
		self.wbias = withbias
		self.outputfunc = 'softmax'
		self.maxnum = 0.001
		#self.activation = ActivationFunc().getActivation(mactivation)
		
	
	def add(self, densesize, actstr):
		tsize = self.lastsize
		if self.wbias:
			tsize += 1

		self.para.append(np.random.rand(densesize, tsize) - 0.5)
		self.grad.append(np.zeros((densesize, tsize)))
		
		self.lastsize = densesize
		self.activation.append(self.activationclass.tdict[actstr])
		self.deactivation.append(self.activationclass.ddict[actstr])
		self.layerlen += 1
		self.outputfunc = actstr

	def forward(self,input):
		self.layerout = []
		if self.wbias:
			self.layerout.append(np.append(np.array(input),1))
		else:
			self.layerout.append(np.array(input))
		for i in range(self.layerlen):
			#print(self.layerout[-1].shape, self.para[i].shape)
			
			if self.wbias and i != self.layerlen - 1:
				self.layerout.append(np.append(self.activation[i](np.dot(self.para[i], self.layerout[-1].T)), 1))
			else:
				self.layerout.append(self.activation[i](np.dot(self.para[i], self.layerout[-1].T)))
		return self.layerout[-1]

		
	def backward(self, y, y_label):
		self.maxnum = 0.001
		tsumy = sum(y)
		y[y_label] -= tsumy
		
		#self.maxnum = max(self.maxnum, max(y))
		self.backout = []
		self.backout.append(np.matrix(y).T)

		for i in range(self.layerlen, 0, -1):
			#print(self.backout[-1].shape, np.matrix(self.layerout[i - 1]).shape)
			self.grad[i - 1] += np.dot(self.backout[-1], np.matrix(self.layerout[i - 1]))
			self.maxnum = max(np.abs(self.grad[i - 1]).max().max(), self.maxnum)
			if i > 1:
				if self.wbias:
					self.backout.append(np.multiply(self.deactivation[i - 2](self.layerout[i - 1]), np.dot(self.backout[-1].T, self.para[i - 1])).T[:-1,:])
				else:
					self.backout.append(np.multiply(self.deactivation[i - 2](self.layerout[i - 1]), np.dot(self.backout[-1].T, self.para[i - 1])).T)

	def zero_grad(self):
		for obj in self.grad:
			obj.fill(0)
		self.maxnum = 0.001
	def step(self):
		for obj1, obj2 in zip(self.para, self.grad):
			obj1 -= self.lr * obj2 / self.maxnum 
		self.zero_grad()

	def predict(self, input):
		y = self.forward(input)
		y /= np.sum(y)
		return y



model = NNetwork(2, withbias = True)
#model.add(16, 'relu')
model.add(16, 'relu')
model.add(8, 'relu')
model.add(4, 'relu')
model.add(2, 'softmax')
X = [[0,0],[0,1],[1,0],[1,1]]
y = [0, 1, 1, 0]

for i in range(100000):
	tid = i % 4
	#model.zero_grad()
	output = model.forward(X[tid])
	model.backward(output, y[tid])
	if tid == 3: 
		model.step()


print(model.predict([1,1]))
print(model.predict([0,1]))
print(model.predict([0,0]))
print(model.predict([1,0]))

亲测relu也可以成功预测了,sigmoid只能使用一层,使用多层会出错,可能是梯度消失,relu和tanh使用多少层都没事