神经网络3：多隐层感知器

▶ 终于把之前的单隐层感知机改成了支持任意层数神经元的网络

● 代码，参考【https://www.zybuluo.com/hanbingtao/note/476663】（原文错误太多，几乎是重写了）。采用了 class FullConnectedLayer 封装同一层的多个神经元，采用 class Network 封装所有神经元层，包含了相应的前向、后向等计算方法

  1 import numpy as np
  2 from functools import reduce
  3 import matplotlib.pyplot as plt
  4 import matplotlib.ticker as ticker
  5 from mpl_toolkits.mplot3d import Axes3D
  6 from mpl_toolkits.mplot3d.art3d import Poly3DCollection
  7 from matplotlib.patches import Rectangle
  8 from datetime import datetime as dt
  9 
 10 global_dataSize = 1000                              # 总数据大小
 11 global_trainRatio = 0.3                             # 训练集占比
 12 global_ita = 0.4                                    # 学习率
 13 global_epsilon = 0.01                               # 单轮累计误差要求
 14 global_maxTurn = 300                                # 最大训练轮数
 15 
 16 def dataSplit(data, part):                          # 将数据集分割
 17     return data[0:part,:],data[part:,:]
 18 
 19 def myColor(x):                                     # 颜色函数，用于对散点染色
 20     r = np.select([x < 1/2, x < 3/4, x <= 1, True],[0, 4 * x - 2, 1, 0])
 21     g = np.select([x < 1/4, x < 3/4, x <= 1, True],[4 * x, 1, 4 - 4 * x, 0])
 22     b = np.select([x < 1/4, x < 1/2, x <= 1, True],[1, 2 - 4 * x, 0, 0])
 23     return [r,g,b]
 24 
 25 class Activator(object):                            # 激活函数类，采用 sigmoid 函数
 26     def σ(self, x):
 27         return 1.0 / (1.0 + np.exp(-x))
 28     def dσ(self, x):
 29         return x * (1.0 - x)                        # 注意输入的 x 是原神经元的输出值而不是激活前的值
 30 
 31 def createData(dimIn, dimOut, len, linear):         # 生成测试数据
 32     np.random.seed(103)
 33     inputData = np.random.rand(len, dimIn)
 34     if linear:
 35         transform = np.random.rand(dimIn, dimOut)   # 线性变换
 36         outputData = np.matmul(inputData,transform) # 线性变换
 37         #print(transform)                           # 输出变换矩阵
 38     else:
 39         outputData = np.array(list( map(lambda x: [ np.sum(x**k) for k in range(1,dimOut+1) ], inputData) ))    # 非线性变换
 40     return inputData, outputData
 41 
 42 class FullConnectedLayer(object):                               # 全连接层类
 43     def __init__(self, dimIn, dimOut, activator):               # 构造函数
 44         self.dimIn = dimIn                                      # 本层输入向量的维度
 45         self.dimOut = dimOut                                    # 本层输出向量的维度
 46         self.activator = activator                              # 激活函数
 47         self.W = np.random.uniform(-0.1, 0.1,(dimOut, dimIn))   # 权重数组
 48         self.b = np.zeros(dimOut)                               # 偏置项
 49         self.output = np.zeros((dimOut, 1))                     # 输出向量
 50 
 51     def forward(self, xArray):                                  # 前向计算
 52         self.input = xArray
 53         self.output = self.activator.σ(np.dot(self.W, xArray) + self.b)
 54         return self.output
 55 
 56     def backward(self, wMultiplyDeltaNextLayer, isLastLayer):   # 反向计算，输入上一层的误差项
 57         if isLastLayer:
 58             self.delta = self.activator.dσ(self.output) * wMultiplyDeltaNextLayer
 59         else:
 60             self.delta = self.activator.dσ(self.output) * wMultiplyDeltaNextLayer
 61         self.W_grad = np.matmul(np.mat(self.delta).T, np.mat(self.input))
 62         self.b_grad = self.delta
 63         return np.matmul(self.W.T, self.delta)                  # 返回乘积，方便上一层神经元计算 delta
 64 
 65     def update(self, ita):                                      # 梯度下降算法更新权重
 66         self.W += ita * self.W_grad
 67         self.b += ita * self.b_grad
 68         #return self.W, self.b
 69 
 70 class Network(object):                                          # 神经网络类
 71     def __init__(self, layers):
 72         self.layers = []
 73         for i in range(len(layers) - 1):
 74             self.layers.append(FullConnectedLayer(layers[i], layers[i+1], Activator()))
 75 
 76     def predict(self, sample):                                  # 计算单个样本的结果
 77         temp = sample
 78         for layer in self.layers:
 79             temp = layer.forward(temp)
 80         return temp
 81 
 82     def train(self, x, y, ita, maxTurn, enablePrint):
 83         row, dimOut = np.shape(y)
 84         errorTable = np.zeros([row,dimOut])
 85         for turn in range(maxTurn):
 86             for i in range(len(x)):
 87                 res = self.predict(x[i])                            # 当前样本在当前网络上的结果，副作用更新各 layer 数据
 88                 errorTable[i] = y[i] - res
 89                 res = self.layers[-1].backward(errorTable[i], True) # 最后一层神经元的 delta
 90                 for layer in self.layers[-2::-1]:                   # 倒着计算各层 delta
 91                     res = layer.backward(res, False)
 92                 for layer in self.layers:                           # 正着调整各层权值
 93                     layer.update(ita)
 94             errorRatio = np.sum(errorTable**2) / (row * dimOut)
 95             if enablePrint and (turn == 0 or turn == maxTurn-1 or errorRatio < global_epsilon):
 96                 print( '%s turn %3d, errorRatio = %f' % (dt.now(), turn, errorRatio) )
 97             if errorRatio < global_epsilon:
 98                 break
 99 
100 def test(dimArray, linear = True, dataSize = global_dataSize, trainRatio = global_trainRatio, ita = global_ita, maxTurn = global_maxTurn): # 测试函数，给定每层神经元个数
101     allInData, allOutData = createData(dimArray[0], dimArray[-1], dataSize, linear) # 创建数据
102     trainInData, testInData = dataSplit(allInData, int(dataSize * trainRatio))
103     trainOutData, testOutData = dataSplit(allOutData, int(dataSize * trainRatio))
104 
105     network = Network(dimArray)                                                     # 创建神经网络
106 
107     network.train(trainInData, trainOutData, ita, maxTurn, True)                    # 训练神经网络
108 
109     myResult = [ network.predict(sample) for sample in testInData ]                 # 在测试集上计算结果
110 
111     errorTable = np.sum((np.array(myResult) - testOutData)**2, 1)
112     errorMax = np.max(errorTable)
113     errorRatio = np.sum(errorTable) / (dataSize*(1 - trainRatio) * dimArray[-1])    # 计算测试集错误率
114     print( "dimArray = %s, errorRatio = %f\n" % (dimArray, round(errorRatio,4)) )
115 
116     if dimArray[0] >= 4:                                                            # 画图部分，4维以上不画图，只输出测试错误率
117         return
118 
119     fig = plt.figure(figsize=(10, 8))
120     cm = plt.cm.get_cmap('gist_rainbow')
121 
122     if dimArray[0] == 1:
123         plt.xlim(0.0,1.0)
124         plt.ylim(-0.15,0.15)
125         plt.scatter(testInData,np.zeros(len(testInData)), facecolors='none',c = errorTable, vmin = 0, vmax = max(errorTable),cmap = cm,s = 10)
126         plt.colorbar(ticks = np.linspace(0, max(errorTable), 5))
127         plt.text(0.8, 0.1, "errorRatio = " + str(round(errorRatio,4)) + "\nmaxError^2 = " + str(round(errorMax,3)), \
128             size = 15, ha="center", va="center", bbox=dict(boxstyle="round", ec=(1., 0.5, 0.5), fc=(1., 1., 1.)))
129 
130     if dimArray[0] == 2:
131         plt.xlim(0.0,1.0)
132         plt.ylim(0.0,1.0)
133         plt.scatter(testInData[:,0],testInData[:,1], facecolors='none',c = errorTable, vmin = 0, vmax = max(errorTable),cmap = cm,s = 10)
134         plt.colorbar(ticks = np.linspace(0, max(errorTable), 5))
135         plt.text(0.2, 0.95, "errorRatio = " + str(round(errorRatio,4)) + "\nmaxError^2 = " + str(round(errorMax,3)), \
136             size = 15, ha="center", va="center", bbox=dict(boxstyle="round", ec=(1., 0.5, 0.5), fc=(1., 1., 1.)))
137 
138     if dimArray[0] == 3:
139         ax = Axes3D(fig)
140         ax.set_xlim3d(0.0, 1.0)
141         ax.set_ylim3d(0.0, 1.0)
142         ax.set_zlim3d(0.0, 1.0)
143         ax.set_xlabel('X', fontdict={'size': 15, 'color': 'k'})
144         ax.set_ylabel('Y', fontdict={'size': 15, 'color': 'k'})
145         ax.set_zlabel('Z', fontdict={'size': 15, 'color': 'k'})
146         sc = ax.scatter(testInData[:,0],testInData[:,1], testInData[:,2], c = errorTable, vmin = 0, vmax = max(errorTable),cmap = cm, s = 10)
147         plt.colorbar(sc, cax = fig.add_axes([0.05,0.1,0.02,0.8]), ticks = np.linspace(0, max(errorTable), 5))
148         ax.text3D(0.8, 1, 1.15, "errorRatio = " + str(round(errorRatio,4)) + "\nmaxError^2 = " + str(round(errorMax,3)), \
149             size = 12, ha="center", va="center", bbox=dict(boxstyle="round", ec=(1, 0.5, 0.5), fc=(1, 1, 1)))
150 
151     fig.savefig("R:\\dimIn" + str(dimArray) + str(linear) + ".png")
152     plt.close()
153 
154 if __name__ == '__main__':
155     test((1,3,1), True)
156     test((1,3,1), False)
157 
158     test((2,3,2), True)
159     test((2,3,2), False)
160     test((2,10,2), False)
161 
162     test((3,3,2), True)
163     test((3,10,2), True)
164     test((3,3,2), False)
165     test((3,10,2), False)
166 
167     test((4,3,2), True)
168     test((4,3,2), False)
169 
170     test((2,4,3,2), False)
171     test((3,4,3,2), True)
172     test((3,4,3,2), False)
173     test((4,6,3,3,2), True)
174     test((4,6,3,3,2), False)

● 输出结果

2019-09-21 17:29:34.113820 turn   0, errorRatio = 0.016535
2019-09-21 17:29:34.811177 turn   8, errorRatio = 0.009880
dimArray = (1, 3, 1), errorRatio = 0.010400

2019-09-21 17:29:35.501720 turn   0, errorRatio = 0.080407
2019-09-21 17:29:36.152499 turn   8, errorRatio = 0.007243
dimArray = (1, 3, 1), errorRatio = 0.005900

2019-09-21 17:29:36.666430 turn   0, errorRatio = 0.033850
2019-09-21 17:29:37.292406 turn   8, errorRatio = 0.007348
dimArray = (2, 3, 2), errorRatio = 0.005600

2019-09-21 17:29:37.836257 turn   0, errorRatio = 0.176799
2019-09-21 17:30:01.532073 turn 299, errorRatio = 0.054540
dimArray = (2, 3, 2), errorRatio = 0.057900

2019-09-21 17:30:02.095402 turn   0, errorRatio = 0.174731
2019-09-21 17:30:26.013079 turn 299, errorRatio = 0.054420
dimArray = (2, 10, 2), errorRatio = 0.057800

2019-09-21 17:30:26.522060 turn   0, errorRatio = 0.108589
2019-09-21 17:30:50.394160 turn 299, errorRatio = 0.041377
dimArray = (3, 3, 2), errorRatio = 0.046000

2019-09-21 17:30:51.152020 turn   0, errorRatio = 0.105360
2019-09-21 17:31:14.782947 turn 299, errorRatio = 0.041334
dimArray = (3, 10, 2), errorRatio = 0.046000

2019-09-21 17:31:15.476451 turn   0, errorRatio = 0.381080
2019-09-21 17:31:39.135285 turn 299, errorRatio = 0.300513
dimArray = (3, 3, 2), errorRatio = 0.320200

2019-09-21 17:31:39.871999 turn   0, errorRatio = 0.372009
2019-09-21 17:32:03.424233 turn 299, errorRatio = 0.300301
dimArray = (3, 10, 2), errorRatio = 0.320200

2019-09-21 17:32:04.101851 turn   0, errorRatio = 0.277614
2019-09-21 17:32:27.685676 turn 299, errorRatio = 0.228789
dimArray = (4, 3, 2), errorRatio = 0.235300

2019-09-21 17:32:27.822091 turn   0, errorRatio = 0.933335
2019-09-21 17:32:51.404807 turn 299, errorRatio = 0.893136
dimArray = (4, 3, 2), errorRatio = 0.914600

2019-09-21 17:32:51.572806 turn   0, errorRatio = 0.178735
2019-09-21 17:33:26.344111 turn 299, errorRatio = 0.054565
dimArray = (2, 4, 3, 2), errorRatio = 0.057900

2019-09-21 17:33:26.901960 turn   0, errorRatio = 0.110790
2019-09-21 17:34:01.600245 turn 299, errorRatio = 0.041538
dimArray = (3, 4, 3, 2), errorRatio = 0.046300

2019-09-21 17:34:02.336221 turn   0, errorRatio = 0.384966
2019-09-21 17:34:37.024351 turn 299, errorRatio = 0.302855
dimArray = (3, 4, 3, 2), errorRatio = 0.322500

2019-09-21 17:34:37.790311 turn   0, errorRatio = 0.278739
2019-09-21 17:35:25.160092 turn 299, errorRatio = 0.256081
dimArray = (4, 6, 3, 3, 2), errorRatio = 0.262400

2019-09-21 17:35:25.385607 turn   0, errorRatio = 0.934552
2019-09-21 17:36:12.910735 turn 299, errorRatio = 0.893136
dimArray = (4, 6, 3, 3, 2), errorRatio = 0.914600