上面一章的 `update_mini_batch` 函数是每个数据集独立计算后向传播，后面再做平均值，
def update_mini_batch(self, mini_batch, eta):
"""Update the network's weights and biases by applying
gradient descent using backpropagation to a single mini batch.
The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta``
is the learning rate."""
# 初始化梯度值矩阵为 0
# Nabla算子，在中文中也叫向量微分算子、劈形算子、倒三角算子
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
for x, y in mini_batch:
# 迭代计算梯度矩阵和
# 获取当前样本通过反向传播算法得到的 delta 梯度值
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
# 把 mini_batch 里面每个数据算出来的梯度做加和，后面再取平均
nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
# 把梯度值取平均，并乘以系数 eta，然后更新权重和偏置矩阵
self.weights = [w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]
self.biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)]
这里介绍一下如何通过矩阵的方式直接计算一个 mini batch 的梯度值向量。代码来源：https://github.com/hindSellouk/Matrix-Based-Backpropagation/blob/master/Network1.py
import random
from os import access
import numpy as np
import time
class Network(object):
def __init__(self, sizes):
self.num_layers = len(sizes)
self.sizes = sizes
self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
self.weights = [np.random.randn(y, x)
for x, y in zip(sizes[:-1], sizes[1:])]
def SGD(self, training_data, epochs, mini_batch_size, eta,
test_data=None):
if test_data: n_test = len(test_data)
n = len(training_data)
for j in xrange(epochs):
start_time = time.time()
random.shuffle(training_data)
mini_batches = [
training_data[k:k+mini_batch_size]
for k in xrange(0, n, mini_batch_size)]
for mini_batch in mini_batches:
self.update_mini_batch(mini_batch, eta)
print("--- %s seconds elapsed---" % (time.time() - start_time))
if test_data:
print "Epoch {0}: {1} / {2}".format(
j, self.evaluate(test_data), n_test)
else:
print "Epoch {0} complete".format(j)
def update_mini_batch(self, mini_batch, eta):
# mini batch 是一个 list，list 里面每个元素是一个 tuple (x, y)
matrix_X=mini_batch[0][0]
matrix_Y=mini_batch[0][1]
#create matrix_X of examples and a matrix_Y of labels
for x,y in mini_batch[1:]:
# 将 mini batch 里面的每个数据拼接起来，成一个 matrix
matrix_X = np.concatenate((matrix_X,x), axis=1)
matrix_Y = np.concatenate((matrix_Y,y), axis=1)
nabla_b, nabla_w = self.backprop(matrix_X,matrix_Y)
# 下面将返回的权重和偏置梯度值做平均
self.weights = [w - (eta / len(mini_batch)) * nw
for w, nw in zip(self.weights, nabla_w)]
self.biases = [b - (eta / len(mini_batch)) * nb
for b, nb in zip(self.biases, nabla_b) ]
def backprop(self, x, y):
# 这里 x 和 y 就都是一个 matrix，包括一个 batch 大小的数据
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
# feedforward
activation = x
activations = [x] # list to store all the activation matrices, layer by layer
zs = [] # list to store all the "sum of weighted inputs z" matrices, layer by layer
i=0
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation) + b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
# 这里 delta 就是一个矩阵，如果 batch 为 10 大小的话
# ('backprop delta', (10, 10), <type 'numpy.ndarray'>)
delta = self.cost_derivative(activations[-1], y) * \
sigmoid_prime(zs[-1])
# 这里先将上面的 delta 矩阵做了加和，变成了
# (10,), <type 'numpy.ndarray'>)
# 然后做了扩展，变成如下
# ('nabla_b[-1]', (10, 1), <type 'numpy.ndarray'>)
nabla_b[-1] = np.expand_dims(np.sum(delta,axis=1),axis=1)
# 最后一层的权重矩阵
# ('nabla_w[-1]', (10, 30), <type 'numpy.ndarray'>)
# 这里 np.dot 将 batch 的所有权重做了线性和
nabla_w[-1] = np.dot(delta, activations[-2].transpose())
for l in xrange(2, self.num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
# 10 * 10
delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp
# 这里 np.sum 将 batch 的所有误差做了线性和
nabla_b[-l] = np.expand_dims(np.sum(delta,axis=1),axis=1)
# 这里 np.dot 将 batch 的所有权重做了线性和
# 原来 delta 是 n * 1, activations[-l - 1].transpose() 是 1 * m
# batch 的情况下，delta 是 n * batch size, activations[-l - 1].transpose() 是 batch size * m
nabla_w[-l] = np.dot(delta, activations[-l - 1].transpose())
return (nabla_b, nabla_w)
def feedforward(self, a):
for b, w in zip(self.biases, self.weights):
a = sigmoid(np.dot(w, a)+b)
return a
def evaluate(self, test_data):
test_results = [(np.argmax(self.feedforward(x)), y)
for (x, y) in test_data]
return sum(int(x == y) for (x, y) in test_results)
def cost_derivative(self, output_activations, y):
return (output_activations-y)
def sigmoid(z):
return 1.0/(1.0+np.exp(-z))
def sigmoid_prime(z):
return sigmoid(z)*(1-sigmoid(z))
执行代码：
import mnist_loader
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
import network1
net = network1.Network([784, 30, 10])
from datetime import datetime as datetime
print(datetime.now())
net.SGD(training_data, 30, 10, 3.0, test_data=test_data);
print(datetime.now())
基于矩阵运算的算法一个 Epoch 大约为 3 秒:
--- 3.12179398537 seconds elapsed---
Epoch 0: 9061 / 10000
--- 3.03969407082 seconds elapsed---
Epoch 1: 9177 / 10000
--- 3.00676393509 seconds elapsed---
Epoch 2: 9261 / 10000
不基于矩阵运算的算法一个 Epoch 大约为 11 秒:
--- 11.0207378864 seconds elapsed---
Epoch 0: 9079 / 10000
--- 11.092361927 seconds elapsed---
Epoch 1: 9247 / 10000
--- 11.0678529739 seconds elapsed---
Epoch 2: 9326 / 10000