Deep learning I - III Shallow Neural Network - Backpropagation intuition反向传播算法启发

Backpropagation intuition

简单的2层浅神经网络，第一层的activation function为 $tanh(z)$ ，第二层的activation function为 $sigmoid(z)$ 。
神经网络architecture如下图：
这里写图片描述
使用计算流图(computational graphs)表示如下图：

这里写图片描述

在下面的公式中, $\log a^{[2]} \ {\rm means} \ \ln a^{[2]}$ ； ${\rm d}a^{[2]},{\rm d}z^{[2]}$ 等等是标记相应的导数的符号；并且，下面的公式是单个instance的，并没有矩阵化。

\begin{matrix} (1.1) & L (a^{[2]}, y) = - y \log a^{[2]} - (1 - y) \log (1 - a^{[2]}) \end{matrix}

${\cal L}(a^{[2]}, y) = -y\log a^{[2]}-(1-y)\log (1-a^{[2]})\tag{1.1}$

\begin{matrix} (1.2) & d a_{[1 \times 1]}^{[2]} = \frac{d}{d a^{[2]}} L (a^{[2]}, y) = - \frac{y}{a^{[2]}} + \frac{1 - y}{1 - a^{[2]}} \end{matrix}

${\rm d}a^{[2]}_{[1\times1]}=\frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) = -\frac{y}{a^{[2]}} + \frac{1-y}{1-a^{[2]}}\tag{1.2}$

\begin{matrix} (1.3) & g (z^{[2]}) = s i g m o i d (z^{[2]}) = a^{[2]} \end{matrix}

$g(z^{[2]}) = sigmoid(z^{[2]}) = a^{[2]}\tag{1.3}$

\begin{matrix} (1.4) & \begin{aligned} d z_{[1 \times 1]}^{[2]} & = \frac{d}{d z^{[2]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \\ = d a^{[2]} \cdot g^{'} (z^{[2]}) \\ = (- \frac{y}{a^{[2]}} + \frac{1 - y}{1 - a^{[2]}}) \cdot (g (z^{[2]}) (1 - g (z^{[2]}))) \\ = (- \frac{y}{a^{[2]}} + \frac{1 - y}{1 - a^{[2]}}) \cdot a^{[2]} \cdot (1 - a^{[2]}) \\ = a^{[2]} - y \end{aligned} \end{matrix}

$\begin{split}{\rm d}z^{[2]}_{[1\times1]} &= \frac{\rm d}{{\rm d}z^{[2]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \\&= {\rm d}a^{[2]} \cdot g'(z^{[2]}) \\&=(-\frac{y}{a^{[2]}} + \frac{1-y}{1-a^{[2]}}) \cdot (g(z^{[2]})(1-g(z^{[2]}))) \\&=(-\frac{y}{a^{[2]}} + \frac{1-y}{1-a^{[2]}}) \cdot a^{[2]} \cdot (1-a^{[2]}) \\& = a^{[2]} - y\end{split}\tag{1.4}$

\begin{matrix} (1.5) & \begin{aligned} d W_{[1 \times 4]}^{[2]} & = \frac{d}{d W^{[2]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d W^{[2]}} z^{[2]} \\ = d z^{[2]} \cdot x \\ = d z_{[1 \times 1]}^{[2]} (a_{[4 \times 1]}^{[1]})^{T} \end{aligned} \end{matrix}

$\begin{split}{\rm d}W^{[2]}_{[1\times4]} &= \frac{{\rm d}}{{\rm d}W^{[2]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}W ^ {[2]}}z^{[2]} \\&={\rm d}z^{[2]} \cdot x \\&= {\rm d}z^{[2]}_{[1\times1]}(a^{[1]}_{[4\times1]})^{T}\end{split}\tag{1.5}$

\begin{matrix} (1.6) & \begin{aligned} d b_{[1 \times 1]}^{[2]} & = \frac{d}{d b^{[2]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d b^{[2]}} z^{[2]} \\ = d z_{[1 \times 1]}^{[2]} \end{aligned} \end{matrix}

$\begin{split}{\rm d}b^{[2]}_{[1\times1]} &= \frac{{\rm d}}{{\rm d}b^{[2]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}b ^ {[2]}}z^{[2]} \\&={\rm d}z^{[2]}_{[1\times1]} \end{split}\tag{1.6}$

\begin{matrix} (1.7) & \begin{aligned} d a_{[4 \times 1]}^{[1]} & = \frac{d}{d a^{[1]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d a^{[1]}} z^{[2]} \\ = d z^{[2]} \cdot W^{[2]} \\ = (W_{[1 \times 4]}^{[2]})^{T} d z_{[1 \times 1]}^{[2]} \end{aligned} \end{matrix}

$\begin{split}{\rm d}a^{[1]}_{[4\times1]} &= \frac{{\rm d}}{{\rm d}a^{[1]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}a ^ {[1]}}z^{[2]} \\& ={\rm d}z^{[2]} \cdot W^{[2]}\\&=(W^{[2]}_{[1\times4]})^{T}{\rm d}z^{[2]}_{[1\times1]}\end{split}\tag{1.7}$

\begin{matrix} (1.8) & g (z^{[1]}) = \tanh (z^{[1]}) = a^{[1]} \end{matrix}

$g(z^{[1]}) = \tanh(z^{[1]}) = a^{[1]}\tag{1.8}$

\begin{matrix} (1.9) & \begin{aligned} d z_{[4 \times 1]}^{[1]} & = \frac{d}{d z^{[1]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d a^{[1]}} z^{[2]} \cdot \frac{d}{d z^{[1]}} a^{[1]} \\ = d a^{[1]} \cdot g^{'} (z^{[1]}) \\ = (W_{[1 \times 4]}^{[2]})^{T} d z_{[1 \times 1]}^{[2]} * g^{'} (z^{[1]})_{[4 \times 1]} \end{aligned} \end{matrix}

$\begin{split}{\rm d}z^{[1]}_{[4\times1]} &=\frac{{\rm d}}{{\rm d}z^{[1]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}a ^ {[1]}}z^{[2]} \cdot \frac{{\rm d}}{{\rm d}z^{[1]}}a^{[1]} \\&={\rm d}a^{[1]} \cdot g'(z^{[1]}) \\&= (W^{[2]}_{[1\times4]})^{T}{\rm d}z^{[2]}_{[1\times1]} * g'(z^{[1]})_{[4\times1]}\end{split}\tag{1.9}$

\begin{matrix} (1.10) & \begin{aligned} d W_{[4 \times 3]}^{[1]} & = \frac{d}{d W^{[1]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d a^{[1]}} z^{[2]} \cdot \frac{d}{d z^{[1]}} a^{[1]} \cdot \frac{d}{d W^{[1]}} z^{[1]} \\ = d z^{[1]} \cdot x \\ = d z_{[4 \times 1]}^{[1]} (a_{[3 \times 1]}^{[0]})^{T} \end{aligned} \end{matrix}

$\begin{split}{\rm d}W^{[1]}_{[4\times3]} &= \frac{{\rm d}}{{\rm d}W^{[1]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}a ^ {[1]}}z^{[2]} \cdot \frac{{\rm d}}{{\rm d}z^{[1]}}a^{[1]} \cdot \frac{{\rm d}}{{\rm d}W^{[1]}}z^{[1]} \\& = {\rm d}z^{[1]} \cdot x \\& = {\rm d}z^{[1]}_{[4\times1]}(a^{[0]}_{[3\times1]})^T \end{split}\tag{1.10}$

\begin{matrix} (1.11) & \begin{aligned} d b_{[4 \times 1]}^{[1]} & = \frac{d}{d W^{[1]}} L (a^{[2]}, y) \\ = \frac{d}{d a^{[2]}} L (a^{[2]}, y) \cdot \frac{d}{d z^{[2]}} a^{[2]} \cdot \frac{d}{d a^{[1]}} z^{[2]} \cdot \frac{d}{d z^{[1]}} a^{[1]} \cdot \frac{d}{d b^{[1]}} z^{[1]} \\ = d z_{[4 \times 1]}^{[1]} \end{aligned} \end{matrix}

$\begin{split}{\rm d}b^{[1]}_{[4\times1]} &= \frac{{\rm d}}{{\rm d}W^{[1]}}{\cal L}(a^{[2]}, y) \\&= \frac{{\rm d}}{{\rm d}a^{[2]}}{\cal L}(a^{[2]}, y) \cdot \frac{{\rm d}}{{\rm d}z^{[2]}}a^{[2]} \cdot \frac{{\rm d}}{{\rm d}a ^ {[1]}}z^{[2]} \cdot \frac{{\rm d}}{{\rm d}z^{[1]}}a^{[1]} \cdot \frac{{\rm d}}{{\rm d}b^{[1]}}z^{[1]} \\& = {\rm d}z^{[1]}_{[4\times1]} \end{split}\tag{1.11}$

下面是vectorization后的反向传播算法公式：

\begin{matrix} (2.1) & L (A^{[2]}, Y) = \frac{1}{m} \sum_{i = 1}^{m} - y^{(i)} \log A^{[2] (i)} - (1 - y^{(i)}) \log (1 - A^{[2] (i)}) \end{matrix}

${\cal L}(A^{[2]}, Y) =\frac{1}{m}\sum_{i = 1}^m -y^{(i)}\log A^{[2](i)}-(1-y^{(i)})\log (1-A^{[2](i)})\tag{2.1}$

\begin{matrix} (2.2) & \begin{aligned} d A_{[1 \times m]}^{[2]} & = [(- \frac{Y^{(1)}}{A^{[2] (1)}} + \frac{1 - Y^{(1)}}{1 - A^{[2] (1)}}), \dots, (- \frac{Y^{(m)}}{A^{[2] (m)}} + \frac{1 - Y^{(m)}}{1 - A^{[2] (m)}})] \end{aligned} \end{matrix}

$\begin{split}{\rm d}A^{[2]}_{[1\times m]} &=[(-\frac{Y^{(1)}}{A^{[2](1)}} + \frac{1-Y^{(1)}}{1-A^{[2](1)}}), \cdots ,(-\frac{Y^{(m)}}{A^{[2](m)}} + \frac{1-Y^{(m)}}{1-A^{[2](m)}})]\end{split}\tag{2.2}$

\begin{matrix} (2.3) & \begin{aligned} d Z_{[1 \times m]}^{[2]} & = [(- \frac{Y^{(1)}}{A^{[2] (1)}} + \frac{1 - Y^{(1)}}{1 - A^{[2] (1)}}), \dots, (- \frac{Y^{(m)}}{A^{[2] (m)}} + \frac{1 - Y^{(m)}}{1 - A^{[2] (m)}})] * [A^{[2] (1)} (1 - A^{[2] (1)}), \dots, A^{[2] (m)} (1 - A^{[2] (m)})] \\ = [(A^{[2] (1)} - Y^{(1)}), \dots, (A^{[2] (m)} - Y^{(m)})] \\ = A^{[2]} - Y \end{aligned} \end{matrix}

$\begin{split}{\rm d}Z^{[2]}_{[1\times m]} &=[(-\frac{Y^{(1)}}{A^{[2](1)}} + \frac{1-Y^{(1)}}{1-A^{[2](1)}}), \cdots ,(-\frac{Y^{(m)}}{A^{[2](m)}} + \frac{1-Y^{(m)}}{1-A^{[2](m)}})] * [A^{[2](1)}(1-A^{[2](1)}), \cdots,A^{[2](m)}(1-A^{[2](m)})] \\&=[(A^{[2](1)}-Y^{(1)}), \cdots,(A^{[2](m)}-Y^{(m)})] \\&=A^{[2]}-Y \end{split}\tag{2.3}$

\begin{matrix} (2.4) & d W_{[1 \times 4]}^{[2]} = \frac{1}{m} d Z_{[1 \times m]}^{[2]} (A_{[4 \times m]}^{[1]})^{T} \end{matrix}

${\rm d}W^{[2]}_{[1\times4]} = \frac{1}{m}{\rm d}Z^{[2]}_{[1\times m]}(A^{[1]}_{[4\times m]})^T\tag{2.4}$

\begin{matrix} (2.5) & d b_{[1 \times 1]}^{[2]} = \frac{1}{m} n p . s u m (d Z^{[2]}, a x i s = 1, k e e p d i m s = T r u e) \end{matrix}

${\rm d}b^{[2]}_{[1\times1]} = \frac{1}{m}np.sum({\rm d}Z^{[2]},axis =1,keepdims =True)\tag{2.5}$

d Z_{[4 \times m]}^{[1]} = (W_{[1 \times 4]}^{[2]})^{T} d Z_{[1 \times m]}^{[2]} * g^{[1]}^{'} (Z^{[1]})_{[4 \times m]}

${\rm d}Z^{[1]}_{[4\times m]} = (W^{[2]}_{[1\times4]})^{T}{\rm d}Z^{[2]}_{[1\times m]}*g^{[1]}{'}(Z^{[1]})_{[4\times m]}$

\begin{matrix} (2.6) & d W_{[4 \times 3]}^{[1]} = \frac{1}{m} d Z_{[4 \times m]}^{[1]} (A_{[3 \times m]}^{[0]})^{T} \end{matrix}

${\rm d}W^{[1]}_{[4\times3]}=\frac{1}{m}{\rm d}Z^{[1]}_{[4\times m]}(A^{[0]}_{[3\times m]})^T\tag{2.6}$

\begin{matrix} (2.7) & d b_{[4 \times 1]}^{[1]} = \frac{1}{m} s p . s u m (d Z_{[4 \times m]}^{[1]}, a x i s = 1, k e e p d i m s = T r u e) \end{matrix}

${\rm d}b^{[1]}_{[4\times1]} = \frac{1}{m}sp.sum({\rm d}Z^{[1]}_{[4\times m]},axis =1,keepdims=True) \tag{2.7}$

总结

这里写图片描述