May 8, 2023
Each ML model optimizer approaches the point of optimization uniquely. Therefore, different models may require different optimizers. Instead of using the intuitive stochastic gradient descent (SGD), this article uses the popular adaptive momentum estimation (ADAM) optimizer. This implementation of the adam optimizer is inspired by Keras. It incorporates the concept of exponentially weighted averages. Refer to the commented code below to understand the mechanics.
# this code is a walk through of 2-layer CNN forward and backward propagation from scratch, using the adam optimizer
import numpy as np
mnistlocation = “INSERT MNIST FILE LOCATION” #you can download the file here
# For example: “/Users/enrichmentcap/Downloads/mndata.npz”
trains, tests = 1000, 1000 #depends on computer processing speed. If computer isn't fast, reduce size of training and test dataset
trainimages, trainlabels, testimages, testlabels = np.load(mnistlocation)['trainimages'][:trains] / 255, np.load(mnistlocation)['trainlabels'][:trains], np.load(mnistlocation)['testimages'][:tests] / 255, np.load(mnistlocation)['testlabels'][:tests]
##from tensorflow.keras.datasets import mnist #uncomment if using tensorflow library to retrieve dataset
##(trainimages, trainlabels), (testimages, testlabels) = mnist.load_data()
##trainimages, testimages = trainimages[:train]/255, testimages[:test]/255
np.random.seed(0)
classes = len(np.unique(trainlabels))
imw = trainimages.shape[2]
imh = trainimages.shape[1]
lr = .001 #adam optimizer requires different learn rate
fsw = 8 #kernel1 width
fsh = 8 #kernel1 height
fsw2 = 4 #kernel2 width
fsh2 = 4 #kernel2 height
filts = 16 #kernel1 number of filters
filts2 = 8 #kernel2 number of filters
kern = np.random.rand(filts, fsh, fsw) - .5
kb = np.zeros(filts)
kern2 = np.random.rand(filts2, filts, fsh2, fsw2) - .5
kb2 = np.zeros(filts2)
step = 1 #step sizes of kernel convolutions
step2 = 2
rsw = (imw - fsw) // step + 1 #resulting shape width of kernel1 convolution
rsh = (imh - fsh) // step + 1 #resulting shape height of kernel1 convolution
rsw2 = (rsw - fsw2) // step2 + 1 #resulting shape width of kernel2 convolution
rsh2 = (rsh - fsh2) // step2 + 1 #resulting shape height of kernel2 convolution
w = np.random.rand(classes, filts2, rsh2, rsw2) - .5
b = np.random.rand(classes) - .5
b1 = .9 #adam optimizer parameters
b2 = .99
eps = 1e-7
mw = 0
vw = 0
mb = 0
vb = 0
mk = 0
vk = 0
mk2 = 0
vk2 = 0
for i in range(trains):
## print(i)
xx, label, label[trainlabels[i]] = trainimages[i], np.zeros(classes), 1
k = np.zeros((filts, rsh, rsw))
kk = np.zeros((filts2, rsh2, rsw2))
for j in range(rsh):
for jj in range(rsw):
k[:,j,jj] = (kern * xx[step*j:step*j+fsh, step*jj:step*jj+fsw].reshape(1,fsh,fsw)).sum(axis=(1,2)) + kb #k.shape = [filts,rsize,rsize] = [16,21,21]
for j0 in range(rsh2):
for jj0 in range(rsw2):
kk[:,j0,jj0] = (kern2 * k[:,step2*j0:step2*j0+fsh2, step2*jj0:step2*jj0+fsw2].reshape(1,filts,fsh2,fsw2)).sum(axis=(1,2,3)) + kb2 #kk.shape = (filts2, rsize2, rsize2)
x = (w * kk.reshape(1,filts2,rsh2,rsw2)).sum(axis=(1,2,3)) + b
y = np.exp(x) / np.sum(np.exp(x))
dydx = -np.exp(x)[trainlabels[i]] * np.exp(x) / np.sum(np.exp(x))**2
dydx[trainlabels[i]] = np.exp(x)[trainlabels[i]] * (np.sum(np.exp(x)) - np.exp(x)[trainlabels[i]]) / np.sum(np.exp(x))**2
dLdy = 1 / y[trainlabels[i]]
dLdx = dLdy * dydx
dxdw, dxdb = kk, 1
dLdw = dLdx.reshape(classes,1,1,1) * dxdw.reshape(1, filts2, rsh2, rsw2)
dLdb = dLdx * dxdb
## adam optimizer code block
lrt = lr * (1 - b2**(i+1))**.5 / (1 - b1**(i+1)) #as training increases lrt starts high goes low real fast then gradually heads towards 1
mw = b1 * mw + (1 - b1) * dLdw #takes large ratio of a small ratio of the previous slope, and adds to a small ratio of current slope
vw = b2 * vw + (1 - b2) * dLdw**2 #takes really large ratio of a really small ratio of the previous slope squared, and adds really small ratio of current slope squared
w = w + lrt * mw / (vw**.5 + eps) #better prelim results with abs(dLdw) in vw and removing square root of vw in w; independent from prior mw and vw, this results is a tanh shape more intuitive for what one would like as slope increases or decreases
mb = b1 * mb + (1 - b1) * dLdb
vb = b2 * vb + (1 - b2) * dLdb**2
b = b + lrt * mb / (vb**.5 + eps)
dxdkk = w #(classes, filts2*rsize2**2)
dLdkk = (dLdx.reshape(classes, 1, 1, 1) * dxdkk).sum(axis=0)#dLdkk represents all class loss integrated into appropriate positions in kk output; meaning which positions in kk led to more loss or error
dLdkern2 = np.zeros((filts2,filts,fsh2,fsw2))
dkkdkern2 = k
dkkdkb2 = 1
for f in range(filts2):
for j000 in range(fsh2):
for jj000 in range(fsw2):
dLdkern2[f, :, j000, jj000] = (dLdkk[f].reshape(1,rsh2,rsw2) * dkkdkern2[:,j000:j000+rsh2,jj000:jj000+rsw2]).sum(axis=(1,2))
dLdkb2 = dLdkk.sum(axis=(1,2)) * dkkdkb2
kb2 = kb2 + dLdkb2 * lr
## adam optimizer code block
mk2 = b1 * mk2 + (1 - b1) * dLdkern2
vk2 = b2 * vk2 + (1 - b2) * dLdkern2**2
kern2 = kern2 + lrt * mk2 / (vk2**.5 + eps)
dkkdk = kern2 # shape [filts2, filts * fsize2**2]
dLdk = np.zeros((filts, rsh, rsw))
for ooo in range(filts):
for o in range(rsh2):
for oo in range(rsw2):
dLdk[ooo,o*step2:o*step2+fsh2,oo*step2:oo*step2+fsw2] += (dLdkk[:,o,oo].reshape(filts2,1,1) * dkkdk[:,ooo,:,:]).sum(axis=0) #[filts2,] @ [filts2,fsize2,fsize2]
dkdkern = xx
dkdkb = 1
dLdkern = np.zeros((filts,fsh,fsw))
for j00 in range(fsh):
for jj00 in range(fsw):
dLdkern[:,j00,jj00] = (dkdkern[j00:j00+rsw, jj00:jj00+rsh].reshape(1,rsw,rsh) * dLdk).sum(axis=(1,2))
dLdkb = dLdk.sum(axis=(1,2)) * dkdkb
kb = kb + dLdkb * lr
## adam optimizer code block
mk = b1 * mk + (1 - b1) * dLdkern
vk = b2 * vk + (1 - b2) * dLdkern**2
kern = kern + lrt * mk / (vk**.5 + eps)
##loss = np.array([loss[i:i+100].mean() for i in range(trains - 100)])
checke = np.zeros(tests)
for i in range(tests):
xx = testimages[i]
k = np.zeros((filts, rsh, rsw))
kk = np.zeros((filts2, rsh2, rsw2))
for j in range(rsh):
for jj in range(rsw):
k[:,j,jj] = (kern * xx[step*j:step*j+fsh, step*jj:step*jj+fsw]).sum(axis=(1,2)) + kb #k.shape = [filts,rsize,rsize] = [16,21,21]
for j0 in range(rsh2):
for jj0 in range(rsw2):
kk[:,j0,jj0] = (kern2 * k[:,step2*j0:step2*j0+fsh2, step2*jj0:step2*jj0+fsw2].reshape(1,filts,fsh2,fsw2)).sum(axis=(1,2,3)) + kb2 #kk.shape = (filts2, rsize2, rsize2)
x = (w * kk).sum(axis=(1,2,3)) + b
if testlabels[i] == np.argmax(x):
checke[i] = 1
print(len(np.flatnonzero(checke==1))/tests)