AdaGrad

1 import grain.autograd;
2 import numir;
3 
4 {
5     float lr = 0.1;
6     float eps = 1e-8;
7     auto model = MLP!(float, HostStorage)(3);
8     auto optim = AdaGrad!(typeof(model))(model, lr, eps);
9     static assert(isOptimizer!(typeof(optim)));
10     model.fc1.weight.data.zero_();
11     model.fc1.weight.grad = [[0.2f, 0.0f, 0.0f], [0.0f, 0.0f, 0.0f]].variable
12         .data;
13     optim.update();
14     auto w = model.fc1.weight;
15     assert(approxEqual(w.sliced, [[-lr * 0.2 / (0.2 * 0.2 + eps) ^^ 0.5, 0.0,
16             0.0], [0.0, 0.0, 0.0]].nparray));
17     auto m = optim.memory[".fc1.weight"].to!(typeof(w));
18     assert(approxEqual(m.sliced, [[0.2 * 0.2, 0.0, 0.0], [0.0, 0.0, 0.0]]
19             .nparray));
20 }
21 version (grain_cuda) {
22     auto model = MLP!(float, DeviceStorage)(3);
23     auto optim = AdaGrad!(typeof(model))(model, 0.1);
24     optim.update();
25 }