crossEntropy

1 /* pytorch equivalent
2    >>> x = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], requires_grad=True)
3    >>> t = torch.tensor([1, 0, -100], dtype=torch.long)
4    >>> l = torch.nn.functional.cross_entropy(x, t)
5    >>> l
6    tensor(0.6944)
7    >>> l.backward()
8    >>> x.grad
9    tensor([[ 0.2375, -0.2375],
10            [-0.2625,  0.2625],
11            [ 0.0000,  0.0000]])
12  */
13 import std.stdio;
14 import std.typecons;
15 import mir.ndslice;
16 import grain.autograd;
17 import numir;
18 static import grain.config;
19 grain.config.backprop = true;
20 
21 auto hx = [[0.1f, 0.2f], [0.3f, 0.4f], [0.5f, 0.6f]].variable(true);
22 auto ht = [1, 0, -100].variable;
23 auto hl = crossEntropy(hx, ht);
24 hl.backward();
25 assert(approxEqual(hx.gradSliced, [[0.2375, -0.2375], [-0.2625, 0.2625],
26         [0.0000, 0.0000]].nparray));
27 
28 version (grain_cuda) {
29     auto dx = hx.to!DeviceStorage;
30     dx.grad.zero_();
31     auto dt = ht.to!DeviceStorage;
32     auto dl = crossEntropy(dx, dt);
33     assert(approxEqual(hl.sliced, dl.to!HostStorage.sliced));
34     dl.backward();
35     assert(approxEqual(dx.to!HostStorage.gradSliced, hx.gradSliced));
36 }