135                 torch.autograd.grad(out.sum(), y)
139             torch.autograd.grad(out.sum(), x)
142             torch.autograd.grad(out.sum(), y_safe)
145             torch.autograd.grad(out.sum(), (x, y_safe))
162         torch.autograd.grad(out.sum(), (b, y_safe))
208             # Accessing .grad on leaf
210             foo = dummy.grad
213             # Accessing .grad on non-leaf
215             foo = dummy.grad
218             # Accessing .grad on non-leaf that retains gradients
220             foo = dummy.grad
230         self.assertEqual(x.grad, y + torch.ones(5, 5))
231         self.assertEqual(y.grad, x + torch.ones(5, 5) * 2)
232         self.assertIsNotNone(x.grad.grad_fn)
233         self.assertIsNotNone(y.grad.grad_fn)
260         x_grad_desc = graph_desc(x.grad.grad_fn)
261         y_grad_desc = graph_desc(y.grad.grad_fn)
286             graph_desc(x.grad.grad_fn),
290             graph_desc(y.grad.grad_fn),
301             def backward(ctx, grad):  argument
302                 return grad * 2
307             self.assertEqual(v.grad, torch.full(shape, 2.0))
310                 v.grad.zero_()
312             self.assertEqual(v.grad, torch.full(shape, 2.0))
321             def backward(ctx, grad):  argument
329         self.assertIsNone(x.grad)
332         self.assertIsNone(x.grad)
335         self.assertIsNone(x.grad)
338             torch.autograd.grad(MyFunction.apply(x), x, allow_unused=True)[0]
348             def backward(ctx, grad):  argument
349                 self.assertEqual(grad, torch.zeros(1))
350                 return grad
363             def backward(ctx, grad):  argument
364                 self.assertIsNone(grad)
365                 return grad
473         self.assertIsNotNone(t1.grad)
474         self.assertIsNotNone(t2.grad)
475         self.assertIsNone(t3.grad)
527         # test to ensure grad(grad)check runs successfully even if there is an
541         # if grad for nextafter ends up being implemented, this should be changed
654         # .grad is NOT OK when leaf is passed (this is the current state, subject to change)
656             RuntimeError, "are currently running autograd.grad()"
658             torch.autograd.grad(out, (a,))
660         # .grad is OK when non-leaf is passed
671         torch.autograd.grad(b.sum(), (a,))
718         (gx,) = torch.autograd.grad(y, x)
742         (gx,) = torch.autograd.grad(y, x)
774             (gx,) = torch.autograd.grad(y, x)
779             (gx_expected,) = torch.autograd.grad(y_expected, x)
809             x_grad = x.grad
810             x_grad_clone = x.grad.clone()
830             params.grad = params_grad_tensor
831             grad_saved = params.grad
834                 id(grad_saved) == id(params.grad), should_preserve_reference
838             # Accumulate dense gradient to sparse gradient will change the `params.grad` reference
848             # Accumulate dense gradient to dense gradient will preserve the `params.grad` reference,
857 …         # Accumulate sparse gradient to sparse gradient will preserve the `params.grad` reference,
877         self.assertEqual(b.grad, torch.zeros(4, 1))
878         self.assertEqual(a.grad, torch.zeros(4, 0))
890         self.assertEqual(x.grad, x_grad)
891         self.assertEqual(y.grad, y_grad)
893         grad_sum = 2 * x.grad + y.grad
897         self.assertEqual(x.grad, x_grad + x_hv)
898         self.assertEqual(y.grad, y_grad + y_hv)
908         self.assertEqual(x.grad, x_grad)
909         self.assertEqual(y.grad, y_grad)
911         grad_sum = 2 * x.grad + y.grad
912         x_hv = torch.autograd.grad(
922         self.assertEqual(x.grad, x_grad)
923         self.assertEqual(y.grad, y_grad)
928             torch.autograd.grad(
947             ref = torch.autograd.grad(out.sum(), inp)
950             new = torch.autograd.grad(out.sum(), edge)
980             torch.autograd.grad(out.sum(), x)
983         torch.autograd.grad(out.sum(), x_edge)
990         g_old, g_new = torch.autograd.grad(x.sum(), (x_edge, x))
1000         ref = torch.autograd.grad(out.sum(), (x, y))
1006         new = torch.autograd.grad(out.sum(), inp_edges)
1019         torch.autograd.grad(
1022         torch.autograd.grad(
1025         torch.autograd.grad(out.sum(), (x, edge_y), allow_unused=True)
1031             torch.autograd.grad(
1107         (tmp_grad,) = torch.autograd.grad(out, (tmp_edge,))
1109         (x_grad,) = torch.autograd.grad(tmp_edge, (x,), grad_outputs=(tmp_grad,))
1113         (x_grad_ref,) = torch.autograd.grad(out, (x,))
1120             RuntimeError, "grad can be implicitly created only for scalar output"
1122             torch.autograd.grad(tmp_edge, (x,))
1126         torch.autograd.grad(tmp_edge, (x,))
1130         (tmp_grad,) = torch.autograd.grad(out, (tmp_edge,))
1132             torch.autograd.grad(
1138         (tmp_grad,) = torch.autograd.grad(out, (tmp_edge,))
1140             torch.autograd.grad(
1156             (grad_x,) = torch.autograd.grad(
1161             self.assertIsNone(y.grad)
1162             self.assertIsNone(x.grad)
1172         self.assertIsNotNone(y.grad)
1173         self.assertIsNotNone(x_init.grad)
1189         grad_a, grad_b = torch.autograd.grad(
1196         self.assertIsNone(x.grad)
1200         # We want to capture grad of a nonleaf tensor,
1208         def hook(grad):  argument
1209             hook_results[0] = grad
1216         self.assertEqual(x.grad, expected_grad)
1217         self.assertIsNone(x_list[0].grad)
1223             self.assertEqual(x.grad, expected_grad)
1224             self.assertIsNone(x_list[i].grad)
1230         dydx = torch.autograd.grad(y, x, create_graph=True)
1231         d2ydx2_none = torch.autograd.grad(dydx, x, create_graph=True, allow_unused=True)
1232         d2ydx2 = torch.autograd.grad(
1236         d3ydx3 = torch.autograd.grad(d2ydx2, x, materialize_grads=True)
1243             torch.autograd.grad(y, x, allow_unused=False, materialize_grads=True)
1254             "post accumulate grad hooks cannot be registered on non-leaf tensors",
1260             tensor.sub_(tensor.grad)
1276             tensor.sub_(tensor.grad)
1292             return tensor.grad
1363                 tensor.sub_(tensor.grad)
1386         def pre_hook(grad):  argument
1387             return grad.sub(2.0)
1393             tensor.grad.add_(0.5)
1396             tensor.grad = grad_out[0].mul(10)
1411         self.assertEqual(torch.tensor([3.0, 3.0, 3.0]), tensor.grad)
1416             def __call__(self, grad):  argument
1417                 return grad.clone()
1452         # grad also runs hooks on accumulate grad nodes, even though
1453         # the accumulate grad nodes are not actually executed
1455         torch.autograd.grad(b.sum(), inputs=(a,), retain_graph=True)
1463         self.assertEqual(b.grad.item(), 3)
1476         self.assertEqual(b.grad.item(), 4)
1484         self.assertEqual(b.grad.item(), 2)
1488         # grad made by tensor prehooks
1528         # grad executes the tensor hooks of the next node but not
1561         torch.autograd.grad(c, inputs=(b), retain_graph=True)
1567         torch.autograd.grad(c, inputs=(a, b), retain_graph=True)
1598             for d_i in torch.autograd.grad(loss, intermediates_batch)
1604             self.assertEqual(l.grad, i * i * (1 + l))
1608         with self.assertRaisesRegex(RuntimeError, "does not require grad"):
1614         with self.assertRaisesRegex(RuntimeError, "does not require grad"):
1615             torch.autograd.grad(x, y)
1616         with self.assertRaisesRegex(RuntimeError, "does not require grad"):
1617             torch.autograd.grad(y, x)
1621         torch.autograd.grad(y, x)  # this should succeed now
1625         with self.assertRaisesRegex(ValueError, "grad requires non-empty inputs."):
1626             torch.autograd.grad(2 * x, [], grad_outputs=torch.tensor([1.0]))
1642         # Make sure x and y have grad accumulators allocated
1646         grad_x, grad_y = torch.autograd.grad(x * 2, [x, y], allow_unused=True)
1651         # have a grad accumulator allocated.
1653         grad_x, grad_z = torch.autograd.grad(x * 2, [x, z], allow_unused=True)
1659             grad_x, grad_y = torch.autograd.grad(x * 2, [x, y], allow_unused=False)
1675         (gY,) = torch.autograd.grad(x, (y,), allow_unused=True)
1681         (gY, gZ) = torch.autograd.grad(x + z, (y, z), allow_unused=True)
1688         self.assertIsNone(y.grad)
1697         (grad,) = torch.autograd.grad(out, (x,), (batched_grad,), is_grads_batched=True)
1699             grad, torch.arange(3).expand(2, 2, 3).transpose(0, 2).to(dtype=grad.dtype)
1707             torch.autograd.grad(
1717         (grad,) = torch.autograd.grad(out, (x,), (batched_grad,), is_grads_batched=True)
1719             grad, torch.arange(3).expand(2, 2, 3).transpose(0, 2).to(dtype=grad.dtype)
1727             torch.autograd.grad(
1741         def bw_hook(inc, grad):  argument
1742             self.assertIsInstance(grad, torch.Tensor)
1759         def bw_hook_modify(grad):  argument
1760             return grad.mul(2)
1765             y.grad.zero_()
1767         self.assertEqual(y.grad, (x + 1) * 2)
1771             y.grad.zero_()
1773         self.assertEqual(y.grad, (x + 1) * 4)
1839             self.assertTrue(torch.allclose(a.grad, torch.ones(3, 3) * 2))
1862         self.assertTrue(torch.allclose(a.grad, b.grad * 2))
1891         self.assertTrue(torch.allclose(a.grad, torch.ones(1) * 4 * k))
1892         self.assertTrue(torch.allclose(b.grad, torch.ones(1) * 4 * k))
1912             self.assertTrue(torch.allclose(a.grad, torch.ones(3, 3) * 2))
1936             self.assertTrue(torch.allclose(a.grad, torch.ones(3, 3) * 2))
1975         ref_grad = a.grad.clone()
1980         # Verify that the post hook got called and the grad propagation worked
1981         self.assertEqual(ref_grad / 2.0 + ref_grad, a.grad)
1991         def bw_hook(grad):  argument
1993             return grad * 2
2002             x.grad, torch.ones(5, 5, dtype=torch.double) * 2, atol=1e-5, rtol=0
2021         def hook(grad):  argument
2022             self.assertIsNotNone(grad)
2044         self.assertEqual(h1 * 2, h1.grad)
2046         self.assertEqual(h1 * 4, h1.grad)
2049             input.grad.zero_()
2054         self.assertEqual(input * 18, input.grad)
2063         self.assertEqual(a.grad, torch.tensor([1.0]))
2071         self.assertEqual(a.grad, torch.tensor([1.0]))
2081         # The grad is 1, not 4, because we are computing grad wrt the latest
2083         self.assertEqual(a.grad, torch.tensor([1.0]))
2085         # If the base did not originally require grad, there should be no hook
2112             gvar = var.grad
2113             gmean = mean.grad
2119             gvar_expected, gmean_expected = torch.autograd.grad(out, inputs=(var, mean))
2133         # so if the retains grad were not properly updated to the new grad_fn,
2134         # the grad would still be None
2135         self.assertEqual(view.grad, view2.grad)
2136         self.assertEqual(view.grad, torch.tensor([1.0]))
2143         def fn1(grad):  argument
2146             self.assertEqual(grad, torch.tensor([4.0]))
2147             return grad * 2
2149         def fn2(grad):  argument
2151             self.assertEqual(grad, torch.tensor([1.0]))
2152             return grad * 2
2162         self.assertEqual(a.grad, torch.tensor([8.0]))
2166         def fn3(grad):  argument
2168             self.assertEqual(grad, torch.tensor([4.0]))
2169             return grad * 2
2179         self.assertEqual(a.grad, torch.tensor([8.0]))
2196             def fn0(grad):  argument
2198                 self.assertEqual(grad, torch.ones_like(out1) * 2)
2200             def fn1(grad):  argument
2202                 self.assertEqual(grad, torch.ones_like(out1) * 3)
2204             def fn2(grad):  argument
2206                 self.assertEqual(grad, torch.ones_like(out1))
2224         def fn0(grad):  argument
2227         def fn1(grad):  argument
2230         def fn2(grad):  argument
2232             self.assertEqual(grad, torch.tensor([1.0]))
2269         self.assertEqual(v.grad, grad_output)
2276         self.assertEqual(x.grad, x_grad * grad_output)
2277         self.assertEqual(y.grad, y_grad * grad_output)
2278         self.assertEqual(z.grad, z_grad * grad_output)
2325                     grad_res = torch.autograd.grad(self_to, self_from, grad_to)[0]
2337             # input needs to require grad so we can call a backward pass
2354             a_grad = None if a.grad is None else a.grad.clone().detach()
2355             b_grad = None if b.grad is None else b.grad.clone().detach()
2374             self.assertEqual(a_grad, a.grad)
2375             self.assertEqual(b_grad, b.grad)
2393         self.assertEqual(x.grad, grad_z)
2394         self.assertEqual(y.grad, grad_z)
2395         self.assertEqual(a.grad, grad_c * b)
2396         self.assertEqual(b.grad, grad_c * a)
2397         self.assertEqual(q.grad, (grad_c + grad_z) * 2)
2427             x.grad.zero_()
2428             y.grad.zero_()
2431         self.assertEqual(x.grad, x_grad_expected)
2432         self.assertEqual(y.grad, y_grad_expected)
2436         self.assertEqual(x.grad, x_grad_expected)
2437         self.assertEqual(y.grad, torch.zeros(2, 2), exact_dtype=False)
2441         self.assertEqual(y.grad, y_grad_expected)
2442         self.assertEqual(x.grad, torch.zeros(2, 2), exact_dtype=False)
2446         self.assertEqual(y.grad, y_grad_expected)
2447         self.assertEqual(x.grad, torch.zeros(2, 2), exact_dtype=False)
2473         self.assertEqual(y.grad, y_grad_expected)
2474         self.assertEqual(x.grad, x_grad_expected)
2475         self.assertEqual(x_nonleaf.grad, x_non_leaf_expected)
2479         # x.grad will simply be None.
2483         self.assertIsNone(z.grad)
2495         self.assertEqual(x.grad, 2 * xd * go_y + 6 * xd.pow(5) * go_z)
2506             def backward(ctx, *grad):  argument
2507                 return torch.cat(grad)
2912         """Python Functions should respect grad mode."""
2941             self.assertEqual(y.grad, expected_grad)
2944             if y.grad is not None:
2946                     y.grad.zero_()
3002         if y.grad is not None:
3004                 y.grad.zero_()
3018         self.assertEqual(y.grad, expected_grad)
3031         self.assertEqual(y.grad, expected_grad)
3045         self.assertEqual(y.grad, expected_grad)
3054         self.assertEqual(y.grad, expected_grad)
3161         self.assertEqual(a.grad, torch.ones(2, 3))
3169         self.assertEqual(a.grad, torch.ones(2, 3))
3182         del x.grad
3183         self.assertIsNone(x.grad)
3203         self.assertEqual(a.grad, b * grad_output * 2)
3204         self.assertEqual(b.grad, a * grad_output * 2)
3249             b.grad = None
3257             # Should not warn for grad
3259                 torch.autograd.grad(c, b, torch.ones_like(c), create_graph=True)
3311             x.grad.zero_()
3320         self.assertEqual(x.grad, torch.ones(5, 5) / 2)
3322         self.assertEqual(x.grad, torch.empty(5, 5).fill_((1 + math.e) / 2))
3332         self.assertEqual(leaf.grad, torch.ones(5, 5))
3376         self.assertEqual(x.grad, torch.ones(5, 5))
3452         self.assertEqual(x.grad, expected_grad)
3465         self.assertEqual(x.grad, expected_grad_input)
3466         self.assertEqual(value.grad, torch.ones_like(value))
3473         self.assertEqual(x.size(), x.grad.size())
3529         (result,) = torch.autograd.grad(a.diagonal(), a, v_expanded)
3535         (result,) = torch.autograd.grad(a[0], a, v_expanded)
3543         (result,) = torch.autograd.grad(a[3:5], a, v_expanded)
3556         self.assertEqual(x.grad, expected_grad)
3559             x.grad.zero_()
3565         self.assertEqual(x.grad, expected_grad)
3575         grad = torch.rand_like(out)
3576         out.backward(grad)
3577         grad_dense = x.grad.clone()
3578         x.grad = None
3580         out.backward(grad)
3581         self.assertEqual(grad_dense, x.grad.to_dense())
3733             (gy,) = torch.autograd.grad(y[0], x, create_graph=True)
3734             (ggy_1,) = torch.autograd.grad(gy[0, 0, 0], x, retain_graph=True)
3735             (gy,) = torch.autograd.grad(y[0], x, create_graph=True)
3736             (ggy_2,) = torch.autograd.grad(gy[0, 0, 0], x, retain_graph=True)
3776         torch.autograd.grad(y, x, create_graph=True)
3777         torch.autograd.grad(y, x)
3780         torch.autograd.grad(y, x, create_graph=True)
3781         torch.autograd.grad(y, x)  # should not error!
3800         self.assertEqual(x.grad, torch.ones(10, 10))
3810         self.assertEqual(x.grad, torch.ones(10, 10) * 2)
3811         self.assertEqual(y.grad, torch.ones(10, 10) * 2)
3840         self.assertEqual(fvar.grad, torch.ones_like(fvar))
3841         self.assertEqual(type(fvar.grad), type(fvar))
3844         self.assertEqual(dvar.grad, torch.ones_like(dvar))
3845         self.assertEqual(type(dvar.grad), type(dvar))
3867                 self.assertIs(y.grad.get_device(), 1)
3925 …err_msg_outputs = r"Can't call numpy\(\) on Tensor that requires grad. Use tensor.detach\(\).numpy…
3954         # Make sure hooks only receive grad from usage of q, not x.
3955         def hook(grad):  argument
3957             self.assertEqual(grad, torch.ones(5, 5))
3961         self.assertEqual(x.grad, torch.ones(5, 5) * 3)
3962         self.assertEqual(y.grad, torch.ones(5, 5))
3984         self.assertEqual(y.grad, torch.ones(5, 5))
3996         self.assertEqual(y.grad, torch.ones(5))
3997         self.assertEqual(z.grad, torch.ones(5) * 2)
4068         #    modified the grad in 3rd place)
4093         self.assertEqual(x.grad, torch.ones(5, 5) * 34)
4094         self.assertEqual(y.grad, torch.ones(5, 5) * 17)
4115         self.assertEqual(x.grad, 2 * x)
4130         self.assertEqual(x.grad, torch.ones_like(x))
4175         self.assertEqual(x.grad, torch.ones(x.size()))
4196                     # This should not mutate the global grad mode!
4236                 return ctx.x.grad * grad_output
4242         self.assertEqual(x.grad, y_data)
4261             def backward(ctx, grad):  argument
4264                 return grad
4278         grad = torch.randn(3, 4, 6, 3, requires_grad=True)
4284         torch.autograd.backward(r1, grad)
4285         torch.autograd.backward(r2, grad)
4286         self.assertEqual(input1.grad, input2.grad, rtol=0.01, atol=0.0)
4335             # check whether A.grad is symmetric
4339             self.assertEqual(A.grad, A.grad.mT)
4376             self.assertEqual(input.grad.abs().sum(), 0)
4410             # or grad accumulator
4457         # Accumulate grad node has more than one input
4478             torch.autograd.grad((out, out3, out2), inputs=(a,))
4507             torch.autograd.grad((out,), inputs=(a, b))
4526             torch.autograd.grad((out,), inputs=(a,))
5026             "requires grad is being used in "
5036         (g,) = torch.autograd.grad(
5039         torch.autograd.grad(g.sum(), [x])
5049         gradients = torch.autograd.grad(
5091         # the inputs can require grad if we're in no_grad() mode
5099         # we should throw an exception if the output requires grad
5149         grad = torch.randn(5, 10, requires_grad=True)
5150         out_shape, grad_shape = _calculate_shape(out, grad, False)
5162         grad = torch.nested.as_nested_tensor(
5168         out_shape, grad_shape = _calculate_shape(out, grad, False)
5213         (ginp,) = torch.autograd.grad(out, (inp,), create_graph=True)
5219         (ginp,) = torch.autograd.grad(out, (inp,), create_graph=True)
5238                     (ginp,) = torch.autograd.grad(out, (inp,), create_graph=True)
5271                     torch.autograd.grad(d2.sum(), a)
5291                         torch.autograd.grad(d2.sum(), a)
5302             # backward call in grad mode will work
5310             # grad -> MulBackward -> PyObject -> ExpBackward
5313                 grad = torch.autograd.grad(t, x, torch.ones_like(t), create_graph=True)
5320 …# We want to test that when grad goes out of scope at the end of this function that PyObject is de…
5373             (ginp,) = torch.autograd.grad(out, (inp,), create_graph=True)
5427         # create autograd function that saves grad pointer as class static
5436             def backward(ctx, grad):  argument
5437                 MyFunc.static_grad_ptr = grad.data_ptr()
5438                 return grad, grad
5447             def backward(ctx, grad):  argument
5452         # non-contiguous grad should be copied
5454         self.assertFalse(a.grad.data_ptr() == MyFunc.static_grad_ptr)
5455         self.assertFalse(b.grad.data_ptr() == MyFunc.static_grad_ptr)
5457         a.grad = b.grad = None
5460         p_a = a.grad.data_ptr()
5461         p_b = b.grad.data_ptr()
5462         # check a,b uses different grad buffer
5468         # create autograd function that saves grad pointer as class static
5477             def backward(ctx, grad):  argument
5478                 MyFunc.static_grad_ptr = grad._values().data_ptr()
5479                 return grad, grad
5489             def backward(ctx, grad):  argument
5491                 # and return as grad.
5511         p_a = a.grad._values().data_ptr()
5512         p_b = b.grad._values().data_ptr()
5513         # check a,b uses different grad buffer
5523         a.grad = b.grad = None
5528         p_a = a.grad._values().data_ptr()
5529         p_b = b.grad._values().data_ptr()
5530         # check a,b uses different grad buffer
5691             # when none of the inputs require grad (always raises even if raise_exception=False)
5820             # runtime error while compute batched grad (print big error)
5850                 def hook(grad):  argument
5851                     if grad is not None:
5852                         return grad.to_dense().to_sparse(1)
5853                     return grad
5861                 RuntimeError, "grad is sparse tensor, but has incorrect sparse_dim"
5941             with self.assertRaisesRegex(RuntimeError, "grad is incorrect layout"):
6615     def backward(ctx, grad):
6616         return grad
6801                 # pre-allocate the grad so that increased memory usage is mainly
6805                         lin.weight.grad = torch.ones_like(lin.weight)
6806                         lin.bias.grad = torch.ones_like(lin.bias)
7128             self.assertEqual(checkpoint_param.grad, param.grad)
7137             torch.autograd.grad(b, (a,))
7167                 # out does not require grad
7169                 # Make loss require grad, otherwise we would run into
7170                 # "element 0 of tensors does not require grad and does not have a grad_fn"
7202         without reentrant autograd, for both backward() and autograd.grad().
7208         b_grad = a.grad
7210         a.grad = None
7213         c_grad = a.grad
7215         a.grad = None
7217         (d_grad,) = torch.autograd.grad(d, (a,))
7248         b_grad = a.grad
7250         a.grad = None
7255         c_grad = a.grad
7266         def hook(grad):  argument
7318             self.assertEqual(param.grad, checkpoint_param.grad)
7337             def backward(ctx, grad):  argument
7339                 return grad
7435         def add_gradient_penalty_to_grad(grad):  argument
7437             old_param_grad = grad
7438             param.grad = None
7441                 g = grad.detach().requires_grad_()
7445             res = g.grad + grad
7446             param.grad = old_param_grad
7460         def manual_increase_gradient(grad):  argument
7464                 g = grad.detach().requires_grad_()
7467             res = g.grad + grad
7476         self.assertEqual(param.grad, 6 * param)
7855             def backward(ctx, grad):  argument
7856                 return grad
7858         # Original Tensor does not require grad
7861         # Tensor being written does require grad
7956             def backward(ctx, grad):  argument
7960                 res.select(0, 0).copy_(grad)
8058             # the views are created in grad mode or not
8147         # Special handling for printing view created in no-grad and modified
8148         # in-placed in no-grad.
8195         non_inplace_grad = a_orig.grad
8203         inplace_grad = a_orig.grad
8232             def backward(ctx, grad):  argument
8236                 res.select(0, ctx.idx).copy_(grad)
8273             def backward(ctx, grad):  argument
8275                 return grad, grad
8308             def backward(ctx, grad):  argument
8310                 grad = 3 * grad
8311                 return grad, grad
8407             # jvp must properly modify the input grad if mark_dirty is set
8435         self.assertEqual(z.grad, torch.view_as_real(expected))
8505                     (grad,) = torch.autograd.grad(foo**3, foo, grad_outputs=go)
8508                 return grad
8522         self.assertEqual(b.grad, torch.tensor([-inf, 0.0, 0.0]))
8528         self.assertEqual(b.grad, torch.tensor([-inf, 0.0, 0.0]))
8920             def backward(ctx, grad):  argument
8921                 return grad, None
9057                 self.assertEqual(a.grad, a)
9113             a.grad = None
9114             t.grad = None
9145             def backward(ctx, grad):  argument
9146                 return grad
9214         (gradx_f,) = torch.autograd.grad(f, x, create_graph=True)
9216         _ = torch.autograd.grad(gradx_f, x)
9297                 self.assertEqual(2 * a, a.grad)
9309                 self.assertEqual(3 * a, a.grad)
9317             (g,) = torch.autograd.grad(s, (a,), create_graph=True)
9323                 self.assertEqual(6 * a, a.grad)
9397         self.assertEqual(a.grad, y)
9425             self.assertEqual(2 * a, a.grad)
9433             self.assertEqual(2 * a, a.grad)
9441             self.assertEqual(4 * a, a.grad)
9449         self.assertEqual(2 * a, a.grad)
9461             self.assertEqual(a.grad, y)
9479         self.assertEqual(2 * 5 * 5 * a, a.grad)
9480         self.assertEqual(2 * 3 * 3 * b, b.grad)
9622         b_grad = a.grad.clone()
9623         a.grad.zero_()
9630         c_grad = a.grad.clone()
9631         a.grad.zero_()
9641         d_grad = a.grad.clone()
9683                 self.assertEqual(2 * a, a.grad)
9690             (g,) = torch.autograd.grad(s, (a,), create_graph=True)
9692             self.assertEqual(6 * a, a.grad)
9698         (g,) = torch.autograd.grad(s, (a,), create_graph=True)
9701         self.assertEqual(6 * 2 * a, a.grad)
9707             (g,) = torch.autograd.grad(s, (a,), create_graph=True)
9709             # factor 4 because pow_backward is grad * (exp * self.pow(exp - 1))
9710             # so grad is saved and self (i.e. a) is saved
9711             self.assertEqual(6 * 4 * a, a.grad)
9717             (g,) = torch.autograd.grad(s, (a,), create_graph=True)
9721             self.assertEqual(6 * 8 * a, a.grad)
9732             torch.autograd.grad(out, (a,))
9750                 expected = a.grad
9781         # with grad
9789         # without grad
10270         (y1_expected,) = torch.autograd.grad(y1, x)
10273         (y2_expected,) = torch.autograd.grad(y2, x)
10598         # Tests some private helper functions to enable/disable fwd grad mode
10656         # or weight to be a dual tensor, so grad_input requires forward grad
10824             gfoo, gbar = torch.autograd.grad(
10830             gfoo, gbar = torch.autograd.grad(
10877             # See note [Forward Grad View/inplace] for more details on how these views work.
10945         # the resulting forward grad for self for...
10948         # This is kind of fragile because the above depends on how the forward grad expression
11101             # db -> AutogradMeta -> ForwardGrad -> db's grad
11102             # db's grad -> AutogradMeta -> MulBackward
11126                 self.assertEqual(x.grad.sum(), 1.0)
11127                 self.assertEqual((x.grad == 1 / 3).sum(), 3)
11172                 asd.grad = None
11250         self.assertEqual(x.grad, dense_grad + sparse_grad1 + sparse_grad2)
11258         self.assertEqual(x.grad, dense_grad + sparse_grad1 + sparse_grad2)
11262         self.assertEqual(x.grad, sparse_grad1 + sparse_grad2)
11272         self.assertEqual(tensor.grad, mask.to_dense())
11375         self.assertEqual(x.grad, torch.ones(n, 1, device=device))
11407             def backward(ctx, grad):  argument
11410                 return grad
11422         self.assertIsNone(t2.grad)
11423         self.assertIsNone(t1.grad)
11424         self.assertIsNone(t3.grad)
11531         self.assertFalse(s.grad is None or s.grad.abs().sum().item() == 0)
11550             x.grad = 0
11554             x.grad = torch.randn(2, 2, device=devices[0])
11558             x.grad = torch.randn(5, 5, dtype=torch.long, device=devices[0])
11562             x.grad = x
11564         # Tests device -> cpu grad assignment raises
11568                 t_cpu.grad = torch.randn(5, 5, device=devices[0])
11573             x.grad = torch.zeros_like(x)
11579                 x.grad = torch.randn(5, 5, device=devices[1])
11605         self.assertEqual(x.grad, torch.ones(5, 5) * 2)
11742         self.assertEqual(root.grad.tolist(), [[2, 2], [1, 1]])
11752         self.assertEqual(root.grad.tolist(), [[2, 2], [0, 0]])
11763         self.assertEqual(root.grad.tolist(), [[1, 2], [1, 1]])
11773         # Force a graph update with grad disabled.
11826         # in-place modification to view makes base require grad
11849         self.assertEqual(b.grad.tolist(), [5])
11850         self.assertIsNone(a.grad)
11855         # grad also forces any previous views to require grad and backprop
11885             def backward(ctx, grad):  argument
11886                 return grad, grad
11907         self.assertEqual(root.grad.tolist(), [[1, 2], [1, 1], [1, 1]])
11949             def backward(ctx, grad):  argument
11950                 return grad * 2, None
11954         self.assertIsNone(b.grad)
11955         self.assertEqual(a.grad.item(), 2)
11981         # (1) If leaf is non-overlapping and dense, grad's layout should match its leaf.
11990                 self.assertEqual(a.grad.stride(), a.stride())
11992                 self.assertEqual(b.grad.stride(), b.stride())
11994                 a.grad = None
11995                 b.grad = None
11997                 self.assertEqual(a.grad.stride(), a.stride())
11998                 self.assertEqual(b.grad.stride(), b.stride())
12008         self.assertEqual(c.grad.stride(), (2, 1))
12010         c.grad = None
12012         self.assertEqual(c.grad.stride(), (2, 1))
12025                     inp_r.grad, torch.ones_like(inp_r) * torch.sgn(out_c_inter).real
12040                     inp_r.grad, torch.ones_like(inp_r) * torch.sgn(out_inter).real
12069         msg = "grad can be implicitly created only for real scalar outputs"
12074             torch.autograd.grad(b, a)
12125             return a.grad
12148             return a.grad
12175         self.assertTrue(torch.allclose(a * 2, a.grad))
12185             self.assertTrue(torch.allclose(a * 2, a.grad))
12193             torch.autograd.grad(out, a, create_graph=True)
12194             (da,) = torch.autograd.grad(out, a, create_graph=True)
12195             (d2a,) = torch.autograd.grad(da.sum(), a)
12383                 # that does not require grad
12414                 # that does not require grad
12439                     # leaf variable that requires grad is being used in an inplace
12587                 self.assertEqual(c.grad, torch.ones_like(c))
12618                     "but one of the arguments requires grad"
12737             self.assertEqual(x.grad, x + 3.5)
12753         # the same .grad for each input, and the gradients should
12755         self.assertEqual(x.grad, 10 * (x + 3.5))
12759             grads = torch.autograd.grad(y.sum(), x)
12763         # since we use functional grad() api, gradients will not
12852         def hook(grad):  argument
12946         # then calls backward()/grad() on BOTH threads, like a Y pattern from input at the
12983             x_retain.grad,
12991         # Case 1: multiple grad() calls with jit threads, retain_graph=False
12996             return torch.autograd.grad([y.sum()], [orig_x])
13004             grad = torch.jit._wait(fut)
13005             return grad, grad_hat
13016             return torch.autograd.grad([y.sum()], [orig_x], retain_graph=True)
13023             grad = train_fn_jit_retain(y_retain, x)
13026             return grad, grad1, grad2
13028         grad, grad1, grad2 = train_fn_fork_join_calls_retain(
13031         self.assertEqual(grad, grad1)
13032         self.assertEqual(grad, grad2)
13041             def backward(ctx, *grad):  argument
13115     def grad(fn):  member in TestNestedCheckpoint
13119                 (grad_input,) = torch.autograd.grad(out, inputs=(x,), create_graph=True)
13141         grad, c = self.grad, self.checkpoint
13146             (grad(fn), (grad(c(fn)), grad(c(c(fn))))),
13148                 grad(grad(fn)),
13149                 (grad(c(grad(fn))), c(grad(grad(c(fn)))), grad(c(grad(c(fn))))),
13152                 grad(grad(grad(fn))),
13153                 (grad(c(grad(grad(c(fn))))), grad(c(grad(c(grad(c(fn))))))),
13214                 (ga,) = torch.autograd.grad(a, x)
13215                 (gb,) = torch.autograd.grad(b, x)
13230             grad, sum, c = self.grad, self.sum, self.checkpoint
13245             expected = grad(sum(grad(sum(h))))(a)
13246             actual = grad(sum(grad(sum(c(hc)))))(a)
13249             actual = grad(sum(c(grad(sum(c(hc))))))(a)
13252             self.check_graph_dies(grad(c(hc)))
13253             self.check_graph_dies(grad(sum(grad(sum(c(hc))))))
13254             self.check_graph_dies(grad(sum(c(grad(sum(c(hc)))))))
13270         actual_grads = torch.autograd.grad(out, (a, b))
13273         expected_grads = torch.autograd.grad(out, (a, b))
13290             actual_grads = torch.autograd.grad(out, (a, b))
13293             expected_grads = torch.autograd.grad(out, (a, b))
13786         x_grad = torch.autograd.grad(out.sum(), (x,))
13787         x_grad_ref = torch.autograd.grad(fn(x).sum(), (x,))
13806             def backward(ctx, grad, _unused):  argument
13808                 return grad * x.cos()
13826             x_grad = torch.autograd.grad(out.sum(), (x,))
13828             x_grad_ref = torch.autograd.grad(fn(x).sum(), (x,))
13860         grad = torch.randn(3, 3, device=device)
13861         out.backward(grad)
13864             # bogus default gradient registered for Autograd is grad + 1
13865             self.assertEqual(t.grad, grad + 1)
13867             # bogus gradient registered for AutogradCUDA is grad * 2
13868             self.assertEqual(t.grad, grad * 2)
13889         # bogus gradient for AutogradNestedTensor is grad * grad
13890         self.assertEqual(a.grad, c * c)
13891         self.assertEqual(b.grad, d * d)
13898         grad = torch.randn(3, 3, device=device)
13899         out.backward(grad)
13901 …   # t.grad is just out.grad by composite op since _test_autograd_multiple_dispatch is just a clone
13902         self.assertEqual(t.grad, grad)
13923         # bogus gradient for AutogradNestedTensor is grad * grad + grad
13924         self.assertEqual(a.grad, c * c + c)
13925         self.assertEqual(b.grad, d * d + d)
13955         grad = torch.randn(4, device=device)
13956         t_view_copy.backward(grad)
13957         t_view.backward(grad.clone())
13961         self.assertEqual(t.grad, t_ref.grad)
13964             # gradient registered to AutogradCUDA is grad.reshape_as(self) + 1
13965             self.assertEqual(t.grad, grad.reshape_as(t) + 1)
13967             # Default gradient registered is grad.reshape_as(self)
13968             self.assertEqual(t.grad, grad.reshape_as(t))
14092         def log_grad_order(grad: torch.Tensor, name: str, order):
14094             return grad
14108         self.assertEqual(x.grad, torch.ones_like(x))
14109         self.assertEqual(y.grad, 2 * torch.ones_like(x))
14110         self.assertEqual(z.grad, torch.ones_like(x))