# Owner(s): ["module: tests"] import itertools import math import operator import random import warnings from functools import partial from itertools import chain, product from numbers import Number import numpy as np import torch import torch.autograd.forward_ad as fwAD from torch import inf, nan from torch.testing import make_tensor from torch.testing._internal.common_device_type import ( deviceCountAtLeast, dtypes, dtypesIfCPU, dtypesIfCUDA, expectedFailureMeta, instantiate_device_type_tests, onlyCPU, onlyCUDA, onlyNativeDeviceTypes, OpDTypes, ops, precisionOverride, skipIf, skipMeta, ) from torch.testing._internal.common_dtype import ( all_types_and, all_types_and_complex_and, complex_types, floating_and_complex_types, floating_types_and, get_all_int_dtypes, get_all_math_dtypes, integral_types, integral_types_and, ) from torch.testing._internal.common_methods_invocations import ( binary_ufuncs, binary_ufuncs_and_refs, generate_elementwise_binary_broadcasting_tensors, generate_elementwise_binary_extremal_value_tensors, generate_elementwise_binary_large_value_tensors, generate_elementwise_binary_small_value_tensors, generate_elementwise_binary_tensors, generate_elementwise_binary_with_scalar_and_type_promotion_samples, generate_elementwise_binary_with_scalar_samples, ) from torch.testing._internal.common_utils import ( gradcheck, iter_indices, numpy_to_torch_dtype_dict, run_tests, set_default_dtype, skipIfTorchDynamo, slowTest, TEST_SCIPY, TestCase, torch_to_numpy_dtype_dict, xfailIfTorchDynamo, ) if TEST_SCIPY: import scipy.integrate import scipy.special # TODO: update to use opinfos consistently class TestBinaryUfuncs(TestCase): # Generic tests for elementwise binary (AKA binary universal (u) functions (funcs)) # TODO: below contiguous tensor results are compared with a variety of noncontiguous results. # It would be interesting to have the lhs and rhs have different discontiguities. # Helper for comparing torch tensors and NumPy arrays # TODO: should this or assertEqual also validate that strides are equal? def assertEqualHelper( self, actual, expected, msg, *, dtype, exact_dtype=True, **kwargs ): assert isinstance(actual, torch.Tensor) # Some NumPy functions return scalars, not arrays if isinstance(expected, Number): self.assertEqual(actual.item(), expected, msg=msg, **kwargs) elif isinstance(expected, np.ndarray): # Handles exact dtype comparisons between arrays and tensors if exact_dtype: # Allows array dtype to be float32 when comparing with bfloat16 tensors # since NumPy doesn't support the bfloat16 dtype # Also ops like scipy.special.erf, scipy.special.erfc, etc, promote float16 # to float32 if expected.dtype == np.float32: assert actual.dtype in ( torch.float16, torch.bfloat16, torch.float32, ) else: assert expected.dtype == torch_to_numpy_dtype_dict[actual.dtype] self.assertEqual( actual, torch.from_numpy(expected).to(actual.dtype), msg, exact_device=False, **kwargs, ) else: self.assertEqual(actual, expected, msg, exact_device=False, **kwargs) # Tests that the function and its (array-accepting) reference produce the same # values on given tensors def _test_reference_numerics(self, dtype, op, gen, equal_nan=True): def _helper_reference_numerics( expected, actual, msg, exact_dtype, equal_nan=True ): if not torch.can_cast( numpy_to_torch_dtype_dict[expected.dtype.type], dtype ): exact_dtype = False if dtype is torch.bfloat16 and expected.dtype == np.float32: # Ref: https://github.com/pytorch/pytorch/blob/master/torch/testing/_internal/common_utils.py#L1149 self.assertEqualHelper( actual, expected, msg, dtype=dtype, exact_dtype=exact_dtype, rtol=16e-3, atol=1e-5, ) else: self.assertEqualHelper( actual, expected, msg, dtype=dtype, equal_nan=equal_nan, exact_dtype=exact_dtype, ) for sample in gen: # Each sample input acquired from the generator is just one lhs tensor # and one rhs tensor l = sample.input r = sample.args[0] numpy_sample = sample.numpy() l_numpy = numpy_sample.input r_numpy = numpy_sample.args[0] actual = op(l, r) expected = op.ref(l_numpy, r_numpy) # Crafts a custom error message for smaller, printable tensors def _numel(x): if isinstance(x, torch.Tensor): return x.numel() # Assumes x is a scalar return 1 if _numel(l) <= 100 and _numel(r) <= 100: msg = ( "Failed to produce expected results! Input lhs tensor was" f" {l}, rhs tensor was {r}, torch result is {actual}, and reference result is" f" {expected}." ) else: msg = None exact_dtype = True if isinstance(actual, torch.Tensor): _helper_reference_numerics( expected, actual, msg, exact_dtype, equal_nan ) else: for x, y in zip(expected, actual): # testing multi-outputs results _helper_reference_numerics(x, y, msg, exact_dtype, equal_nan) # The following tests only apply to elementwise binary operators with references binary_ufuncs_with_references = list( filter(lambda op: op.ref is not None and op.ref is not None, binary_ufuncs) ) @ops(binary_ufuncs_with_references) def test_reference_numerics(self, device, dtype, op): gen = generate_elementwise_binary_tensors(op, device=device, dtype=dtype) self._test_reference_numerics(dtype, op, gen, equal_nan=True) @ops(binary_ufuncs_with_references) def test_reference_numerics_small_values(self, device, dtype, op): if dtype is torch.bool: self.skipTest("Doesn't support bool!") gen = generate_elementwise_binary_small_value_tensors( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) @ops( binary_ufuncs_with_references, allowed_dtypes=( torch.int16, torch.int32, torch.int64, torch.float16, torch.bfloat16, torch.float32, torch.float64, torch.complex64, torch.complex128, ), ) def test_reference_numerics_large_values(self, device, dtype, op): gen = generate_elementwise_binary_large_value_tensors( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) @ops( binary_ufuncs_with_references, allowed_dtypes=( torch.float16, torch.bfloat16, torch.float32, torch.float64, torch.complex64, torch.complex128, ), ) def test_reference_numerics_extremal_values(self, device, dtype, op): gen = generate_elementwise_binary_extremal_value_tensors( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) # tests broadcasting and noncontiguous broadcasting behavior @ops( binary_ufuncs_with_references, allowed_dtypes=( torch.long, torch.float32, ), ) def test_broadcasting(self, device, dtype, op): gen = generate_elementwise_binary_broadcasting_tensors( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) @ops( binary_ufuncs_with_references, allowed_dtypes=(torch.long, torch.float32, torch.complex64), ) def test_scalar_support(self, device, dtype, op): gen = generate_elementwise_binary_with_scalar_samples( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) gen = generate_elementwise_binary_with_scalar_and_type_promotion_samples( op, device=device, dtype=dtype ) self._test_reference_numerics(dtype, op, gen, equal_nan=True) @ops(binary_ufuncs) def test_contig_vs_every_other(self, device, dtype, op): lhs = make_tensor( (1026,), device=device, dtype=dtype, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( (1026,), device=device, dtype=dtype, **op.rhs_make_tensor_kwargs ) lhs_non_contig = lhs[::2] rhs_non_contig = rhs[::2] self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertFalse(lhs_non_contig.is_contiguous()) self.assertFalse(rhs_non_contig.is_contiguous()) expected = op(lhs, rhs)[::2] actual = op(lhs_non_contig, rhs_non_contig) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_contig_vs_transposed(self, device, dtype, op): lhs = make_tensor( (789, 357), device=device, dtype=dtype, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( (789, 357), device=device, dtype=dtype, **op.rhs_make_tensor_kwargs ) lhs_non_contig = lhs.T rhs_non_contig = rhs.T self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertFalse(lhs_non_contig.is_contiguous()) self.assertFalse(rhs_non_contig.is_contiguous()) expected = op(lhs, rhs).T actual = op(lhs_non_contig, rhs_non_contig) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_non_contig(self, device, dtype, op): shapes = ((5, 7), (1024,)) for shape in shapes: lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) lhs_non_contig = torch.empty(shape + (2,), device=device, dtype=dtype)[ ..., 0 ] lhs_non_contig.copy_(lhs) rhs_non_contig = torch.empty(shape + (2,), device=device, dtype=dtype)[ ..., 0 ] rhs_non_contig.copy_(rhs) self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertFalse(lhs_non_contig.is_contiguous()) self.assertFalse(rhs_non_contig.is_contiguous()) expected = op(lhs, rhs) actual = op(lhs_non_contig, rhs_non_contig) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_non_contig_index(self, device, dtype, op): shape = (2, 2, 1, 2) lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) lhs_non_contig = lhs[:, 1, ...] lhs = lhs_non_contig.contiguous() rhs_non_contig = rhs[:, 1, ...] rhs = rhs_non_contig.contiguous() self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertFalse(lhs_non_contig.is_contiguous()) self.assertFalse(rhs_non_contig.is_contiguous()) expected = op(lhs, rhs) actual = op(lhs_non_contig, rhs_non_contig) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_non_contig_expand(self, device, dtype, op): shapes = [(1, 3), (1, 7), (5, 7)] for shape in shapes: lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) lhs_non_contig = lhs.clone().expand(3, -1, -1) rhs_non_contig = rhs.clone().expand(3, -1, -1) self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertFalse(lhs_non_contig.is_contiguous()) self.assertFalse(rhs_non_contig.is_contiguous()) expected = op(lhs, rhs) actual = op(lhs_non_contig, rhs_non_contig) for i in range(3): self.assertEqual(expected, actual[i]) @ops(binary_ufuncs) def test_contig_size1(self, device, dtype, op): shape = (5, 100) lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) lhs = lhs[:1, :50] lhs_alt = torch.empty(lhs.size(), device=device, dtype=dtype) lhs_alt.copy_(lhs) rhs = rhs[:1, :50] rhs_alt = torch.empty(rhs.size(), device=device, dtype=dtype) rhs_alt.copy_(rhs) self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertTrue(lhs_alt.is_contiguous()) self.assertTrue(rhs_alt.is_contiguous()) expected = op(lhs, rhs) actual = op(lhs_alt, rhs_alt) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_contig_size1_large_dim(self, device, dtype, op): shape = (5, 2, 3, 1, 4, 5, 3, 2, 1, 2, 3, 4) lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) lhs = lhs[:1, :, :, :, :, :, :, :, :, :, :, :] lhs_alt = torch.empty(lhs.size(), device=device, dtype=dtype) lhs_alt.copy_(lhs) rhs = rhs[:1, :, :, :, :, :, :, :, :, :, :, :] rhs_alt = torch.empty(rhs.size(), device=device, dtype=dtype) rhs_alt.copy_(rhs) self.assertTrue(lhs.is_contiguous()) self.assertTrue(rhs.is_contiguous()) self.assertTrue(lhs_alt.is_contiguous()) self.assertTrue(rhs_alt.is_contiguous()) expected = op(lhs, rhs) actual = op(lhs_alt, rhs_alt) self.assertEqual(expected, actual) @ops(binary_ufuncs) def test_batch_vs_slicing(self, device, dtype, op): shape = (32, 512) lhs = make_tensor( shape, dtype=dtype, device=device, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape, dtype=dtype, device=device, **op.rhs_make_tensor_kwargs ) expected = op(lhs, rhs) actual = [] for idx in range(32): actual.append(op(lhs[idx], rhs[idx])) actual = torch.stack(actual) self.assertEqual(expected, actual) # Tests that elementwise binary operators participate in type promotion properly # NOTE: because the cross-product of all possible type promotion tests is huge, this # just spot checks some handwritten cases. # NOTE: It may be possible to refactor this test into something simpler @ops(binary_ufuncs_and_refs, dtypes=OpDTypes.none) def test_type_promotion(self, device, op): supported_dtypes = op.supported_dtypes(torch.device(device).type) make_lhs = partial( make_tensor, (5,), device=device, **op.lhs_make_tensor_kwargs ) make_rhs = partial( make_tensor, (5,), device=device, **op.rhs_make_tensor_kwargs ) make_rhs_scalar_tensor = partial( make_tensor, (), device="cpu", **op.rhs_make_tensor_kwargs ) def _supported(dtypes): return all(x in supported_dtypes for x in dtypes) # int x int type promotion if _supported((torch.int16, torch.int32, torch.int64)): lhs_i16 = make_lhs(dtype=torch.int16) lhs_i32 = make_lhs(dtype=torch.int32) lhs_i64 = make_lhs(dtype=torch.int64) rhs_i16 = make_rhs(dtype=torch.int16) rhs_i32 = make_rhs(dtype=torch.int32) rhs_i64 = make_rhs(dtype=torch.int64) if op.promotes_int_to_float: default_dtype = torch.get_default_dtype() self.assertEqual(op(lhs_i16, rhs_i32).dtype, default_dtype) self.assertEqual( op(lhs_i16, rhs_i32), op(lhs_i16.to(default_dtype), rhs_i32.to(default_dtype)), ) self.assertEqual(op(lhs_i32, rhs_i64).dtype, default_dtype) self.assertEqual( op(lhs_i32, rhs_i64), op(lhs_i32.to(default_dtype), rhs_i64.to(default_dtype)), ) elif op.always_returns_bool: self.assertEqual(op(lhs_i16, rhs_i32).dtype, torch.bool) self.assertEqual(op(lhs_i32, rhs_i64).dtype, torch.bool) else: # standard type promotion self.assertEqual(op(lhs_i16, rhs_i32).dtype, torch.int32) self.assertEqual( op(lhs_i16, rhs_i32), op(lhs_i16.to(torch.int32), rhs_i32) ) self.assertEqual(op(lhs_i32, rhs_i64).dtype, torch.int64) self.assertEqual( op(lhs_i32, rhs_i64), op(lhs_i32.to(torch.int64), rhs_i64) ) if op.supports_out: if not op.promotes_int_to_float: # Integers can be safely cast to other integer types out = torch.empty_like(lhs_i64) self.assertEqual(op(lhs_i16, rhs_i32, out=out).dtype, torch.int64) self.assertEqual(op(lhs_i16, rhs_i32), out, exact_dtype=False) out = torch.empty_like(lhs_i16) self.assertEqual(op(lhs_i32, rhs_i64, out=out).dtype, torch.int16) else: # Float outs cannot be safely cast to integer types with self.assertRaisesRegex(RuntimeError, "can't be cast"): op(lhs_i16, rhs_i32, out=torch.empty_like(lhs_i64)) if not op.always_returns_bool: # Neither integer nor float outs can be cast to bool with self.assertRaisesRegex(RuntimeError, "can't be cast"): op( lhs_i16, rhs_i32, out=torch.empty_like(lhs_i64, dtype=torch.bool), ) # All these output types can be cast to any float or complex type out = torch.empty_like(lhs_i64, dtype=torch.float16) self.assertEqual(op(lhs_i16, rhs_i32, out=out).dtype, torch.float16) out = torch.empty_like(lhs_i64, dtype=torch.bfloat16) self.assertEqual(op(lhs_i16, rhs_i32, out=out).dtype, torch.bfloat16) out = torch.empty_like(lhs_i64, dtype=torch.float32) self.assertEqual(op(lhs_i16, rhs_i32, out=out).dtype, torch.float32) self.assertEqual(op(lhs_i16, rhs_i32), out, exact_dtype=False) out = torch.empty_like(lhs_i64, dtype=torch.complex64) self.assertEqual(op(lhs_i16, rhs_i32, out=out).dtype, torch.complex64) self.assertEqual(op(lhs_i16, rhs_i32), out, exact_dtype=False) # float x float type promotion if _supported((torch.float32, torch.float64)): lhs_f32 = make_lhs(dtype=torch.float32) lhs_f64 = make_lhs(dtype=torch.float64) rhs_f32 = make_rhs(dtype=torch.float32) rhs_f64 = make_rhs(dtype=torch.float64) if op.always_returns_bool: self.assertEqual(op(lhs_f32, rhs_f64).dtype, torch.bool) else: # normal float type promotion self.assertEqual(op(lhs_f32, rhs_f64).dtype, torch.float64) self.assertEqual( op(lhs_f32, rhs_f64), op(lhs_f32.to(torch.float64), rhs_f64) ) if op.supports_out: # All these output types can be cast to any float or complex type out = torch.empty_like(lhs_f64, dtype=torch.float16) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.float16) out = torch.empty_like(lhs_f64, dtype=torch.bfloat16) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.bfloat16) self.assertEqual(op(lhs_f32, rhs_f64), out, exact_dtype=False) out = torch.empty_like(lhs_f64, dtype=torch.float32) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.float32) self.assertEqual(op(lhs_f32, rhs_f64), out, exact_dtype=False) out = torch.empty_like(lhs_f64, dtype=torch.complex64) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.complex64) self.assertEqual(op(lhs_f32, rhs_f64), out, exact_dtype=False) if not op.always_returns_bool: # float outs can't be cast to an integer dtype with self.assertRaisesRegex(RuntimeError, "can't be cast"): op( lhs_f32, rhs_f64, out=torch.empty_like(lhs_f64, dtype=torch.int64), ) else: # bool outs can be cast to an integer dtype out = torch.empty_like(lhs_f64, dtype=torch.int64) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.int64) self.assertEqual(op(lhs_f32, rhs_f64), out, exact_dtype=False) # complex x complex type promotion if _supported((torch.complex64, torch.complex128)): lhs_c64 = make_lhs(dtype=torch.complex64) lhs_c128 = make_lhs(dtype=torch.complex128) rhs_c64 = make_rhs(dtype=torch.complex64) rhs_c128 = make_rhs(dtype=torch.complex128) if op.always_returns_bool: self.assertEqual(op(lhs_c64, lhs_c128).dtype, torch.bool) else: # normal complex type promotion self.assertEqual(op(lhs_c64, rhs_c128).dtype, torch.complex128) self.assertEqual( op(lhs_c64, rhs_c128), op(lhs_c64.to(torch.complex128), rhs_c128) ) if op.supports_out: # All these output types can be cast to any or complex type out = torch.empty_like(lhs_c64, dtype=torch.complex64) self.assertEqual(op(lhs_c64, rhs_c128, out=out).dtype, torch.complex64) result = op(lhs_c64, rhs_c128) self.assertEqual(result, out.to(result.dtype)) if not op.always_returns_bool: # complex outs can't be cast to float types with self.assertRaisesRegex(RuntimeError, "can't be cast"): op( lhs_c64, rhs_c128, out=torch.empty_like(lhs_c64, dtype=torch.float64), ) # complex outs can't be cast to an integer dtype with self.assertRaisesRegex(RuntimeError, "can't be cast"): op( lhs_c64, rhs_c128, out=torch.empty_like(lhs_c64, dtype=torch.int64), ) else: # bool outs can be cast to a float type out = torch.empty_like(lhs_c64, dtype=torch.float64) self.assertEqual( op(lhs_c64, rhs_c128, out=out).dtype, torch.float64 ) self.assertEqual(op(lhs_c64, rhs_c128), out, exact_dtype=False) # bool outs can be cast to an integer dtype out = torch.empty_like(lhs_f64, dtype=torch.int64) self.assertEqual(op(lhs_f32, rhs_f64, out=out).dtype, torch.int64) self.assertEqual(op(lhs_f32, rhs_f64), out, exact_dtype=False) # int x float type promotion # Note: float type is the result dtype if _supported((torch.long, torch.float32)): lhs_i64 = make_lhs(dtype=torch.int64) rhs_f32 = make_rhs(dtype=torch.float32) result = op(lhs_i64, rhs_f32) expected_dtype = torch.float32 if not op.always_returns_bool else torch.bool self.assertEqual(result.dtype, expected_dtype) # float x complex type promotion # Note: complex type with highest "value type" is the result dtype if _supported((torch.float64, torch.complex64)): lhs_f64 = make_lhs(dtype=torch.float64) rhs_c64 = make_rhs(dtype=torch.complex64) result = op(lhs_f64, rhs_c64) expected_dtype = ( torch.complex128 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # int x float scalar type promotion # Note: default float dtype is the result dtype if _supported((torch.int64, torch.float32)) and op.supports_rhs_python_scalar: lhs_i64 = make_lhs(dtype=torch.int64) rhs_f_scalar = 1.0 result = op(lhs_i64, rhs_f_scalar) expected_dtype = ( torch.get_default_dtype() if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # repeats with a scalar float tensor, which should set the dtype rhs_f32_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.float32) result = op(lhs_i64, rhs_f32_scalar_tensor) expected_dtype = torch.float32 if not op.always_returns_bool else torch.bool self.assertEqual(result.dtype, expected_dtype) # Additional test with double if _supported((torch.float64,)): rhs_f64_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.float64) result = op(lhs_i64, rhs_f64_scalar_tensor) expected_dtype = ( torch.float64 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # float x complex scalar type promotion # Note: result dtype is complex with highest "value type" among all tensors if ( _supported((torch.float32, torch.complex64)) and op.supports_rhs_python_scalar ): lhs_f32 = make_lhs(dtype=torch.float32) rhs_c_scalar = complex(1, 1) result = op(lhs_f32, rhs_c_scalar) expected_dtype = ( torch.complex64 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # repeats with a scalar complex tensor rhs_c64_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.complex64) result = op(lhs_f32, rhs_c64_scalar_tensor) expected_dtype = ( torch.complex64 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # Additional test with complexdouble if _supported((torch.complex128,)): rhs_c128_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.complex128) result = op(lhs_f32, rhs_c128_scalar_tensor) # Value type of 1D+ Tensor (lhs_f32) takes priority over scalar tensor (rhs_c128). expected_dtype = ( torch.complex64 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # float x float scalar tensor # Note: result dtype is the type of the float tensor if _supported((torch.float32, torch.float64)) and op.supports_rhs_python_scalar: lhs_f32 = make_lhs(dtype=torch.float32) rhs_f64_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.float64) result = op(lhs_f32, rhs_f64_scalar_tensor) expected_dtype = torch.float32 if not op.always_returns_bool else torch.bool self.assertEqual(result.dtype, expected_dtype) # complex x complex scalar tensor # Note: result dtype is the type of the complex tensor if ( _supported((torch.complex64, torch.complex128)) and op.supports_rhs_python_scalar ): lhs_c64 = make_lhs(dtype=torch.complex64) rhs_c128_scalar_tensor = make_rhs_scalar_tensor(dtype=torch.complex128) result = op(lhs_c64, rhs_c128_scalar_tensor) expected_dtype = ( torch.complex64 if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # scalar x scalar # Note: result dtype is default float type if op.supports_two_python_scalars and _supported((torch.long, torch.float32)): rhs_f_scalar = 2.0 for lhs in (1, 1.0): result = op(lhs, rhs_f_scalar) expected_dtype = ( torch.get_default_dtype() if not op.always_returns_bool else torch.bool ) self.assertEqual(result.dtype, expected_dtype) # TODO: move to error input test @ops(binary_ufuncs, allowed_dtypes=(torch.float32,)) def test_not_broadcastable(self, device, dtype, op): for shape_lhs, shape_rhs in ( ((2,), (3,)), ((3, 1), (2, 1)), ((1, 3, 2), (3,)), ((3, 1, 2), (2, 1, 2)), ): lhs = make_tensor( shape_lhs, device=device, dtype=dtype, **op.lhs_make_tensor_kwargs ) rhs = make_tensor( shape_rhs, device=device, dtype=dtype, **op.rhs_make_tensor_kwargs ) try: broadcasted_shape = op(lhs, rhs).shape except RuntimeError: continue msg = ( f"On {device}, torch.{op.name} broadcasts inputs shapes {shape_lhs} and {shape_rhs} into " f"{broadcasted_shape}, although they are not broadcastable." ) raise AssertionError(msg) def test_add_broadcast_empty(self, device): # empty + empty self.assertRaises( RuntimeError, lambda: torch.randn(5, 0, device=device) + torch.randn(0, 5, device=device), ) self.assertEqual( torch.randn(5, 0, device=device), torch.randn(0, device=device) + torch.randn(5, 0, device=device), ) self.assertEqual( torch.randn(5, 0, 0, device=device), torch.randn(0, device=device) + torch.randn(5, 0, 1, device=device), ) # scalar + empty self.assertEqual( torch.randn(5, 0, 6, device=device), torch.randn((), device=device) + torch.randn(5, 0, 6, device=device), ) # non-empty, empty self.assertEqual( torch.randn(0, device=device), torch.randn(0, device=device) + torch.randn(1, device=device), ) self.assertEqual( torch.randn(0, 7, 0, 6, 5, 0, 7, device=device), torch.randn(0, 7, 0, 6, 5, 0, 1, device=device) + torch.randn(1, 1, 5, 1, 7, device=device), ) self.assertRaises( RuntimeError, lambda: torch.randn(7, 0, device=device) + torch.randn(2, 1, device=device), ) def test_addcmul_scalars_as_floats(self, device): # zero-dim variables that don't require grad should bind to scalar arguments x = torch.tensor(2.0) y = torch.tensor(3.0, device=device) # 3 + (3 * 3) * 2 self.assertEqual(y.addcmul(y, y, value=x), 21) x = torch.tensor(2.0, requires_grad=True) self.assertRaises(Exception, lambda: y.addcmul(y, y, value=x)) # Tests that the binary operators and, or, and xor (as well as their reflected and inplace versions) # work properly (AKA &, ||, ^ and &=, |=, ^=) @dtypes(*integral_types_and(torch.bool)) def test_bitwise_ops(self, device, dtype): # Tensor x Tensor and Tensor x Scalar ops ops = ( operator.and_, operator.iand, operator.or_, operator.ior, operator.xor, operator.ixor, ) inplace_ops = (operator.iand, operator.ior, operator.ixor) shapes = ((5,), (15, 15), (500, 500)) for op, shape in itertools.product(ops, shapes): # Tests tensor x tensor case a = make_tensor(shape, device=device, dtype=dtype) b = make_tensor(shape, device=device, dtype=dtype) a_np = a.cpu().clone().numpy() b_np = b.cpu().clone().numpy() self.assertEqual(op(a, b), op(a_np, b_np)) # Tests tensor x scalar case a = make_tensor(shape, device=device, dtype=dtype) b_scalar = make_tensor((), device="cpu", dtype=dtype).item() a_np = a.cpu().clone().numpy() self.assertEqual(op(a, b_scalar), op(a_np, b_scalar)) # Tests scalar x tensor case a_scalar = make_tensor((), device="cpu", dtype=dtype).item() b = make_tensor(shape, device=device, dtype=dtype) b_np = b.cpu().clone().numpy() self.assertEqual(op(a_scalar, b), op(a_scalar, b_np)) # Tests scalar x tensor case (for ops which aren't inplace) if op in inplace_ops: # Tests tensor x tensor case a = make_tensor(shape, device=device, dtype=dtype) b = make_tensor(shape, device=device, dtype=dtype) a_np = a.cpu().clone().numpy() b_np = b.cpu().clone().numpy() op(a, b) op(a_np, b_np) self.assertEqual(a, a_np) # Tests tensor x scalar case a = make_tensor(shape, device=device, dtype=dtype) b_scalar = make_tensor((), device="cpu", dtype=dtype).item() a_np = a.cpu().clone().numpy() op(a, b_scalar) op(a_np, b_scalar) self.assertEqual(a, a_np) def test_inplace_division(self, device): t = torch.rand(5, 5, device=device) id_before = id(t) t /= 2 id_after = id(t) self.assertEqual(id_before, id_after) @dtypes(*all_types_and(torch.half, torch.bfloat16)) def test_div_rounding_modes(self, device, dtype): if dtype.is_floating_point: low, high = -10.0, 10.0 else: info = torch.iinfo(dtype) low, high = info.min, info.max a = make_tensor((100,), dtype=dtype, device=device, low=low, high=high) b = make_tensor((100,), dtype=dtype, device=device, low=low, high=high) # Avoid division by zero so we can test (a / b) * b == a if dtype.is_floating_point: eps = 0.1 b[(-eps < b) & (b < eps)] = eps else: b[b == 0] = 1 if not dtype.is_floating_point: # floor(a / b) * b can be < a, so fixup slightly to avoid underflow a = torch.where(a < 0, a + b, a) d_true = torch.divide(a, b, rounding_mode=None) self.assertTrue(d_true.is_floating_point()) self.assertEqual(d_true * b, a.to(d_true.dtype)) d_floor = torch.divide(a, b, rounding_mode="floor") if dtype not in (torch.bfloat16, torch.half): self.assertEqual(d_floor * b + torch.remainder(a, b), a) else: self.assertEqual( d_floor * b + torch.remainder(a.float(), b.float()), a, exact_dtype=False, ) d_trunc = torch.divide(a, b, rounding_mode="trunc") rounding_unsupported = ( dtype == torch.half and device != "cuda" or dtype == torch.bfloat16 and device != "cpu" ) d_ref = d_true.float() if rounding_unsupported else d_true self.assertEqual(d_trunc, d_ref.trunc().to(dtype)) @dtypes(*floating_types_and(torch.bfloat16, torch.float16)) def test_floor_div_extremal(self, device, dtype): for num, denom, shape in itertools.product( [torch.finfo(dtype).max * 0.7], [0.5, -0.5, 0.0], [(), (32,)], # Scalar and vectorized ): a = torch.full(shape, num, dtype=dtype, device=device) b = torch.full(shape, denom, dtype=dtype, device=device) ref = np.floor_divide(num, denom).item() if ref > torch.finfo(dtype).max: ref = np.inf elif ref < torch.finfo(dtype).min: ref = -np.inf expect = torch.full(shape, ref, dtype=dtype, device=device) actual = torch.div(a, b, rounding_mode="floor") self.assertEqual(expect, actual) @dtypes(torch.bfloat16, torch.half, torch.float32, torch.float64) def test_div_rounding_nonfinite(self, device, dtype): # Compare division of special floating point values against NumPy num = torch.tensor( [1.0, -1.0, 0, 0.1, -0.1, np.pi, -np.pi, np.inf, -np.inf, np.nan], dtype=dtype, device=device, ) # Divide by zero is tested separately denom = num[num != 0] a, b = num[None, :].clone(), denom[:, None].clone() # Compare bfloat16 against NumPy float exact_dtype = dtype != torch.bfloat16 if exact_dtype: an, bn = a.cpu().numpy(), b.cpu().numpy() else: an, bn = a.float().cpu().numpy(), b.float().cpu().numpy() for mode, np_ref in ((None, np.true_divide), ("floor", np.floor_divide)): expect = np_ref(an, bn) kwargs = dict(rounding_mode=mode) if mode is not None else {} with set_default_dtype(torch.double): actual = torch.divide(a, b, **kwargs) self.assertEqual( actual, torch.from_numpy(expect), exact_device=False, exact_dtype=exact_dtype, ) # Compare contiguous (likely vectorized) against non-contiguous (not vectorized) a_noncontig = torch.empty([2 * i for i in a.shape], dtype=dtype, device=device)[ ::2, ::2 ] a_noncontig[:] = a b_noncontig = torch.empty([2 * i for i in b.shape], dtype=dtype, device=device)[ ::2, ::2 ] b_noncontig[:] = b for rounding_mode in (None, "trunc", "floor"): expect = torch.divide(a_noncontig, b_noncontig, rounding_mode=rounding_mode) actual = torch.divide(a, b, rounding_mode=rounding_mode) self.assertEqual(actual, expect) @dtypes(torch.bfloat16, torch.half, torch.float32, torch.float64) def test_divide_by_zero_rounding(self, device, dtype): a = torch.tensor( [1.0, -1.0, 0, 0.1, -0.1, np.pi, -np.pi, np.inf, -np.inf, np.nan], dtype=dtype, ) exact_dtype = dtype != torch.bfloat16 if exact_dtype: an = a.cpu().numpy() else: an = a.float().cpu().numpy() zero = torch.zeros_like(a) # NOTE: NumPy's floor_divide rounding changed in 1.20.0 to be consistent with divide expect = np.divide(an, 0) for rounding_mode in (None, "floor"): # CPU scalar actual = torch.divide(a, 0, rounding_mode=rounding_mode) self.assertEqual(actual, expect, exact_dtype=exact_dtype) # Device tensor actual = torch.divide(a, zero, rounding_mode=rounding_mode) self.assertEqual(actual, expect, exact_dtype=exact_dtype) @dtypes(*all_types_and(torch.half)) def test_div_rounding_numpy(self, device, dtype): info = torch.finfo(dtype) if dtype.is_floating_point else torch.iinfo(dtype) low, high = info.min, info.max # Compare division of random values against NumPy a = make_tensor((4096,), dtype=dtype, device=device, low=low, high=high) b = make_tensor((4096,), dtype=dtype, device=device, low=low, high=high) # Avoid division by zero which raises for integers and, for floats, # NumPy 1.20 changed floor_divide to follow IEEE rules for inf/nan # after dividing by zero. b[b == 0] = 1 # Compare bfloat16 against NumPy float exact_dtype = dtype != torch.bfloat16 if exact_dtype: an, bn = a.cpu().numpy(), b.cpu().numpy() else: an, bn = a.float().cpu().numpy(), b.float().cpu().numpy() for mode, np_ref in ( (None, np.true_divide), ("floor", np.floor_divide), ("trunc", lambda a, b: np.trunc(np.true_divide(a, b)).astype(a.dtype)), ): expect = torch.from_numpy(np_ref(an, bn)) kwargs = dict(rounding_mode=mode) if mode is not None else {} # Contiguous (likely vectorized) with set_default_dtype(torch.double): actual = torch.divide(a, b, **kwargs) self.assertEqual( actual, expect, exact_device=False, exact_dtype=exact_dtype ) # Non-contiguous (not vectorized) expect = expect[::2] with set_default_dtype(torch.double): actual = torch.divide(a[::2], b[::2], **kwargs) self.assertEqual( actual, expect, exact_device=False, exact_dtype=exact_dtype ) @dtypes(*complex_types()) def test_complex_div_underflow_overflow(self, device, dtype): # test to make sure the complex division does not produce underflow or overflow # in the intermediate of its calculations # NOTE: the calculation still produces an error if the number is greater than # finfo.max / 2, but hopefully people realized that it's a dangerous region to work with finfo = torch.finfo(dtype) nom_lst = [ complex(finfo.min / 2, finfo.min / 2), complex(finfo.max / 2, finfo.max / 2), complex(finfo.tiny, finfo.tiny), complex(finfo.tiny, 0.0), complex(0.0, 0.0), ] denom_lst = [ complex(finfo.min / 2, finfo.min / 2), complex(finfo.max / 2, finfo.max / 2), complex(finfo.tiny, finfo.tiny), complex(0.0, finfo.tiny), complex(finfo.tiny, finfo.tiny), ] expected_lst = [ complex(1.0, 0.0), complex(1.0, 0.0), complex(1.0, 0.0), complex(0.0, -1.0), complex(0.0, 0.0), ] nom = torch.tensor(nom_lst, dtype=dtype, device=device) denom = torch.tensor(denom_lst, dtype=dtype, device=device) expected = torch.tensor(expected_lst, dtype=dtype, device=device) res = nom / denom self.assertEqual(res, expected) # Tests that trying to add, inplace, a CUDA tensor to a CPU tensor # throws the correct error message @onlyCUDA def test_cross_device_inplace_error_msg(self, device): a = torch.tensor(2.0) b = torch.tensor(2.0, device=device) with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device" ): a += b # TODO: refactor this test into a more generic one, it's parked here currently @onlyNativeDeviceTypes def test_out_resize_warning(self, device): a = torch.tensor((1, 2, 3), device=device, dtype=torch.float32) b = torch.tensor((4, 5, 6), device=device, dtype=torch.float32) unary_inputs = (a,) binary_inputs = (a, b) unary_ops = (torch.ceil, torch.exp) binary_ops = (torch.add, torch.sub) for op in unary_ops + binary_ops: with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") inputs = unary_inputs if op in unary_ops else binary_inputs # No warnings op(*inputs, out=torch.empty(3, device=device)) op(*inputs, out=torch.empty(0, device=device)) self.assertEqual(len(w), 0) # Cases that throw warnings op(*inputs, out=torch.empty(2, device=device)) self.assertEqual(len(w), 1) # test that multi-d out doesn't trigger segfault arg1 = (torch.ones(2, 1, device=device), torch.ones(1, device=device)) arg2 = (torch.ones(2, device=device), torch.ones(1, 1, device=device)) outs = ( torch.ones(2, 1, 1, 1, device=device), torch.ones(2, 2, 2, 2, device=device), ) for a1, a2, o in zip(arg1, arg2, outs): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") torch.mul(a1, a2, out=o) self.assertEqual(len(w), 1) # Verifies that the inplace dunders (like idiv) actually are in place @expectedFailureMeta # UserWarning not triggered @onlyNativeDeviceTypes def test_inplace_dunders(self, device): t = torch.randn((1,), device=device) expected = t.data_ptr() t += 1 t -= 1 t *= 1 t /= 1 t **= 1 t //= 1 t %= 1 self.assertEqual(expected, t.data_ptr()) def check_internal_mem_overlap( self, inplace_op, num_inputs, dtype, device, expected_failure=False ): if isinstance(inplace_op, str): inplace_op = getattr(torch.Tensor, inplace_op) input = torch.randn(1, dtype=dtype, device=device).expand(3, 3) inputs = [input] + [torch.randn_like(input) for i in range(num_inputs - 1)] if not expected_failure: with self.assertRaisesRegex(RuntimeError, "single memory location"): inplace_op(*inputs) else: with self.assertRaises(AssertionError): with self.assertRaisesRegex(RuntimeError, "single memory location"): inplace_op(*inputs) def unary_check_input_output_mem_overlap( self, data, sz, op, expected_failure=False ): def _test(op, output, input): output_exp = torch.empty_like(output) op(input, out=output_exp) self.assertEqual(op(input, out=output), output_exp, msg=op.__name__) # output is identical to input: _test(op, output=data[0:sz], input=data[0:sz]) # output and input are independent: _test(op, output=data[0:sz], input=data[sz : 2 * sz]) # output partially overlaps with input: if not expected_failure: with self.assertRaisesRegex(RuntimeError, "unsupported operation"): _test(op, data[0:sz], data[1 : sz + 1]) else: with self.assertRaises(AssertionError): with self.assertRaisesRegex(RuntimeError, "unsupported operation"): _test(op, data[0:sz], data[1 : sz + 1]) def binary_check_input_output_mem_overlap(self, op, device, expected_failure=False): sz = 3 data = torch.randn(2 * sz, device=device) other = torch.randn(sz, device=device) self.unary_check_input_output_mem_overlap( data, sz, lambda input, out: op(other, input, out=out), expected_failure=expected_failure, ) self.unary_check_input_output_mem_overlap( data, sz, lambda input, out: op(input, other, out=out), expected_failure=expected_failure, ) # https://github.com/pytorch/pytorch/issues/126474 @xfailIfTorchDynamo @dtypes(torch.double) def test_binary_op_mem_overlap(self, device, dtype): ops = [ ("add", True, True, "cpu"), ("add", True, True, "cuda"), ("mul", True, True, "cpu"), ("mul", True, True, "cuda"), ("sub", True, True, "cpu"), ("sub", True, True, "cuda"), ("div", True, True, "cpu"), ("div", True, True, "cuda"), ("pow", True, True, "cpu"), ("pow", True, True, "cuda"), ("fmod", True, True, "cpu"), ("fmod", True, True, "cuda"), ("atan2", True, True, "cpu"), ("atan2", True, True, "cuda"), ("hypot", True, True, "cpu"), ("hypot", True, True, "cuda"), ("igamma", True, True, "cpu"), ("igamma", True, True, "cuda"), ("igammac", True, True, "cpu"), ("igammac", True, True, "cuda"), ("nextafter", True, True, "cpu"), ("nextafter", True, True, "cuda"), ("le", True, True, "cpu"), ("le", True, True, "cuda"), ("lt", True, True, "cpu"), ("lt", True, True, "cuda"), ("ge", True, True, "cpu"), ("ge", True, True, "cuda"), ("gt", True, True, "cpu"), ("gt", True, True, "cuda"), ("eq", True, True, "cpu"), ("eq", True, True, "cuda"), ("ne", True, True, "cpu"), ("ne", True, True, "cuda"), ("logical_and", True, True, "cpu"), ("logical_and", True, True, "cuda"), ("logical_or", True, True, "cpu"), ("logical_or", True, True, "cuda"), ("logical_xor", True, True, "cpu"), ("logical_xor", True, True, "cuda"), ] for ( fn, has_input_output_mem_overlap_check, has_internal_mem_overlap_check, dev, ) in ops: if dev != device: continue out_op = getattr(torch, fn) inplace_op = getattr(torch.Tensor, fn + "_") self.check_internal_mem_overlap( inplace_op, 2, dtype, device, expected_failure=not has_internal_mem_overlap_check, ) self.binary_check_input_output_mem_overlap( out_op, device, expected_failure=not has_input_output_mem_overlap_check ) def _do_pow_for_exponents(self, m1, exponents, pow_fn, atol): for num in exponents: if ( isinstance(num, int) and num < 0 and not m1.is_floating_point() and not m1.is_complex() ): with self.assertRaisesRegex( RuntimeError, r"Integers to negative integer powers are not allowed\.", ): torch.pow(m1[4], num) else: # base - tensor, exponent - number # contiguous res1 = torch.pow(m1[4], num) res2 = res1.clone().zero_() # `math.pow` has issues with complex exponentiation so we need to resort to normal `pow`. for i in range(res2.size(0)): res2[i] = pow_fn(m1[4][i], num) rtol = 0 if atol is not None else None self.assertEqual(res1, res2, atol=atol, rtol=rtol) # non-contiguous res1 = torch.pow(m1[:, 4], num) res2 = res1.clone().zero_() for i in range(res2.size(0)): res2[i] = pow_fn(m1[i, 4], num) self.assertEqual(res1, res2, atol=atol, rtol=rtol) # scalar ** tensor to enforce correct handling of dtypes for __rpow__(). expected_dtype = torch.result_type(num, m1) res1 = num ** m1[4] res2 = ( torch.tensor(num, dtype=expected_dtype, device=m1.device) ** m1[4] ) self.assertEqual(res1, res2) self.assertEqual(res1.dtype, expected_dtype) @dtypes(*all_types_and_complex_and(torch.half, torch.bfloat16)) def test_pow(self, device, dtype): m1 = torch.empty(0, dtype=dtype, device=device) if m1.is_floating_point() or m1.is_complex(): m1 = ( make_tensor((100, 100), low=0, high=1, dtype=dtype, device=device) + 0.5 ) else: # math.pow will overflow and throw exceptions for large integers range_high = 4 if dtype in (torch.int8, torch.uint8) else 10 m1 = make_tensor( (100, 100), low=1, high=range_high, dtype=dtype, device=device ) exponents = [-2.8, -2, -1, -0.5, 0, 0.5, 1, 2, 3, 4, 3.3, True, False] complex_exponents = [ -2.5j, -1.0j, 0j, 1.0j, 2.5j, 1.0 + 1.0j, -1.0 - 1.5j, 3.3j, ] if m1.is_complex(): self._do_pow_for_exponents(m1, exponents + complex_exponents, pow, 10e-4) else: self._do_pow_for_exponents(m1, exponents, math.pow, None) will_raise_error = ( dtype is torch.half and torch.device(device).type == "cpu" ) if will_raise_error: # On CPU, # Half Tensor with complex exponents leads to computation dtype # of ComplexHalf for which this ops is not supported yet with self.assertRaisesRegex( RuntimeError, "not implemented for 'ComplexHalf'" ): self._do_pow_for_exponents(m1, complex_exponents, pow, 10e-4) else: self._do_pow_for_exponents(m1, complex_exponents, pow, 10e-4) # base - number, exponent - tensor # contiguous res1 = torch.pow(3, m1[4]) res2 = res1.clone().zero_() for i in range(res2.size(0)): res2[i] = pow(3, m1[4, i]) self.assertEqual(res1, res2) # non-contiguous res1 = torch.pow(3, m1[:, 4]) res2 = res1.clone().zero_() for i in range(res2.size(0)): res2[i] = pow(3, m1[i][4]) self.assertEqual(res1, res2) # TODO: refactor all these tests using opinfos properly def _test_pow(self, base, exponent, np_exponent=None): if np_exponent is None: np_exponent = exponent def to_np(value): if isinstance(value, torch.Tensor): return value.cpu().numpy() return value try: np_res = np.power(to_np(base), to_np(np_exponent)) expected = ( torch.from_numpy(np_res) if isinstance(np_res, np.ndarray) else torch.tensor(np_res, dtype=base.dtype) ) except ValueError as e: err_msg = "Integers to negative integer powers are not allowed." self.assertEqual(str(e), err_msg) out = torch.empty_like(base) test_cases = [ lambda: base.pow(exponent), lambda: base.pow_(exponent), lambda: torch.pow(base, exponent), lambda: torch.pow(base, exponent, out=out), ] for test_case in test_cases: self.assertRaisesRegex(RuntimeError, err_msg, test_case) else: if isinstance(base, torch.Tensor): actual = base.pow(exponent) self.assertEqual(actual, expected.to(actual)) actual = base.clone() # When base is a 0-dim cpu tensor and exp is a cuda tensor, we exp `pow` to work but `pow_` to fail, since # `pow` will try to create the output tensor on a cuda device, but `pow_` needs to use the cpu tensor as the output if ( isinstance(exponent, torch.Tensor) and base.dim() == 0 and base.device.type == "cpu" and exponent.device.type == "cuda" ): regex = "Expected all tensors to be on the same device, but found at least two devices, cuda.* and cpu!" self.assertRaisesRegex(RuntimeError, regex, base.pow_, exponent) elif torch.can_cast(torch.result_type(base, exponent), base.dtype): actual2 = actual.pow_(exponent) self.assertEqual(actual, expected) self.assertEqual(actual2, expected) else: self.assertRaisesRegex( RuntimeError, "Found dtype \\w+ but expected \\w+", lambda: actual.pow_(exponent), ) actual = torch.pow(base, exponent) self.assertEqual(actual, expected.to(actual)) actual2 = torch.pow(base, exponent, out=actual) self.assertEqual(actual, expected.to(actual)) self.assertEqual(actual2, expected.to(actual)) # We can potentially merge this into OpInfo, but one blocker is that the # first input must be a scalar. It is not as simple as just wrapping this in # a lambada that switches the inputs, because we also want to test samples inputs # where the second input is a scalar. The wrapper would need some more logic. def test_pow_scalar_base(self, device): a = ( torch.arange(1, 13, dtype=torch.double, device=device) .view(3, 4) .requires_grad_() ) gradcheck(lambda a: torch.pow(2, a), (a,)) # Tests pow() for integral, floating-type tensors, with integral, floating-type # exponents (tensor or scalar), respectively. noncontiguous tensors are also tested. def test_int_and_float_pow(self, device): def _test_int_and_float_pow(dt, low, high, dev): test_cases = ( ((4, 4), 0, (4, 1)), ((3, 1), 4, (3, 1)), ((2,), 4, (1,)), ((1,), 2, ()), ((513, 513), 4, (513,)), ((5, 5, 5), 5, (5,)), ((), 2, ()), ) for base_shape, exp_scalar, exp_shape in test_cases: base_tensor = make_tensor( base_shape, dtype=dt, device=dev, low=low, high=high ) # int tensors don't take negative exponents if dt in [ torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64, ]: exp_tensor = make_tensor( exp_shape, dtype=dt, device=dev, low=0, high=high ) else: exp_tensor = make_tensor( exp_shape, dtype=dt, device=dev, low=low, high=high ) self._test_pow(base_tensor, exp_scalar) self._test_pow(base_tensor, exp_tensor) # test non-contiguous tensors as well base_tensor = make_tensor( base_shape, dtype=dt, device=dev, low=low, high=high, noncontiguous=True, ) if dt in [ torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64, ]: exp_tensor = make_tensor( exp_shape, dtype=dt, device=dev, low=0, high=high, noncontiguous=True, ) else: exp_tensor = make_tensor( exp_shape, dtype=dt, device=dev, low=low, high=high, noncontiguous=True, ) self._test_pow(base_tensor, exp_scalar) self._test_pow(base_tensor, exp_tensor) _test_int_and_float_pow(torch.int8, -2, 2, device) _test_int_and_float_pow(torch.uint8, 0, 3, device) _test_int_and_float_pow(torch.int16, -5, 5, device) _test_int_and_float_pow(torch.int64, -10, 10, device) _test_int_and_float_pow(torch.int32, -10, 10, device) _test_int_and_float_pow(torch.float16, 0.0, 5.0, device) _test_int_and_float_pow(torch.float32, 0.0, 10.0, device) _test_int_and_float_pow(torch.float64, 0.0, 10.0, device) # pow's output would have some NaNs as well _test_int_and_float_pow(torch.float32, -10.0, 10.0, device) _test_int_and_float_pow(torch.float64, -10.0, 10.0, device) # Tests that a Runtime error occurs when a base tensor cannot be resized # by pow's inplace variant due to PyTorch's broadcasting semantics. def test_pow_inplace_resizing_exception(self, device): test_cases = ( ((), (3,)), ((2,), (2, 1)), ((2, 1), (2, 2)), ((2, 2), (2, 1, 1)), ) test_inputs = [ ( make_tensor( base_size, dtype=torch.float64, device=device, high=10.0, low=0.0 ), make_tensor( exp_size, dtype=torch.float64, device=device, high=10.0, low=0.0 ), ) for base_size, exp_size in test_cases ] for base, exponent in test_inputs: regex = "doesn't match the broadcast shape" self.assertRaisesRegex(RuntimeError, regex, base.pow_, exponent) def test_int_tensor_pow_neg_ints(self, device): ints = [ torch.iinfo(torch.int32).min, -3, -2, -1, 0, 1, 2, 3, torch.iinfo(torch.int32).max, ] neg_ints = [torch.iinfo(torch.int32).min, -3, -2, -1] tensor = torch.tensor(ints, dtype=torch.int32, device=device) for pow in neg_ints: self._test_pow(tensor, pow) def test_long_tensor_pow_floats(self, device): ints = [0, 1, 23, 4567] floats = [0.0, 1 / 3, 1 / 2, 1.0, 3 / 2, 2.0] tensor = torch.tensor(ints, dtype=torch.int64, device=device) for pow in floats: self._test_pow(tensor, pow) @dtypes(*[torch.float32, torch.float64]) def test_float_scalar_pow_float_tensor(self, device, dtype): floats = [2.0, -3 / 2, -1.0, -1 / 2, -1 / 3, 0.0, 1 / 3, 1 / 2, 1.0, 3 / 2, 2.0] exponent_shapes = ( (1,), (2, 2), (2, 1), (2, 2, 2), ) tensors = [ make_tensor(shape, dtype=dtype, device=device, low=0) for shape in exponent_shapes ] floats_tensor = torch.tensor(floats, dtype=dtype, device=device) for base in floats: self._test_pow(base, floats_tensor) for tensor in tensors: self._test_pow(base, tensor) @onlyCUDA def test_cuda_tensor_pow_scalar_tensor(self, device): cuda_tensors = [ torch.randn((3, 3), device=device), torch.tensor(3.0, device=device), ] scalar_tensors = [ torch.tensor(5.0, device="cpu"), torch.tensor(-3), torch.tensor(1), ] for base, exp in product(cuda_tensors, scalar_tensors): self._test_pow(base, exp) @onlyCUDA def test_cpu_tensor_pow_cuda_scalar_tensor(self, device): cuda_tensors = [ torch.tensor(5.0, device="cuda"), torch.tensor(-3, device="cuda"), ] for exp in cuda_tensors: base = torch.randn((3, 3), device="cpu") regex = "Expected all tensors to be on the same device, but found at least two devices, cuda.* and cpu!" self.assertRaisesRegex(RuntimeError, regex, torch.pow, base, exp) for exp in cuda_tensors: # Binary ops with a cpu + cuda tensor are allowed if the cpu tensor has 0 dimension base = torch.tensor(3.0, device="cpu") self._test_pow(base, exp) @onlyCUDA @dtypes(torch.complex64, torch.complex128) def test_pow_cuda_complex_extremal_failing(self, device, dtype): t = torch.tensor(complex(-1.0, float("inf")), dtype=dtype, device=device) with self.assertRaises(AssertionError): cuda_out = t.pow(2) cpu_out = t.cpu().pow(2) self.assertEqual(cpu_out, cuda_out) @skipIfTorchDynamo() @onlyNativeDeviceTypes @dtypes(*all_types_and_complex_and(torch.half)) def test_complex_scalar_pow_tensor(self, device, dtype): complexes = [0.5j, 1.0 + 1.0j, -1.5j, 2.2 - 1.6j, 1 + 0j] first_exp = make_tensor((100,), dtype=dtype, device=device, low=-2, high=2) second_exp = make_tensor( (100,), dtype=dtype, device=device, low=-2, high=2, noncontiguous=True ) first_exp[0] = first_exp[10] = first_exp[20] = 0 second_exp[0] = second_exp[10] = second_exp[20] = 0 for base in complexes: # On CPU, # Half Tensor with complex base leads to computation dtype # of ComplexHalf for which this ops is not supported yet # NOTE: pow has fast-path when base is 1 which supports # ComplexHalf will_raise_error = ( torch.device(device).type == "cpu" and dtype is torch.half and base != (1 + 0j) ) if will_raise_error: with self.assertRaisesRegex( RuntimeError, "not implemented for 'ComplexHalf'" ): self._test_pow(base, first_exp) self._test_pow(base, second_exp) else: self._test_pow(base, first_exp) self._test_pow(base, second_exp) @onlyNativeDeviceTypes @skipMeta def test_pow_scalar_type_promotion(self, device): # Test against a scalar and non-scalar input inputs = [17, [17]] for input in inputs: # We expect the computation to be performed in uint8 (overflowing to 0), and then cast to int64 input_tensor_uint8 = torch.tensor(input, dtype=torch.uint8, device=device) out_uint8_computation = torch.pow( 2, input_tensor_uint8, out=torch.tensor(0, dtype=torch.int64, device=device), ) # Computation should run in int64, and not overflow input_tensor_int64 = torch.tensor(input, dtype=torch.int64, device=device) out_int64_computation = torch.pow( 2, input_tensor_int64, out=torch.tensor(0, dtype=torch.int64, device=device), ) self.assertNotEqual(out_uint8_computation, out_int64_computation) self.assertEqual( out_uint8_computation.to(dtype=torch.uint8), out_int64_computation.to(dtype=torch.uint8), ) def test_tensor_pow_tensor(self, device): def rotate(l, n): return l[-n:] + l[:-n] def test_tensor_pow_tensor(values, torch_type, numpy_type): vals_tensor = torch.tensor(values, dtype=torch_type, device=device) for i in range(len(values)): pows = rotate(values, i) pows_tensor = torch.tensor(pows, dtype=torch_type, device=device) self._test_pow(vals_tensor, pows_tensor) ints = [0, 1, 2, 3] test_tensor_pow_tensor(ints, torch.uint8, np.uint8) test_tensor_pow_tensor(ints, torch.int8, np.int8) test_tensor_pow_tensor(ints, torch.int16, np.int16) test_tensor_pow_tensor(ints, torch.int32, np.int32) test_tensor_pow_tensor(ints, torch.int64, np.int64) floats = [-3.0, -2.0, -1.0, -1 / 2, -1 / 3, 0.0, 1 / 3, 1 / 2, 1.0, 2.0, 3.0] test_tensor_pow_tensor(floats, torch.float16, np.float16) test_tensor_pow_tensor(floats, torch.float32, np.float32) test_tensor_pow_tensor(floats, torch.float64, np.float64) def test_logical_xor_with_nontrivial_alignment(self, device): # test tensor that is not aligned to multiple of 16 bytes size = 128 a = torch.randn(size, device=device) > 0 b = torch.randn(size, device=device) > 0 c = torch.randn(size, device=device) > 0 non_trivial_alignment = [1, 2, 4, 8, 15] for i in non_trivial_alignment: for j in non_trivial_alignment: for k in non_trivial_alignment: a_ = a[i : 100 + i] b_ = b[j : 100 + j] c_ = c[k : 100 + k] torch.logical_xor(a_, b_, out=c_) for x, y, z in zip(a_.tolist(), b_.tolist(), c_.tolist()): self.assertEqual(x ^ y, z) @dtypes(torch.float) def test_add_with_tail(self, device, dtype): # test tensor where there is a tail which is not a multiple # of GPU warp size for tail_size in [1, 63, 67, 130]: size = 4096 + tail_size a = torch.randn(size, device=device, dtype=dtype) b = torch.randn(size, device=device, dtype=dtype) c = a + b for x, y, z in zip(a.tolist(), b.tolist(), c.tolist()): self.assertEqual(x + y, z) # Tests that CUDA tensors on different devices cannot be used in the same # binary operation, and that CUDA "scalars" cannot be used in the same # binary operation as non-scalar CPU tensors. @deviceCountAtLeast(2) @onlyCUDA def test_cross_device_binary_ops(self, devices): vals = (1.0, (2.0,)) cpu_tensor = torch.randn(2, 2) def do_test(op, a, b): with self.assertRaisesRegex(RuntimeError, "Expected all tensors.+"): op(a, b) with self.assertRaisesRegex(RuntimeError, "Expected all tensors.+"): op(b, a) with self.assertRaisesRegex(RuntimeError, "Expected all tensors.+"): op(a, cpu_tensor) with self.assertRaisesRegex(RuntimeError, "Expected all tensors.+"): op(cpu_tensor, a) for op in ( operator.add, torch.add, operator.sub, torch.sub, operator.mul, torch.mul, operator.truediv, torch.true_divide, operator.floordiv, torch.floor_divide, ): for a, b in product(vals, vals): a = torch.tensor(a, device=devices[0]) b = torch.tensor(b, device=devices[1]) do_test(op, a, b) # This test ensures that a scalar Tensor can be safely used # in a binary operation in conjunction with a Tensor on all # available CUDA devices @deviceCountAtLeast(2) @onlyCUDA def test_binary_op_scalar_device_unspecified(self, devices): scalar_val = torch.tensor(1.0) for default_device in devices: with torch.cuda.device(default_device): for device in devices: device_obj = torch.device(device) x = torch.rand(3, device=device) y0 = x * scalar_val self.assertEqual(y0.device, device_obj) y1 = scalar_val * x self.assertEqual(y1.device, device_obj) self.assertEqual(y0, y1) def test_div_and_floordiv_vs_python(self, device): # Tests torch division ops which can handle both arguments being # scalars. def _scalar_helper(python_op, torch_op): for a, b in product(range(-10, 10), range(-10, 10)): for op in (lambda x: x * 0.5, lambda x: math.floor(x)): a = op(a) b = op(b) # Skips zero divisors if b == 0: continue expected = python_op(a, b) for op in (operator.truediv, torch.true_divide): actual_scalar = torch_op(a, b) a_t = torch.tensor(a, device=device) b_t = torch.tensor(b, device=device) actual_tensor = torch_op(a_t, b_t) actual_first_tensor = torch_op(a_t, b) actual_second_tensor = torch_op(a, b_t) self.assertEqual(actual_scalar, expected) self.assertEqual(actual_tensor.item(), expected) self.assertEqual(actual_first_tensor, actual_tensor) self.assertEqual(actual_second_tensor, actual_tensor) _scalar_helper(operator.truediv, operator.truediv) _scalar_helper(operator.truediv, torch.true_divide) _scalar_helper(lambda a, b: math.floor(a / b), operator.floordiv) _scalar_helper(lambda a, b: math.floor(a / b), torch.floor_divide) @onlyNativeDeviceTypes @skipIfTorchDynamo("Not a suitable test for TorchDynamo") def test_div_and_floordiv_script_vs_python(self, device): # Creates jitted functions of two tensors def _wrapped_div(a, b): return a / b def _wrapped_floordiv(a, b): return a // b scripted_div = torch.jit.script(_wrapped_div) scripted_floordiv = torch.jit.script(_wrapped_floordiv) for a, b in product(range(-10, 10), range(-10, 10)): for op in (lambda x: x * 0.5, lambda x: math.floor(x)): a = op(a) b = op(b) # Skips zero divisors if b == 0: continue expected_div = a / b expected_floordiv = math.floor(a / b) a_t = torch.tensor(a, device=device) b_t = torch.tensor(b, device=device) self.assertEqual(scripted_div(a_t, b_t), expected_div) self.assertEqual(scripted_floordiv(a_t, b_t), expected_floordiv) # Creates jitted functions of one tensor def _wrapped_div_scalar(a): return a / 5 # NOTE: the JIT implements division as torch.reciprocal(a) * 5 def _wrapped_rdiv_scalar(a): return 5 / a def _wrapped_floordiv_scalar(a): return a // 5 # NOTE: this fails if the input is not an integer tensor # See https://github.com/pytorch/pytorch/issues/45199 def _wrapped_rfloordiv_scalar(a): return 5 // a scripted_div_scalar = torch.jit.script(_wrapped_div_scalar) scripted_rdiv_scalar = torch.jit.script(_wrapped_rdiv_scalar) scripted_floordiv_scalar = torch.jit.script(_wrapped_floordiv_scalar) scripted_rfloordiv_scalar = torch.jit.script(_wrapped_rfloordiv_scalar) for a in range(-10, 10): for op in (lambda x: x * 0.5, lambda x: math.floor(x)): a = op(a) a_t = torch.tensor(a, device=device) self.assertEqual(a / 5, scripted_div_scalar(a_t)) # Skips zero divisors if a == 0: continue self.assertEqual(5 / a, scripted_rdiv_scalar(a_t)) # Handles Issue 45199 (see comment above) if a_t.is_floating_point(): with self.assertRaises(RuntimeError): scripted_rfloordiv_scalar(a_t) else: # This should emit a UserWarning, why doesn't it? # See issue gh-52387 self.assertEqual(5 // a, scripted_rfloordiv_scalar(a_t)) @onlyNativeDeviceTypes @skipIfTorchDynamo("Not a suitable test for TorchDynamo") def test_idiv_and_ifloordiv_vs_python(self, device): def _wrapped_idiv_tensor(a, b): a /= b return a def _wrapped_idiv_scalar(a): a /= 5 return a def _wrapped_true_divide__tensor(a, b): a.true_divide_(b) return a def _wrapped_true_divide__scalar(a): a.true_divide_(5) return a def _wrapped_floor_divide__tensor(a, b): a.floor_divide_(b) return a def _wrapped_floor_divide__scalar(a): a.floor_divide_(5) return a # The following functions are unsupported by the JIT def _wrapped_ifloordiv_tensor(a, b): a //= b return a def _wrapped_ifloordiv_scalar(a): a //= 5 return a with self.assertRaises(torch.jit.frontend.NotSupportedError): scripted_ifloordiv_tensor = torch.jit.script(_wrapped_ifloordiv_tensor) with self.assertRaises(torch.jit.frontend.NotSupportedError): scripted_ifloordiv_scalar = torch.jit.script(_wrapped_ifloordiv_scalar) scripted_idiv_tensor = torch.jit.script(_wrapped_idiv_tensor) scripted_idiv_scalar = torch.jit.script(_wrapped_idiv_scalar) scripted_true_divide__tensor = torch.jit.script(_wrapped_true_divide__tensor) scripted_true_divide__scalar = torch.jit.script(_wrapped_true_divide__scalar) scripted_floor_divide__tensor = torch.jit.script(_wrapped_floor_divide__tensor) scripted_floor_divide__scalar = torch.jit.script(_wrapped_floor_divide__scalar) for a, b in product(range(-10, 10), range(-10, 10)): for op in (lambda x: x * 0.5, lambda x: math.floor(x)): a = op(a) b = op(b) # Skips zero divisors if b == 0: continue expected_idiv = a / b expected_ifloordiv = a // b a_t = torch.tensor(a, device=device) b_t = torch.tensor(b, device=device) if a_t.is_floating_point(): tmp0 = a_t.clone() tmp0 /= b tmp1 = a_t.clone() tmp1 /= b_t self.assertEqual(tmp0.item(), expected_idiv) self.assertEqual(tmp1.item(), expected_idiv) self.assertEqual( scripted_true_divide__tensor(a_t.clone(), b_t).item(), expected_idiv, ) self.assertEqual( scripted_true_divide__scalar(a_t.clone()).item(), a / 5 ) else: tmp = a_t.clone() with self.assertRaises(RuntimeError): tmp /= b with self.assertRaises(RuntimeError): tmp /= b_t with self.assertRaises(RuntimeError): scripted_true_divide__tensor(tmp, b_t) with self.assertRaises(RuntimeError): scripted_true_divide__scalar(tmp) if not a_t.is_floating_point() and b_t.is_floating_point(): # Inplace modification fails because a float tensor is required # if the divisor is a float tensor a_t.clone().floor_divide_(b_t) scripted_floor_divide__tensor(a_t.clone(), b_t) tmp = a_t.clone() tmp //= b_t else: # Inplace modification is OK when both or neither tensor is # a float tensor self.assertEqual( a_t.clone().floor_divide_(b_t).item(), expected_ifloordiv ) self.assertEqual( scripted_floor_divide__tensor(a_t.clone(), b_t).item(), expected_ifloordiv, ) tmp = a_t.clone() tmp //= b_t self.assertEqual(tmp.item(), expected_ifloordiv) self.assertEqual(scripted_floor_divide__scalar(a_t), math.floor(a / 5)) # Tests binary op equivalence with Python builtin ops # Also tests that reverse operations are equivalent to forward ops # NOTE: division ops are tested separately above def test_binary_ops_with_scalars(self, device): for python_op, torch_op in ( (operator.add, torch.add), (operator.sub, torch.sub), (operator.mul, torch.mul), (operator.truediv, torch.div), ): for a, b in product(range(-10, 10), range(-10, 10)): for op in (lambda x: x * 0.5, lambda x: math.floor(x)): a = op(a) b = op(b) # Skips zero divisors if b == 0 or a == 0: continue a_tensor = torch.tensor(a, device=device) b_tensor = torch.tensor(b, device=device) a_tensor_cpu = a_tensor.cpu() b_tensor_cpu = b_tensor.cpu() vals = (a, b, a_tensor, b_tensor, a_tensor_cpu, b_tensor_cpu) for args in product(vals, vals): first, second = args first_scalar = ( first if not isinstance(first, torch.Tensor) else first.item() ) second_scalar = ( second if not isinstance(second, torch.Tensor) else second.item() ) expected = python_op(first_scalar, second_scalar) self.assertEqual(expected, python_op(first, second)) self.assertEqual(expected, torch_op(first, second)) @dtypes( *product( all_types_and(torch.half, torch.bfloat16, torch.bool), all_types_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_maximum_minimum_type_promotion(self, device, dtypes): a = torch.tensor((0, 1), device=device, dtype=dtypes[0]) b = torch.tensor((1, 0), device=device, dtype=dtypes[1]) for op in ( torch.maximum, torch.max, torch.fmax, torch.minimum, torch.min, torch.fmin, ): result = op(a, b) self.assertEqual(result.dtype, torch.result_type(a, b)) @dtypes(*integral_types_and(torch.bool)) def test_maximum_minimum_int_and_bool(self, device, dtype): ops = ( (torch.maximum, torch.max, np.maximum), (torch.minimum, torch.min, np.minimum), (torch.fmax, None, np.fmax), (torch.fmin, None, np.fmin), ) rng = np.random.default_rng() a_np = np.array( rng.integers(-100, 100, size=10), dtype=torch_to_numpy_dtype_dict[dtype] ) b_np = np.array( rng.integers(-100, 100, size=10), dtype=torch_to_numpy_dtype_dict[dtype] ) for torch_op, alias, numpy_op in ops: a_tensor = torch.from_numpy(a_np).to(device=device, dtype=dtype) b_tensor = torch.from_numpy(b_np).to(device=device, dtype=dtype) tensor_result = torch_op(a_tensor, b_tensor) out = torch.empty_like(a_tensor) torch_op(a_tensor, b_tensor, out=out) numpy_result = numpy_op(a_np, b_np) if alias is not None: alias_result = alias(a_tensor, b_tensor) self.assertEqual(alias_result, tensor_result) self.assertEqual(tensor_result, numpy_result) self.assertEqual(out, numpy_result) @precisionOverride({torch.bfloat16: 1e-2}) @dtypes(*(floating_types_and(torch.half, torch.bfloat16))) def test_maximum_minimum_float(self, device, dtype): ops = ( (torch.maximum, torch.max, np.maximum), (torch.minimum, torch.min, np.minimum), (torch.fmax, None, np.fmax), (torch.fmin, None, np.fmin), ) if dtype == torch.bfloat16: a_np = np.random.randn(10).astype(np.float64) b_np = np.random.randn(10).astype(np.float64) else: a_np = np.random.randn(10).astype(torch_to_numpy_dtype_dict[dtype]) b_np = np.random.randn(10).astype(torch_to_numpy_dtype_dict[dtype]) for torch_op, alias, numpy_op in ops: numpy_result = numpy_op(a_np, b_np) a_tensor = torch.from_numpy(a_np).to(device=device, dtype=dtype) b_tensor = torch.from_numpy(b_np).to(device=device, dtype=dtype) tensor_result = torch_op(a_tensor, b_tensor) out = torch.empty_like(a_tensor) torch_op(a_tensor, b_tensor, out=out) if alias is not None: alias_result = alias(a_tensor, b_tensor) self.assertEqual(alias_result, tensor_result, exact_dtype=False) self.assertEqual(tensor_result, numpy_result, exact_dtype=False) self.assertEqual(out, numpy_result, exact_dtype=False) @dtypes(*(floating_types_and(torch.half, torch.bfloat16))) def test_maximum_minimum_float_nan_and_inf(self, device, dtype): # np.maximum and np.minimum functions compare input arrays element-wisely. # if one of the elements being compared is a NaN, then that element is returned. ops = ( (torch.maximum, torch.max, np.maximum), (torch.minimum, torch.min, np.minimum), (torch.fmax, None, np.fmax), (torch.fmin, None, np.fmin), ) a_vals = ( float("inf"), -float("inf"), float("nan"), float("inf"), float("nan"), float("nan"), 1, float("nan"), ) b_vals = ( -float("inf"), float("inf"), float("inf"), float("nan"), float("nan"), 0, float("nan"), -5, ) if dtype == torch.bfloat16: a_np = np.array(a_vals, dtype=np.float64) b_np = np.array(b_vals, dtype=np.float64) else: a_np = np.array(a_vals, dtype=torch_to_numpy_dtype_dict[dtype]) b_np = np.array(b_vals, dtype=torch_to_numpy_dtype_dict[dtype]) for torch_op, alias, numpy_op in ops: numpy_result = numpy_op(a_np, b_np) a_tensor = torch.from_numpy(a_np).to(device=device, dtype=dtype) b_tensor = torch.from_numpy(b_np).to(device=device, dtype=dtype) tensor_result = torch_op(a_tensor, b_tensor) out = torch.empty_like(a_tensor) torch_op(a_tensor, b_tensor, out=out) if alias is not None: alias_result = alias(a_tensor, b_tensor) self.assertEqual(alias_result, tensor_result) if dtype == torch.bfloat16: self.assertEqual(tensor_result, numpy_result, exact_dtype=False) self.assertEqual(out, numpy_result, exact_dtype=False) else: self.assertEqual(tensor_result, numpy_result) self.assertEqual(out, numpy_result) @dtypes( *product( complex_types(), all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_maximum_minimum_complex(self, device, dtypes): for torch_op in ( torch.maximum, torch.minimum, torch.max, torch.min, torch.fmax, torch.fmin, ): with self.assertRaisesRegex(RuntimeError, ".+not implemented for.+"): torch_op( torch.ones(1, device=device, dtype=dtypes[0]), torch.ones(1, device=device, dtype=dtypes[1]), ) with self.assertRaisesRegex(RuntimeError, ".+not implemented for.+"): torch_op( torch.ones(1, device=device, dtype=dtypes[1]), torch.ones(1, device=device, dtype=dtypes[0]), ) @onlyCUDA def test_maximum_minimum_cross_device(self, device): a = torch.tensor((1, 2, -1)) b = torch.tensor((3, 0, 4), device=device) ops = (torch.maximum, torch.minimum) for torch_op in ops: with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device" ): torch_op(a, b) with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device" ): torch_op(b, a) # test cuda tensor and cpu scalar ops = ((torch.maximum, np.maximum), (torch.minimum, np.minimum)) a_np = np.array(1) b_np = np.array([3, 0, 4]) for torch_op, numpy_op in ops: a_tensor = torch.from_numpy(a_np) b_tensor = torch.from_numpy(b_np).to(device=device) tensor_result_1 = torch_op(a_tensor, b_tensor) numpy_result_1 = numpy_op(a_np, b_np) tensor_result_2 = torch_op(b_tensor, a_tensor) numpy_result_2 = numpy_op(b_np, a_np) self.assertEqual(tensor_result_1, numpy_result_1) self.assertEqual(tensor_result_2, numpy_result_2) @dtypes( *product( floating_types_and(torch.half, torch.bfloat16), floating_types_and(torch.half, torch.bfloat16), ) ) def test_maximum_and_minimum_subgradient(self, device, dtypes): def run_test(f, a, b, expected_a_grad, expected_b_grad): a = torch.tensor(a, requires_grad=True, device=device, dtype=dtypes[0]) b = torch.tensor(b, requires_grad=True, device=device, dtype=dtypes[1]) z = f(a, b) z.sum().backward() self.assertEqual(a.grad, expected_a_grad) self.assertEqual(b.grad, expected_b_grad) run_test( torch.maximum, [0.0, 1.0, 2.0], [1.0, 1.0, 1.0], [0.0, 0.5, 1.0], [1.0, 0.5, 0.0], ) run_test( torch.minimum, [0.0, 1.0, 2.0], [1.0, 1.0, 1.0], [1.0, 0.5, 0.0], [0.0, 0.5, 1.0], ) def test_maximum_minimum_forward_ad_float32(self, device): # TODO: This should really be covered by OpInfo but it isn't. The problem # is that our gradient tests test using float64 but it should also test # float32 x = torch.randn(3, device=device, dtype=torch.float32) y = torch.randn(3, device=device, dtype=torch.float32) tx = torch.randn(3, device=device, dtype=torch.float32) ty = torch.randn(3, device=device, dtype=torch.float32) with fwAD.dual_level(): x_dual = fwAD.make_dual(x, tx) y_dual = fwAD.make_dual(y, ty) result = torch.maximum(x_dual, y_dual) _, result_tangent = fwAD.unpack_dual(result) expected = torch.where(x > y, tx, ty) self.assertEqual(result_tangent, expected) with fwAD.dual_level(): x_dual = fwAD.make_dual(x, tx) y_dual = fwAD.make_dual(y, ty) result = torch.minimum(x_dual, y_dual) _, result_tangent = fwAD.unpack_dual(result) expected = torch.where(x < y, tx, ty) self.assertEqual(result_tangent, expected) # TODO: tests like this should be generic @dtypesIfCUDA(torch.half, torch.float, torch.double) @dtypes(torch.float, torch.double) def test_mul_intertype_scalar(self, device, dtype): x = torch.tensor(1.5, dtype=dtype, device=device) y = torch.tensor(3, dtype=torch.int32, device=device) self.assertEqual(x * y, 4.5) self.assertEqual(y * x, 4.5) with self.assertRaisesRegex( RuntimeError, "can't be cast to the desired output type" ): y *= x x *= y self.assertEqual(x, 4.5) @onlyCPU @dtypes(*all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool)) def test_sub(self, device, dtype): if dtype in integral_types(): # Before Python 3.10, floats were implicitly converted to ints, but with # DeprecationWarning: an integer is required (got type float). # Implicit conversion to integers using __int__ is deprecated, # and may be removed in a future version of Python. # Since Python 3.10, that attempt gives an error. m1 = torch.tensor([2, 4], dtype=dtype, device=device) m2 = torch.tensor([1, 2], dtype=dtype, device=device) diff = torch.tensor([1, 2], dtype=dtype) else: m1 = torch.tensor([2.34, 4.44], dtype=dtype, device=device) m2 = torch.tensor([1.23, 2.33], dtype=dtype, device=device) diff = torch.tensor([1.11, 2.11], dtype=dtype) if dtype == torch.bool: self.assertRaises(RuntimeError, lambda: m1 - m2) elif dtype == torch.bfloat16 or dtype == torch.half: # bfloat16 has a lower precision so we have to have a separate check for it self.assertEqual(m1 - m2, diff, atol=0.01, rtol=0) else: self.assertEqual(m1 - m2, diff) # TODO: what is this test testing? @onlyCPU @dtypes(torch.float) def test_csub(self, device, dtype): # with a tensor a = torch.randn(100, 90, dtype=dtype, device=device) b = a.clone().normal_() res_add = torch.add(a, b, alpha=-1) res_csub = a.clone() res_csub.sub_(b) self.assertEqual(res_add, res_csub) # with a scalar a = torch.randn(100, 100, dtype=dtype, device=device) scalar = 123.5 res_add = torch.add(a, -scalar) res_csub = a.clone() res_csub.sub_(scalar) self.assertEqual(res_add, res_csub) # TODO: reconcile with minimum/maximum tests @dtypesIfCUDA(torch.half, torch.float, torch.double) @dtypes(torch.float, torch.double) def test_min_max_binary_op_nan(self, device, dtype): a = torch.rand(1000, dtype=dtype, device=device) b = torch.rand(1000, dtype=dtype, device=device) # 0:250: a -- nan, b -- not nan a[:250] = float("nan") # 250:500: a -- not nan, b -- nan b[250:500] = float("nan") # 500:750: a and b both nan a[500:750] = float("nan") b[500:750] = float("nan") # 750:1000: neither nan ma = torch.max(a, b) mi = torch.min(a, b) for i in range(750): self.assertTrue( torch.isnan(ma[i]), f"max(a, b): {ma[i]}, a: {a[i]}, b: {b[i]}", ) self.assertTrue( torch.isnan(mi[i]), f"min(a, b): {mi[i]}, a: {a[i]}, b: {b[i]}", ) for i in range(750, 1000): self.assertFalse( torch.isnan(ma[i]), f"max(a, b): {ma[i]}, a: {a[i]}, b: {b[i]}", ) self.assertFalse( torch.isnan(mi[i]), f"min(a, b): {mi[i]}, a: {a[i]}, b: {b[i]}", ) @dtypes( *product( all_types_and(torch.half, torch.bfloat16, torch.bool), all_types_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_copysign(self, device, dtypes): def _test_copysign_numpy(a, b): torch_result = torch.copysign(a, b) if a.dtype == torch.bfloat16: np_a = a.to(torch.float).cpu().numpy() else: np_a = a.cpu().numpy() if b.dtype == torch.bfloat16: np_b = b.to(torch.float).cpu().numpy() else: np_b = b.cpu().numpy() expected = torch.from_numpy(np.copysign(np_a, np_b)) # To handle inconsistencies of type promotion between PyTorch and Numpy # Applied for both arguments having integral precision and bfloat16 types = integral_types_and(torch.bool, torch.bfloat16) if a.dtype in types or b.dtype in types: promoted_type = torch.promote_types(torch_result.dtype, expected.dtype) torch_result = torch_result.to(promoted_type) expected = expected.to(promoted_type) # Verify Value self.assertEqual(torch_result, expected) # Verify Sign # Use double copysign to verify the correctnes of 0.0 and -0.0, since # it always True for self.assertEqual(0.0 == -0.0). So, we use 1 as the # magnitude to verify the sign between torch and numpy results, elementwise. # Special case: NaN conversions between FP32 and FP16 is not bitwise # equivalent to pass this assertion. if a.dtype != torch.float16 and b.dtype != torch.float16: self.assertEqual( torch.copysign(torch.tensor(1.0), torch_result), torch.copysign(torch.tensor(1.0), expected), ) # Compare Result with NumPy # Type promotion a = make_tensor((10, 10), device=device, dtype=dtypes[0], low=-9, high=9) b = make_tensor((10, 10), device=device, dtype=dtypes[1], low=-9, high=9) _test_copysign_numpy(a, b) # Broadcast a = make_tensor((10, 1, 10), device=device, dtype=dtypes[0], low=-9, high=9) b = make_tensor((10, 10), device=device, dtype=dtypes[1], low=-9, high=9) _test_copysign_numpy(a, b) a = make_tensor((10, 10), device=device, dtype=dtypes[0], low=-9, high=9) b = make_tensor((10, 1, 10), device=device, dtype=dtypes[1], low=-9, high=9) _test_copysign_numpy(a, b) # 0.0/-0.0/inf/-inf/nan cases = [0.0, -0.0, float("inf"), float("-inf"), float("nan")] # torch.bfloat16 can not hold '-nan' # torch.half can not hold '-nan' on CUDA types = [torch.float32, torch.float64] if device == "cpu": types.append(torch.float16) if dtypes[0] in types: b = make_tensor((10, 10), device=device, dtype=dtypes[1], low=-9, high=9) for case in cases: _test_copysign_numpy( torch.tensor([case], device=device, dtype=dtypes[0]), b ) if dtypes[1] in floating_types_and(torch.half, torch.bfloat16): a = make_tensor((10, 10), device=device, dtype=dtypes[0], low=-9, high=9) for case in cases: _test_copysign_numpy( a, torch.tensor([case], device=device, dtype=dtypes[1]) ) @dtypes( *product( floating_types_and(torch.half, torch.bfloat16), floating_types_and(torch.half, torch.bfloat16), ) ) def test_copysign_subgradient(self, device, dtypes): # Input is 0.0 x = torch.tensor( [0.0, 0.0, 0.0], dtype=dtypes[0], device=device, requires_grad=True ) y = torch.tensor( [-1.0, 0.0, 1.0], dtype=dtypes[1], device=device, requires_grad=True ) out = torch.copysign(x, y) out.sum().backward() self.assertEqual(x.grad.tolist(), [0.0, 0.0, 0.0]) self.assertEqual(y.grad.tolist(), [0.0] * 3) # Input is -0.0 x = torch.tensor( [-0.0, -0.0, -0.0], dtype=dtypes[0], device=device, requires_grad=True ) y = torch.tensor( [-1.0, 0.0, 1.0], dtype=dtypes[1], device=device, requires_grad=True ) out = torch.copysign(x, y) out.sum().backward() self.assertEqual(x.grad.tolist(), [0.0, 0.0, 0.0]) self.assertEqual(y.grad.tolist(), [0.0] * 3) # Other is 0.0 x = torch.tensor( [-1.0, 0.0, 1.0], dtype=dtypes[0], device=device, requires_grad=True ) y = torch.tensor( [0.0, 0.0, 0.0], dtype=dtypes[1], device=device, requires_grad=True ) out = torch.copysign(x, y) out.sum().backward() self.assertEqual(x.grad.tolist(), [-1.0, 0.0, 1.0]) self.assertEqual(y.grad.tolist(), [0.0] * 3) # Other is -0.0 x = torch.tensor( [-1.0, 0.0, 1.0], dtype=dtypes[0], device=device, requires_grad=True ) y = torch.tensor( [-0.0, -0.0, -0.0], dtype=dtypes[1], device=device, requires_grad=True ) out = torch.copysign(x, y) out.sum().backward() self.assertEqual(x.grad.tolist(), [1.0, 0.0, -1.0]) self.assertEqual(y.grad.tolist(), [0.0] * 3) @dtypes(torch.bfloat16, torch.float) def test_div(self, device, dtype): for op, method, inplace in ( (torch.div, torch.Tensor.div, torch.Tensor.div_), (torch.true_divide, torch.Tensor.true_divide, torch.Tensor.true_divide_), ): m1 = torch.randn(10, 10, dtype=torch.float, device=device).to(dtype=dtype) res1 = m1.clone() inplace(res1[:, 3], 2) res2 = m1.clone() for i in range(m1.size(0)): res2[i, 3] = res2[i, 3] / 2 self.assertEqual(res1, res2) if dtype == torch.bfloat16: a1 = torch.tensor([4.2, 6.2], dtype=dtype, device=device) a2 = torch.tensor([2.0, 2.0], dtype=dtype, device=device) self.assertEqual( op(a1, a2), torch.tensor([2.1, 3.1], dtype=dtype, device=device), atol=0.01, rtol=0, ) self.assertEqual(method(a1, a2), op(a1, a2)) @dtypes(torch.bfloat16, torch.float) def test_true_divide_out(self, device, dtype): a1 = torch.tensor([4.2, 6.2], dtype=dtype, device=device) a2 = torch.tensor([2.0, 2.0], dtype=dtype, device=device) res = torch.empty_like(a1) self.assertEqual( torch.true_divide(a1, a2, out=res), torch.tensor([2.1, 3.1], dtype=dtype, device=device), atol=0.01, rtol=0, ) @dtypes(torch.half) def test_divmul_scalar(self, device, dtype): x = torch.tensor(100.0, device=device, dtype=dtype) x_ref = x.float() scale = 1e5 res = x.div(scale) expected = x_ref.div(scale) self.assertEqual(res, expected.to(dtype), atol=0.0, rtol=0.0) x = torch.tensor(1e-5, device=device, dtype=dtype) x_ref = x.float() res = x.mul(scale) expected = x_ref.mul(scale) self.assertEqual(res, expected.to(dtype), atol=0.0, rtol=0.0) res = scale * x self.assertEqual(res, expected.to(dtype), atol=0.0, rtol=0.0) @dtypesIfCUDA( *set(get_all_math_dtypes("cuda")) - {torch.complex64, torch.complex128} ) @dtypes(*set(get_all_math_dtypes("cpu")) - {torch.complex64, torch.complex128}) def test_floor_divide_tensor(self, device, dtype): x = torch.randn(10, device=device).mul(30).to(dtype) y = torch.arange(1, 11, dtype=dtype, device=device) z = x // y z_alt = torch.floor(x.double() / y.double()).to(dtype) self.assertEqual(z.dtype, x.dtype) self.assertEqual(z, z_alt) @dtypesIfCUDA( *set(get_all_math_dtypes("cuda")) - {torch.complex64, torch.complex128} ) @dtypes(*set(get_all_math_dtypes("cpu")) - {torch.complex64, torch.complex128}) def test_floor_divide_scalar(self, device, dtype): x = torch.randn(100, device=device).mul(10).to(dtype) z = x // 3 z_alt = torch.tensor( [math.floor(v.item() / 3.0) for v in x], dtype=x.dtype, device=device ) self.assertEqual(z.dtype, x.dtype) self.assertEqual(z, z_alt) @onlyCPU @dtypes(*get_all_math_dtypes("cpu")) def test_rdiv(self, device, dtype): if dtype is torch.float16: return elif dtype.is_complex: x = torch.rand(100, dtype=dtype, device=device).add(1).mul(4) else: x = torch.rand(100, device=device).add(1).mul(4).to(dtype) y = 30 / x z = torch.tensor([30 / v.item() for v in x], device=device) self.assertEqual(y, z, exact_dtype=False) @dtypes(*floating_types_and(torch.half)) def test_fmod_remainder_by_zero_float(self, device, dtype): fn_list = (torch.fmod, torch.remainder) for fn in fn_list: # check floating-point tensor fmod/remainder to zero is nan on both CPU and GPU x = make_tensor((10, 10), device=device, dtype=dtype, low=-9, high=9) zero = torch.zeros_like(x) self.assertTrue(torch.all(fn(x, 0.0).isnan())) self.assertTrue(torch.all(fn(x, zero).isnan())) @onlyNativeDeviceTypes # Check Issue https://github.com/pytorch/pytorch/issues/48130 @dtypes(*integral_types()) def test_fmod_remainder_by_zero_integral(self, device, dtype): fn_list = (torch.fmod, torch.remainder) for fn in fn_list: # check integral tensor fmod/remainder to zero x = make_tensor((10, 10), device=device, dtype=dtype, low=-9, high=9) zero = torch.zeros_like(x) # RuntimeError on CPU if self.device_type == "cpu": with self.assertRaisesRegex(RuntimeError, "ZeroDivisionError"): fn(x, zero) elif torch.version.hip is not None: # ROCm behavior: x % 0 is a no-op; x is returned self.assertEqual(fn(x, zero), x) else: # CUDA behavior: Different value for different dtype # Due to it's an undefined behavior, CUDA returns a pattern of all 1s # for integral dividend (other than int64) divided by zero. For int64, # CUDA returns all 1s for negative dividend, half 1s for positive dividend. # uint8: 0xff -> 255 # int32: 0xffffffff -> -1 if dtype == torch.int64: self.assertEqual(fn(x, zero) == 4294967295, x >= 0) self.assertEqual(fn(x, zero) == -1, x < 0) else: value = 255 if dtype == torch.uint8 else -1 self.assertTrue(torch.all(fn(x, zero) == value)) @dtypes(*all_types_and(torch.half)) def test_fmod_remainder(self, device, dtype): # Use numpy as reference def _helper(x, mod, fns_list): for fn, inplace_fn, ref_fn in fns_list: np_x = x.cpu().numpy() if torch.is_tensor(x) else x np_mod = mod.cpu().numpy() if torch.is_tensor(mod) else mod exp = ref_fn(np_x, np_mod) exp = torch.from_numpy(exp) res = fn(x, mod) self.assertEqual(res, exp, exact_dtype=False) if torch.is_tensor(x): # out out = torch.empty(0, device=device, dtype=res.dtype) fn(x, mod, out=out) self.assertEqual(out, exp, exact_dtype=False) self.assertEqual(out.size(), torch.Size([10, 10])) # in-place (Type cast runtime error) try: inplace_fn(x, mod) self.assertEqual(x, exp, exact_dtype=False) except RuntimeError as e: self.assertRegex( str(e), "result type (Half|Float|Double) " "can't be cast to the desired output " "type (Byte|Char|Short|Int|Long)", ) x = make_tensor((10, 10), device=device, dtype=dtype, low=-9, high=9) # mod with same dtype as x mod = make_tensor((10, 10), device=device, dtype=dtype, low=-9, high=9) # Exclude 0 mod[mod == 0] = 1 # Mods: Integer, Float, Tensor, Non-contiguous Tensor mods = [3, 2.3, mod, mod.t()] # mod with floating-point dtype if dtype in integral_types(): mod_float = make_tensor( (10, 10), device=device, dtype=torch.float, low=-9, high=9 ) mod[mod == 0] = 1 mods.append(mod_float) for dividend, mod in product([x, x.t()], mods): _helper( dividend, mod, ( (torch.fmod, torch.Tensor.fmod_, np.fmod), (torch.remainder, torch.Tensor.remainder_, np.remainder), ), ) # Tests for torch.remainder(scalar, tensor) for dividend, mod in product([5, 3.14], mods): if torch.is_tensor(mod): _helper( dividend, mod, ((torch.remainder, torch.Tensor.remainder_, np.remainder),), ) @dtypes(torch.float, torch.double) def test_remainder_fmod_large_dividend(self, device, dtype): alarge = 1e9 pi = 3.14159265358979 for avalue in [alarge, -alarge]: for bvalue in [pi, -pi]: a = torch.tensor([avalue], dtype=dtype, device=device) b = torch.tensor([bvalue], dtype=dtype, device=device) c = torch.remainder(a, b) d = torch.fmod(a, b) self.assertTrue( (b[0] > 0) == (c[0] > 0) ) # remainder has same sign as divisor self.assertTrue( (a[0] > 0) == (d[0] > 0) ) # fmod has same sign as dividend self.assertTrue( abs(c[0]) < abs(b[0]) ) # remainder is within range of divisor self.assertTrue( abs(d[0]) < abs(b[0]) ) # fmod is within range of divisor if (a[0] > 0) == (b[0] > 0): self.assertTrue(c[0] == d[0]) # remainder is same as fmod else: self.assertTrue( abs(c[0] - d[0]) == abs(b[0]) ) # differ by one divisor @dtypesIfCPU(torch.bfloat16, torch.half, torch.float32, torch.float64) @dtypes(torch.float32, torch.float64) def test_hypot(self, device, dtype): inputs = [ ( torch.randn(10, device=device).to(dtype), torch.randn(10, device=device).to(dtype), ), ( torch.randn((3, 3, 3), device=device).to(dtype), torch.randn((3, 3, 3), device=device).to(dtype), ), ( torch.randn((10, 1), device=device).to(dtype), torch.randn((10, 1), device=device).to(dtype).transpose(0, 1), ), ( torch.randint(100, (10,), device=device, dtype=torch.long), torch.randn(10, device=device).to(dtype), ), ] for input in inputs: actual = torch.hypot(input[0], input[1]) if dtype in [torch.bfloat16, torch.half]: expected = torch.sqrt(input[0] * input[0] + input[1] * input[1]) else: expected = np.hypot(input[0].cpu().numpy(), input[1].cpu().numpy()) self.assertEqual(actual, expected, exact_dtype=False) @onlyNativeDeviceTypes @dtypes(torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64) def test_gcd(self, device, dtype): # Tests gcd(0, 0), gcd(0, a) cases t1 = torch.tensor([0, 10, 0], dtype=dtype, device=device) t2 = torch.tensor([0, 0, 10], dtype=dtype, device=device) actual = torch.gcd(t1, t2) expected = np.gcd([0, 10, 0], [0, 0, 10]) self.assertEqual(actual, expected, exact_dtype=False) if dtype == torch.uint8: # Test unsigned integers with potential sign issues (i.e., uint8 with value >= 128) a = torch.tensor([190, 210], device=device, dtype=dtype) b = torch.tensor([190, 220], device=device, dtype=dtype) actual = torch.gcd(a, b) expected = torch.tensor([190, 10], device=device, dtype=dtype) self.assertEqual(actual, expected) else: # Compares with NumPy a = torch.randint(-20, 20, (1024,), device=device, dtype=dtype) b = torch.randint(-20, 20, (1024,), device=device, dtype=dtype) actual = torch.gcd(a, b) expected = np.gcd(a.cpu().numpy(), b.cpu().numpy()) self.assertEqual(actual, expected) @onlyNativeDeviceTypes @dtypes(torch.int16, torch.int32, torch.int64) def test_lcm(self, device, dtype): # Tests lcm(0, 0), lcm(0, a) cases t1 = torch.tensor([0, 10, 0], dtype=dtype, device=device) t2 = torch.tensor([0, 0, 10], dtype=dtype, device=device) actual = torch.lcm(t1, t2) expected = np.lcm([0, 10, 0], [0, 0, 10]) self.assertEqual(actual, expected, exact_dtype=False) # Compares with NumPy a = torch.randint(-20, 20, (1024,), device=device, dtype=dtype) b = torch.randint(-20, 20, (1024,), device=device, dtype=dtype) actual = torch.lcm(a, b) expected = np.lcm(a.cpu().numpy(), b.cpu().numpy()) self.assertEqual(actual, expected, exact_dtype=False) @onlyNativeDeviceTypes @dtypesIfCPU(torch.float32, torch.float64, torch.float16) @dtypes(torch.float32, torch.float64) def test_nextafter(self, device, dtype): # Test special cases t1 = torch.tensor([0, 0, 10], device=device, dtype=dtype) t2 = torch.tensor([inf, -inf, 10], device=device, dtype=dtype) actual = torch.nextafter(t1, t2) expected = np.nextafter(t1.cpu().numpy(), t2.cpu().numpy()) self.assertEqual(actual, expected, atol=0, rtol=0) actual = torch.nextafter(t2, t1) expected = np.nextafter(t2.cpu().numpy(), t1.cpu().numpy()) self.assertEqual(actual, expected, atol=0, rtol=0) t1 = torch.tensor([0, nan], device=device, dtype=dtype) t2 = torch.tensor([nan, 0], device=device, dtype=dtype) self.assertTrue(torch.nextafter(t1, t2).isnan().all()) a = torch.randn(100, device=device, dtype=dtype) b = torch.randn(100, device=device, dtype=dtype) actual = torch.nextafter(a, b) expected = np.nextafter(a.cpu().numpy(), b.cpu().numpy()) self.assertEqual(actual, expected, atol=0, rtol=0) @onlyNativeDeviceTypes @dtypes(torch.bfloat16) def test_nextafter_bfloat16(self, device, dtype): nan = float("nan") inf = float("inf") cases = ( # (from, to, expected) (0, 1, 9.183549615799121e-41), (0, -1, -9.183549615799121e-41), (1, -2, 0.99609375), (1, 0, 0.99609375), (1, 2, 1.0078125), (-1, -2, -1.0078125), (-1, 0, -0.99609375), (2, -1, 1.9921875), (2, 1, 1.9921875), (20, 3000, 20.125), (20, -3000, 19.875), (3000, -20, 2992.0), (-3000, 20, -2992.0), (65536, 0, 65280.0), (65536, inf, 66048.0), (-65536, 0, -65280.0), (-65536, -inf, -66048.0), (nan, 0, nan), (0, nan, nan), (nan, nan, nan), (nan, inf, nan), (inf, nan, nan), (inf, -inf, 3.3895313892515355e38), (-inf, inf, -3.3895313892515355e38), (inf, 0, 3.3895313892515355e38), (0, inf, 9.183549615799121e-41), (-inf, 0, -3.3895313892515355e38), (0, -inf, -9.183549615799121e-41), ) for from_v, to_v, expected in cases: from_t = torch.tensor([from_v], device=device, dtype=dtype) to_t = torch.tensor([to_v], device=device, dtype=dtype) actual = torch.nextafter(from_t, to_t).item() self.assertEqual(actual, expected, atol=0, rtol=0) def _test_cop(self, torchfn, mathfn, dtype, device): def reference_implementation(res2): for i, j in iter_indices(sm1): idx1d = i * sm1.size(0) + j res2[i, j] = mathfn(sm1[i, j], sm2[idx1d]) return res2 # contiguous m1 = torch.randn(10, 10, 10, dtype=dtype, device=device) m2 = torch.randn(10, 10 * 10, dtype=dtype, device=device) sm1 = m1[4] sm2 = m2[4] res1 = torchfn(sm1, sm2.view(10, 10)) res2 = reference_implementation(res1.clone()) self.assertEqual(res1, res2) # non-contiguous m1 = torch.randn(10, 10, 10, dtype=dtype, device=device) m2 = torch.randn(10 * 10, 10 * 10, dtype=dtype, device=device) sm1 = m1[:, 4] sm2 = m2[:, 4] # view as sm1.size() sm2.set_( sm2.storage(), sm2.storage_offset(), sm1.size(), (sm2.stride()[0] * 10, sm2.stride()[0]), ) res1 = torchfn(sm1, sm2) # reference_implementation assumes 1-d sm2 sm2.set_( sm2.storage(), sm2.storage_offset(), m2[:, 4].size(), m2[:, 4].stride() ) res2 = reference_implementation(res1.clone()) self.assertEqual(res1, res2) @onlyCPU @dtypes(torch.float) def test_cdiv(self, device, dtype): self._test_cop(torch.div, operator.truediv, dtype, device) @onlyCPU @dtypes(torch.float) def test_cremainder(self, device, dtype): self._test_cop(torch.remainder, operator.mod, dtype, device) @onlyCPU @dtypes(torch.float) def test_cmul(self, device, dtype): self._test_cop(torch.mul, operator.mul, dtype, device) @onlyCPU @dtypes(torch.float) def test_cpow(self, device, dtype): self._test_cop( torch.pow, lambda x, y: nan if x < 0 else math.pow(x, y), dtype, device ) @onlyCPU @dtypes(torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64) def test_floor_divide_zero(self, device, dtype): a = torch.tensor([0, 1], dtype=dtype, device=device) b = torch.tensor([0, 1], dtype=dtype, device=device) with self.assertRaisesRegex(RuntimeError, "ZeroDivisionError"): with self.assertWarnsOnceRegex(UserWarning, "floor_divide"): a // b @dtypes(*all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool)) def test_muldiv_scalar(self, device, dtype): x = make_tensor((10, 3), dtype=dtype, device=device, low=None, high=None) s = make_tensor((1,), dtype=dtype, device="cpu", low=None, high=None).item() y = torch.full_like(x, s) self.assertEqual(x * s, x * y) self.assertEqual(s * x, y * x) self.assertEqual(x / s, x / y) self.assertEqual(s / x, y / x) # TODO: update make_tensor to support extremal additions and remove this in favor of make_tensor def _generate_input(self, shape, dtype, device, with_extremal): if shape == (): x = torch.tensor((), dtype=dtype, device=device) else: if dtype.is_floating_point or dtype.is_complex: # work around torch.randn not being implemented for bfloat16 if dtype == torch.bfloat16: x = torch.randn(*shape, device=device) * random.randint(30, 100) x = x.to(torch.bfloat16) else: x = torch.randn( *shape, dtype=dtype, device=device ) * random.randint(30, 100) x[torch.randn(*shape) > 0.5] = 0 if with_extremal and dtype.is_floating_point: # Use extremal values x[torch.randn(*shape) > 0.5] = float("nan") x[torch.randn(*shape) > 0.5] = float("inf") x[torch.randn(*shape) > 0.5] = float("-inf") elif with_extremal and dtype.is_complex: x[torch.randn(*shape) > 0.5] = complex("nan") x[torch.randn(*shape) > 0.5] = complex("inf") x[torch.randn(*shape) > 0.5] = complex("-inf") elif dtype == torch.bool: x = torch.zeros(shape, dtype=dtype, device=device) x[torch.randn(*shape) > 0.5] = True else: x = torch.randint(15, 100, shape, dtype=dtype, device=device) return x @dtypes( *tuple( itertools.combinations_with_replacement( all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), 2 ) ) ) def test_comparison_ops_type_promotion_and_broadcasting(self, device, dtypes): # issue #42660 # testing all combinations of broadcasting and type promotion # with a range of dtypes and input shapes, and with extremal values def compare_with_numpy_bin_op(torch_fn, np_fn, x, y, out=None): # working around the fact that numpy doesn't support bfloat16 # by letting numpy treat them as float32's x_np = x if x.dtype != torch.bfloat16 else x.to(torch.float32) y_np = ( y.cpu().numpy() if y.dtype != torch.bfloat16 else y.to(torch.float32).cpu().numpy() ) self.compare_with_numpy( lambda inp: torch_fn(inp, y, out=out) if out else torch_fn(inp, y), lambda inp: np_fn(inp, y_np, out=out) if out else np_fn(inp, y_np), x_np, ) complex_op_denylist = [ torch.lt, torch.le, torch.gt, torch.ge, ] # complex not supported input_sizes = [(1,), (10,), (10, 1), (1, 10), (4, 10), (64, 10), (12, 3)] op_pairs = [ (torch.lt, np.less), (torch.le, np.less_equal), (torch.gt, np.greater), (torch.ge, np.greater_equal), (torch.eq, np.equal), (torch.ne, np.not_equal), (torch.logical_and, np.logical_and), (torch.logical_or, np.logical_or), (torch.logical_xor, np.logical_xor), ] for size1 in input_sizes: size2 = (2,) + size1 # perform broadcasting for with_extremal in [False, True]: a = self._generate_input(size1, dtypes[0], device, with_extremal) b = self._generate_input(size2, dtypes[1], device, with_extremal) for torch_op, numpy_op in op_pairs: if ( dtypes[0].is_complex or dtypes[1].is_complex ) and torch_op in complex_op_denylist: continue # functional version of op compare_with_numpy_bin_op(torch_op, numpy_op, a, b) # functional comparison ops always return bool tensors self.assertEqual(torch_op(a, b).dtype, torch.bool) # out version of op out = torch.zeros( 1, dtype=torch.complex128 ) # all casts to complex128 are safe compare_with_numpy_bin_op(torch_op, numpy_op, a, b, out=out) @onlyNativeDeviceTypes @dtypes(torch.int8, torch.int16, torch.int32, torch.int64) def test_signed_shift(self, device, dtype): "Ensure that signed integer bit shifting works as expected." a = torch.tensor([-10, 10], device=device, dtype=dtype) # [11...1110110, 1010] expected_l = torch.tensor( [-40, 40], device=device, dtype=dtype ) # [11...11011000, 101000] self.assertEqual(a << 2, expected_l) self.compare_with_numpy(lambda x: x << 2, lambda x: np.left_shift(x, 2), a) expected_r = torch.tensor( [-5, 5], device=device, dtype=dtype ) # [1111...111011, 101] self.assertEqual(a >> 1, expected_r) self.compare_with_numpy(lambda x: x >> 1, lambda x: np.right_shift(x, 1), a) @onlyNativeDeviceTypes @dtypes(*get_all_int_dtypes()) def test_shift_limits(self, device, dtype): "Ensure that integer bit shifting works as expected with out-of-limits shift values." # Issue #70904 iinfo = torch.iinfo(dtype) bits = iinfo.bits low = iinfo.min high = iinfo.max exact_dtype = ( dtype != torch.uint8 ) # numpy changes dtype from uint8 to int16 for some out-of-limits shift values for input in ( torch.tensor( [-1, 0, 1], device=device, dtype=dtype ), # small for non-vectorized operation torch.tensor( [low, high], device=device, dtype=dtype ), # small for non-vectorized operation make_tensor( (64, 64, 64), low=low, high=high, device=device, dtype=dtype ), # large for vectorized operation ): shift_left_expected = torch.zeros_like(input) shift_right_expected = torch.clamp(input, -1, 0) for shift in chain(range(-100, -1), range(bits, 100)): shift_left = input << shift self.assertEqual(shift_left, shift_left_expected, msg=f"<< {shift}") self.compare_with_numpy( lambda x: x << shift, lambda x: np.left_shift(x, shift), input, exact_dtype=exact_dtype, msg=f"<< {shift}", ) shift_right = input >> shift self.assertEqual(shift_right, shift_right_expected, msg=f">> {shift}") self.compare_with_numpy( lambda x: x >> shift, lambda x: np.right_shift(x, shift), input, exact_dtype=exact_dtype, msg=f">> {shift}", ) @onlyNativeDeviceTypes @dtypes( *list( product( all_types_and(torch.half, torch.bfloat16, torch.bool), all_types_and(torch.half, torch.bfloat16, torch.bool), ) ) ) def test_heaviside(self, device, dtypes): input_dtype = dtypes[0] values_dtype = dtypes[1] rng = np.random.default_rng() input = np.array( rng.integers(-10, 10, size=10), dtype=torch_to_numpy_dtype_dict[ input_dtype if (input_dtype != torch.bfloat16) else torch.float64 ], ) input[0] = input[3] = input[7] = 0 values = np.array( rng.integers(-10, 10, size=10), dtype=torch_to_numpy_dtype_dict[ values_dtype if (values_dtype != torch.bfloat16) else torch.float64 ], ) np_result = torch.from_numpy(np.heaviside(input, values)).to( device=device, dtype=input_dtype ) input = torch.from_numpy(input).to(device=device, dtype=input_dtype) values = torch.from_numpy(values).to(device=device, dtype=values_dtype) out = torch.empty_like(input) if input_dtype == values_dtype: torch_result = torch.heaviside(input, values) self.assertEqual(np_result, torch_result) torch_result = input.heaviside(values) self.assertEqual(np_result, torch_result) torch.heaviside(input, values, out=out) self.assertEqual(np_result, out) input.heaviside_(values) self.assertEqual(np_result, input) else: with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for tensors with different dtypes.", ): torch.heaviside(input, values) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for tensors with different dtypes.", ): input.heaviside(values) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for tensors with different dtypes.", ): torch.heaviside(input, values, out=out) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for tensors with different dtypes.", ): input.heaviside_(values) @onlyCUDA def test_heaviside_cross_device(self, device): x = torch.tensor([-9, 5, 0, 6, -2, 2], device=device) y = torch.tensor(0) result = torch.heaviside(x, y) expect = torch.tensor([0, 1, 0, 1, 0, 1], device=device) self.assertEqual(result, expect) result = torch.heaviside(y, x) expect = torch.tensor([-9, 5, 0, 6, -2, 2], device=device) self.assertEqual(result, expect) x = torch.tensor([-9, 5, 0, 6, -2, 2]) y = torch.tensor(0, device=device) with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device" ): torch.heaviside(x, y) with self.assertRaisesRegex( RuntimeError, "Expected all tensors to be on the same device" ): torch.heaviside(y, x) @dtypes(*list(product(complex_types(), complex_types()))) def test_heaviside_complex(self, device, dtypes): input_dtype = dtypes[0] values_dtype = dtypes[1] data = (complex(0, -6), complex(-1, 3), complex(1, 1)) input = torch.tensor(data, device=device, dtype=input_dtype) values = torch.tensor(data, device=device, dtype=values_dtype) out = torch.empty_like(input) real = input.real with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for complex tensors." ): torch.heaviside(input, real) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for complex tensors." ): real.heaviside(values) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for complex tensors." ): input.heaviside_(values) with self.assertRaisesRegex( RuntimeError, "heaviside is not yet implemented for complex tensors." ): torch.heaviside(real, real, out=out) def _test_logical(self, device, dtypes, op, a_, b_, expected_res_): expected_res = torch.tensor(expected_res_, dtype=dtypes[0], device=device) a = torch.tensor(a_, dtype=dtypes[0], device=device) b = torch.tensor(b_, dtype=dtypes[1], device=device) # new tensor self.assertEqual(expected_res.bool(), getattr(a, op)(b)) # out c = torch.empty(0, dtype=torch.bool, device=device) getattr(torch, op)(a, b, out=c) self.assertEqual(expected_res.bool(), c) getattr(a, op + "_")(b) self.assertEqual(expected_res, a) @dtypes( *product( all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_logical_xor(self, device, dtypes): self._test_logical( device, dtypes, "logical_xor", [10, 0, 1, 0], [1, 0, 0, 10], [0, 0, 1, 1] ) @dtypes( *product( all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_logical_and(self, device, dtypes): self._test_logical( device, dtypes, "logical_and", [10, 0, 1, 0], [1, 0, 0, 10], [1, 0, 0, 0] ) @dtypes( *product( all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), all_types_and_complex_and(torch.half, torch.bfloat16, torch.bool), ) ) def test_logical_or(self, device, dtypes): self._test_logical( device, dtypes, "logical_or", [10, 0, 1, 0], [1, 0, 0, 10], [1, 0, 1, 1] ) def test_remainder_overflow(self, device): # Check Integer Overflows x = torch.tensor(23500, dtype=torch.int64, device=device) q = 392486996410368 self.assertEqual(x % q, x) self.assertEqual(-x % q, q - x) self.assertEqual(x % -q, x - q) self.assertEqual(-x % -q, -x) def test_rpow(self, device): m = torch.randn(10, 10, device=device) self.assertEqual(torch.pow(2, m), 2**m) # test with scalar m = torch.randn(1, device=device).squeeze() assert m.dim() == 0, "m is intentionally a scalar" self.assertEqual(torch.pow(2, m), 2**m) def test_ldexp(self, device): # random values mantissas = torch.randn(64, device=device) exponents = torch.randint(-31, 31, (64,), device=device, dtype=torch.int32) # basic test np_outcome = np.ldexp(mantissas.cpu().numpy(), exponents.cpu().numpy()) pt_outcome_1 = torch.ldexp(mantissas, exponents) pt_outcome_2 = mantissas.ldexp(exponents) self.assertEqual(np_outcome, pt_outcome_1.cpu()) self.assertEqual(np_outcome, pt_outcome_2.cpu()) mantissas.ldexp_(exponents) self.assertEqual(np_outcome, mantissas.cpu()) # test bounds mantissas = torch.tensor( [float("inf"), float("-inf"), float("inf"), float("nan")], device=device ) exponents = torch.randint(0, 31, (4,), device=device, dtype=torch.int32) np_outcome = np.ldexp(mantissas.cpu().numpy(), exponents.cpu().numpy()) pt_outcome = torch.ldexp(mantissas, exponents) self.assertEqual(np_outcome, pt_outcome.cpu()) # test half dtype behavior mantissas = torch.randn(64, device=device, dtype=torch.half) exponents = torch.randint(-5, 5, (64,), device=device) self.assertEqual(torch.ldexp(mantissas, exponents).dtype, torch.half) # test float64 computation mantissas = torch.tensor([1], dtype=torch.float64, device=device) exponents = torch.tensor([128], dtype=torch.int64, device=device) expected = torch.pow( torch.full((1,), 2, device=device, dtype=torch.float64), 128 ) self.assertEqual(torch.ldexp(mantissas, exponents), expected) @dtypes(torch.float, torch.double, torch.cfloat, torch.cdouble) def test_lerp(self, device, dtype): start_end_weight_shapes = [(), (5,), (5, 5)] for shapes in product( start_end_weight_shapes, start_end_weight_shapes, start_end_weight_shapes ): start = torch.randn(shapes[0], device=device, dtype=dtype) end = torch.randn(shapes[1], device=device, dtype=dtype) # Tensor weights weights = [ torch.randn(shapes[2], device=device, dtype=dtype), random.random(), ] if dtype.is_complex: weights += [complex(0, 1), complex(0.4, 1.2)] for weight in weights: actual = torch.lerp(start, end, weight) actual_method = start.lerp(end, weight) self.assertEqual(actual, actual_method) actual_out = torch.tensor(1.0, dtype=dtype, device=device) torch.lerp(start, end, weight, out=actual_out) self.assertEqual(actual, actual_out) expected = start + weight * (end - start) self.assertEqual(expected, actual) @onlyCUDA @dtypes(torch.half, torch.bfloat16) def test_lerp_lowp(self, device, dtype): xvals = (0.0, -30000.0) yvals = (0.1, -20000.0) xs = [torch.full((4,), xval, device=device, dtype=dtype) for xval in xvals] ys = [torch.full((4,), yval, device=device, dtype=dtype) for yval in yvals] weights = [70000, torch.full((4,), 8, device=device, dtype=dtype)] for x, y, w in zip(xs, ys, weights): xref = x.float() yref = y.float() wref = w.float() if isinstance(w, torch.Tensor) else w actual = torch.lerp(x, y, w) expected = torch.lerp(xref, yref, wref).to(dtype) self.assertEqual(actual, expected, atol=0.0, rtol=0.0) @onlyCPU @dtypes(torch.half, torch.bfloat16) def test_lerp_lowp_cpu(self, device, dtype): xvals = (0.0, -30000.0) yvals = (0.1, -20000.0) for shape in [(4,), (20,), (3, 10, 10)]: xs = [torch.full(shape, xval, device=device, dtype=dtype) for xval in xvals] ys = [torch.full(shape, yval, device=device, dtype=dtype) for yval in yvals] weights = [70000, torch.full(shape, 8, device=device, dtype=dtype)] for x, y, w in zip(xs, ys, weights): xref = x.float() yref = y.float() wref = w.float() if isinstance(w, torch.Tensor) else w actual = torch.lerp(x, y, w) expected = torch.lerp(xref, yref, wref).to(dtype) self.assertEqual(actual, expected, atol=0.0, rtol=0.0) def _test_logaddexp(self, device, dtype, base2): if base2: ref_func = np.logaddexp2 our_func = torch.logaddexp2 elif dtype in (torch.complex64, torch.complex128): # numpy has not implemented logaddexp for complex def _ref_func(x, y): return scipy.special.logsumexp(np.stack((x, y), axis=0), axis=0) ref_func = _ref_func our_func = torch.logaddexp else: ref_func = np.logaddexp our_func = torch.logaddexp def _test_helper(a, b): if dtype == torch.bfloat16: ref = ref_func(a.cpu().float().numpy(), b.cpu().float().numpy()) v = our_func(a, b) self.assertEqual(ref, v.float(), atol=0.01, rtol=0.01) else: ref = ref_func(a.cpu().numpy(), b.cpu().numpy()) v = our_func(a, b) self.assertEqual(ref, v) # simple test a = torch.randn(64, 2, dtype=dtype, device=device) - 0.5 b = torch.randn(64, 2, dtype=dtype, device=device) - 0.5 _test_helper(a, b) _test_helper(a[:3], b[:3]) # large value test for numerical stability a *= 10000 b *= 10000 _test_helper(a, b) _test_helper(a[:3], b[:3]) a = torch.tensor( [float("inf"), float("-inf"), float("inf"), float("nan")], dtype=dtype, device=device, ) b = torch.tensor( [float("inf"), float("-inf"), float("-inf"), float("nan")], dtype=dtype, device=device, ) _test_helper(a, b) @skipIfTorchDynamo() # complex infs/nans differ under Dynamo/Inductor @dtypesIfCUDA(torch.float32, torch.float64, torch.bfloat16) @dtypes( torch.float32, torch.float64, torch.bfloat16, torch.complex64, torch.complex128 ) def test_logaddexp(self, device, dtype): self._test_logaddexp(device, dtype, base2=False) @dtypes(torch.float32, torch.float64, torch.bfloat16) def test_logaddexp2(self, device, dtype): self._test_logaddexp(device, dtype, base2=True) def test_add(self, device): dtypes = floating_and_complex_types() for dtype in dtypes: # [res] torch.add([res,] tensor1, tensor2) m1 = torch.randn(100, 100, dtype=dtype, device=device) v1 = torch.randn(100, dtype=dtype, device=device) # contiguous res1 = torch.add(m1[4], v1) res2 = res1.clone().zero_() for i in range(m1.size(1)): res2[i] = m1[4, i] + v1[i] self.assertEqual(res1, res2) m1 = torch.randn(100, 100, device=device) v1 = torch.randn(100, device=device) # non-contiguous res1 = torch.add(m1[:, 4], v1) res2 = res1.clone().zero_() for i in range(m1.size(0)): res2[i] = m1[i, 4] + v1[i] self.assertEqual(res1, res2) # [res] torch.add([res,] tensor, value) m1 = torch.randn(10, 10, device=device) # contiguous res1 = m1.clone() res1[3].add_(2) res2 = m1.clone() for i in range(m1.size(1)): res2[3, i] = res2[3, i] + 2 self.assertEqual(res1, res2) # non-contiguous m1 = torch.randn(10, 10, device=device) res1 = m1.clone() res1[:, 3].add_(2) res2 = m1.clone() for i in range(m1.size(0)): res2[i, 3] = res2[i, 3] + 2 self.assertEqual(res1, res2) # inter-type m1 = torch.randn(10, 10, dtype=dtype, device=device) self.assertEqual(m1 + 3, m1 + torch.tensor(3)) self.assertEqual(3 + m1, torch.tensor(3) + m1) # contiguous + non-contiguous m1 = torch.randn(10, 10, dtype=dtype, device=device) m2 = torch.randn(10, 10, dtype=dtype, device=device).t() res = m1 + m2 self.assertTrue(res.is_contiguous()) self.assertEqual(res, m1 + m2.contiguous()) # 1d + empty m1 = torch.tensor([1.0], dtype=dtype, device=device) m2 = torch.tensor([], dtype=dtype, device=device) self.assertEqual(m1 + m2, []) # inter-type unint8 one = torch.tensor(1, dtype=torch.uint8, device=device) self.assertEqual(torch.add(one, 1), 2) self.assertEqual(torch.add(one, 1).dtype, torch.uint8) # bool m1 = torch.tensor( [True, False, False, True, False, False], dtype=torch.bool, device=device ) m2 = torch.tensor( [True, True, False, False, False, True], dtype=torch.bool, device=device ) expected = torch.tensor( [True, True, False, True, False, True], dtype=torch.bool, device=device ) self.assertEqual(m1 + m2, expected) # fused multiply add a = torch.zeros(2, 3, dtype=torch.bool, device=device) res = torch.add(a, a, alpha=0) expected = torch.zeros(2, 3, device=device).bool() self.assertEqual(res, expected) # bfloat16 m1 = torch.tensor([1.0, 2.0], dtype=torch.bfloat16) m2 = torch.tensor([3.0, 4.0], dtype=torch.bfloat16) self.assertEqual(m1 + m2, torch.tensor([4.0, 6.0], dtype=torch.bfloat16)) # different alpha types m1 = torch.tensor([2 + 3j, 4 + 5j], dtype=torch.complex64, device=device) m2 = torch.tensor([4 + 5j, 2 + 3j], dtype=torch.complex64, device=device) # add complex numbers with float alpha res = torch.add(m1, m2, alpha=0.1) expected = torch.tensor( [2.4000 + 3.5000j, 4.2000 + 5.3000j], dtype=torch.complex64, device=device ) self.assertEqual(res, expected) # add complex numbers with complex alpha res = torch.add(m1, m2, alpha=complex(0.1, 0.2)) expected = torch.tensor( [1.4000 + 4.3000j, 3.6000 + 5.7000j], dtype=torch.complex64, device=device ) self.assertEqual(res, expected) # add complex numbers with integer alpha res = torch.add(m1, m2, alpha=2) expected = torch.tensor( [10.0 + 13.0j, 8.0 + 11.0j], dtype=torch.complex64, device=device ) self.assertEqual(res, expected) # mismatched alpha m1 = torch.tensor([1], dtype=torch.int8, device=device) m2 = torch.tensor([2], dtype=torch.int8, device=device) self.assertRaisesRegex( RuntimeError, r"Boolean alpha only supported for Boolean results\.", lambda: torch.add(m1, m2, alpha=True), ) self.assertRaisesRegex( RuntimeError, r"For integral input tensors, argument alpha must not be a floating point number\.", lambda: torch.add(m1, m2, alpha=1.0), ) # mismatched alpha, float / double tensor and complex alpha msg = r"For non-complex input tensors, argument alpha must not be a complex number\." m1 = torch.tensor([3.0, 4.0], device=device) m2 = torch.tensor([4.0, 3.0], device=device) self.assertRaisesRegex( RuntimeError, msg, lambda: torch.add(m1, m2, alpha=complex(0.1, 0.2)) ) m1 = torch.tensor([3.0, 4.0], dtype=torch.double, device=device) m2 = torch.tensor([4.0, 3.0], dtype=torch.double, device=device) self.assertRaisesRegex( RuntimeError, msg, lambda: torch.add(m1, m2, alpha=complex(0.1, 0.2)) ) # complex m1 = torch.tensor((4.0000 + 4.0000j), dtype=torch.complex64) m2 = torch.tensor(4.0, dtype=torch.float64) self.assertRaisesRegex( RuntimeError, r"result type ComplexFloat can't be cast to the desired output type Double", lambda: torch.add(m1, m1, out=m2), ) @onlyCUDA def test_addsub_half_tensor(self, device): x = torch.tensor([60000.0], dtype=torch.half, device=device) for op, y, alpha in ( (torch.add, torch.tensor([-60000.0], dtype=torch.half, device=device), 2), (torch.sub, torch.tensor([60000.0], dtype=torch.half, device=device), 2), (torch.add, -70000.0, 1), (torch.sub, 70000.0, 1), ): actual = op(x, y, alpha=alpha) self.assertTrue(not (actual.isnan() or actual.isinf())) def test_sub_typing(self, device): m1 = torch.tensor( [True, False, False, True, False, False], dtype=torch.bool, device=device ) m2 = torch.tensor( [True, True, False, False, False, True], dtype=torch.bool, device=device ) self.assertRaisesRegex( RuntimeError, r"Subtraction, the `\-` operator, with two bool tensors is not supported. " r"Use the `\^` or `logical_xor\(\)` operator instead.", lambda: m1 - m2, ) self.assertRaisesRegex( RuntimeError, r"Subtraction, the `\-` operator, with a bool tensor is not supported. " r"If you are trying to invert a mask, use the `\~` or `logical_not\(\)` operator instead.", lambda: 1 - m1, ) self.assertRaisesRegex( RuntimeError, r"Subtraction, the `\-` operator, with a bool tensor is not supported. " r"If you are trying to invert a mask, use the `\~` or `logical_not\(\)` operator instead.", lambda: m2 - 1, ) # mismatched alpha m1 = torch.tensor([1], dtype=torch.int8, device=device) m2 = torch.tensor([2], dtype=torch.int8, device=device) self.assertRaisesRegex( RuntimeError, r"Boolean alpha only supported for Boolean results\.", lambda: torch.sub(m1, m2, alpha=True), ) self.assertRaisesRegex( RuntimeError, r"For integral input tensors, argument alpha must not be a floating point number\.", lambda: torch.sub(m1, m2, alpha=1.0), ) def test_mul(self, device): m1 = torch.randn(10, 10, device=device) res1 = m1.clone() res1[:, 3].mul_(2) res2 = m1.clone() for i in range(res1.size(0)): res2[i, 3] = res2[i, 3] * 2 self.assertEqual(res1, res2) a1 = torch.tensor([True, False, False, True], dtype=torch.bool, device=device) a2 = torch.tensor([True, False, True, False], dtype=torch.bool, device=device) self.assertEqual( a1 * a2, torch.tensor([True, False, False, False], dtype=torch.bool, device=device), ) if device == "cpu": a1 = torch.tensor([0.1, 0.1], dtype=torch.bfloat16, device=device) a2 = torch.tensor([1.1, 0.1], dtype=torch.bfloat16, device=device) self.assertEqual( a1 * a2, torch.tensor([0.11, 0.01], dtype=torch.bfloat16, device=device), atol=0.01, rtol=0, ) self.assertEqual(a1.mul(a2), a1 * a2) def test_bool_tensor_comparison_ops(self, device): a = torch.tensor( [True, False, True, False, True, False], dtype=torch.bool, device=device ) b = torch.tensor( [True, False, True, True, True, True], dtype=torch.bool, device=device ) self.assertEqual( a == b, torch.tensor([1, 1, 1, 0, 1, 0], dtype=torch.bool, device=device) ) self.assertEqual( a != b, torch.tensor([0, 0, 0, 1, 0, 1], dtype=torch.bool, device=device) ) self.assertEqual( a < b, torch.tensor([0, 0, 0, 1, 0, 1], dtype=torch.bool, device=device) ) self.assertEqual( a > b, torch.tensor([0, 0, 0, 0, 0, 0], dtype=torch.bool, device=device) ) self.assertEqual( a >= b, torch.tensor([1, 1, 1, 0, 1, 0], dtype=torch.bool, device=device) ) self.assertEqual( a <= b, torch.tensor([1, 1, 1, 1, 1, 1], dtype=torch.bool, device=device) ) self.assertEqual( a > False, torch.tensor([1, 0, 1, 0, 1, 0], dtype=torch.bool, device=device) ) self.assertEqual( a == torch.tensor(True, dtype=torch.bool, device=device), torch.tensor([1, 0, 1, 0, 1, 0], dtype=torch.bool, device=device), ) self.assertEqual( a == torch.tensor(0, dtype=torch.bool, device=device), torch.tensor([0, 1, 0, 1, 0, 1], dtype=torch.bool, device=device), ) self.assertFalse(a.equal(b)) @dtypes(*all_types_and(torch.half, torch.bfloat16, torch.bool)) def test_logical(self, device, dtype): if dtype != torch.bool: x = torch.tensor([1, 2, 3, 4], device=device, dtype=dtype) b = torch.tensor([2], device=device, dtype=dtype) self.assertEqual(x.lt(2), torch.tensor([True, False, False, False])) self.assertEqual(x.le(2), torch.tensor([True, True, False, False])) self.assertEqual(x.ge(2), torch.tensor([False, True, True, True])) self.assertEqual(x.gt(2), torch.tensor([False, False, True, True])) self.assertEqual(x.eq(2), torch.tensor([False, True, False, False])) self.assertEqual(x.ne(2), torch.tensor([True, False, True, True])) self.assertEqual(x.lt(b), torch.tensor([True, False, False, False])) self.assertEqual(x.le(b), torch.tensor([True, True, False, False])) self.assertEqual(x.ge(b), torch.tensor([False, True, True, True])) self.assertEqual(x.gt(b), torch.tensor([False, False, True, True])) self.assertEqual(x.eq(b), torch.tensor([False, True, False, False])) self.assertEqual(x.ne(b), torch.tensor([True, False, True, True])) else: x = torch.tensor([True, False, True, False], device=device) self.assertEqual(x.lt(True), torch.tensor([False, True, False, True])) self.assertEqual(x.le(True), torch.tensor([True, True, True, True])) self.assertEqual(x.ge(True), torch.tensor([True, False, True, False])) self.assertEqual(x.gt(True), torch.tensor([False, False, False, False])) self.assertEqual(x.eq(True), torch.tensor([True, False, True, False])) self.assertEqual(x.ne(True), torch.tensor([False, True, False, True])) def test_atan2(self, device): def _test_atan2_with_size(size, device): a = torch.rand(size=size, device=device, dtype=torch.double) b = torch.rand(size=size, device=device, dtype=torch.double) actual = a.atan2(b) x = a.view(-1) y = b.view(-1) expected = torch.tensor( [math.atan2(x[i].item(), y[i].item()) for i in range(x.numel())], device=device, dtype=torch.double, ) self.assertEqual(expected, actual.view(-1), rtol=0, atol=0.02) # bfloat16/float16 for lowp_dtype in [torch.bfloat16, torch.float16]: if lowp_dtype == torch.bfloat16: rtol = 0 atol = 0.02 else: rtol = 0 atol = 0.001 a_16 = a.to(dtype=lowp_dtype) b_16 = b.to(dtype=lowp_dtype) actual_16 = a_16.atan2(b_16) self.assertEqual(actual_16, actual.to(dtype=lowp_dtype)) self.assertEqual( expected, actual_16.view(-1), exact_dtype=False, rtol=rtol, atol=atol, ) _test_atan2_with_size((2, 2), device) _test_atan2_with_size((3, 3), device) _test_atan2_with_size((5, 5), device) def test_atan2_edgecases(self, device): def _test_atan2(x, y, expected, device, dtype): expected_tensor = torch.tensor([expected], dtype=dtype, device=device) x_tensor = torch.tensor([x], dtype=dtype, device=device) y_tensor = torch.tensor([y], dtype=dtype, device=device) actual = torch.atan2(y_tensor, x_tensor) self.assertEqual(expected_tensor, actual, rtol=0, atol=0.02) for dtype in [torch.float, torch.double]: _test_atan2(0, 0, 0, device, dtype) _test_atan2(0, 1, math.pi / 2, device, dtype) _test_atan2(0, -1, math.pi / -2, device, dtype) _test_atan2(-1, 0, math.pi, device, dtype) _test_atan2(1, 0, 0, device, dtype) _test_atan2(-1, -1, math.pi * -3 / 4, device, dtype) _test_atan2(1, 1, math.pi / 4, device, dtype) _test_atan2(1, -1, math.pi / -4, device, dtype) _test_atan2(-1, 1, math.pi * 3 / 4, device, dtype) def test_trapezoid(self, device): def test_dx(sizes, dim, dx, device): t = torch.randn(sizes, device=device) actual = torch.trapezoid(t, dx=dx, dim=dim) expected = np.trapz(t.cpu().numpy(), dx=dx, axis=dim) # noqa: NPY201 self.assertEqual(expected.shape, actual.shape) self.assertEqual(expected, actual, exact_dtype=False) def test_x(sizes, dim, x, device): t = torch.randn(sizes, device=device) actual = torch.trapezoid(t, x=torch.tensor(x, device=device), dim=dim) expected = np.trapz(t.cpu().numpy(), x=x, axis=dim) # noqa: NPY201 self.assertEqual(expected.shape, actual.shape) self.assertEqual(expected, actual.cpu(), exact_dtype=False) test_dx((2, 3, 4), 1, 1, device) test_dx((10, 2), 0, 0.1, device) test_dx((1, 10), 0, 2.3, device) test_dx((0, 2), 0, 1.0, device) test_dx((0, 2), 1, 1.0, device) test_x((2, 3, 4), 1, [1.0, 2.0, 3.0], device) test_x( (10, 2), 0, [2.0, 3.0, 4.0, 7.0, 11.0, 14.0, 22.0, 26.0, 26.1, 30.3], device ) test_x((1, 10), 0, [1.0], device) test_x((0, 2), 0, [], device) test_x((0, 2), 1, [1.0, 2.0], device) test_x((2, 3, 4), -1, [1.0, 2.0, 3.0, 4.0], device) test_x((2, 3, 4), 0, [1.0, 2.0], device) test_x((2, 3, 4), 1, [1.0, 2.0, 3.0], device) test_x((2, 3, 4), 2, [1.0, 2.0, 3.0, 4.0], device) test_x((2, 2, 4), -1, [[1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 3.0, 4.0]], device) with self.assertRaisesRegex(IndexError, "Dimension out of range"): test_x((2, 3), 2, [], device) test_dx((2, 3), 2, 1.0, device) with self.assertRaisesRegex( RuntimeError, "There must be one `x` value for each sample point" ): test_x((2, 3), 1, [1.0, 2.0], device) test_x((2, 3), 1, [1.0, 2.0, 3.0, 4.0], device) @skipIf(not TEST_SCIPY, "Scipy required for the test.") def test_cumulative_trapezoid(self, device): import scipy.integrate if hasattr(scipy.integrate, "cumulative_trapezoid"): _scipy_cumulative_trapezoid = scipy.integrate.cumulative_trapezoid else: # Older version of SciPy uses a different name _scipy_cumulative_trapezoid = scipy.integrate.cumtrapz def scipy_cumulative_trapezoid(y, x=None, dx=1.0, axis=-1, initial=None): if y.shape[axis] == 0: return np.empty_like(y) else: return _scipy_cumulative_trapezoid(y, x, dx, axis, initial) def test_dx(sizes, dim, dx, device): t = torch.randn(sizes, device=device) y = t.cpu().numpy() actual = torch.cumulative_trapezoid(t, dx=dx, dim=dim) expected = scipy_cumulative_trapezoid(t.cpu().numpy(), dx=dx, axis=dim) self.assertEqual(expected.shape, actual.shape) self.assertEqual(expected, actual, exact_dtype=False, atol=1e-4, rtol=1e-4) def test_x(sizes, dim, x, device): t = torch.randn(sizes, device=device) actual = torch.cumulative_trapezoid( t, x=torch.tensor(x, device=device), dim=dim ) expected = scipy_cumulative_trapezoid(t.cpu().numpy(), x=x, axis=dim) self.assertEqual(expected.shape, actual.shape) self.assertEqual( expected, actual.cpu(), exact_dtype=False, atol=1e-4, rtol=1e-4 ) def test_empty_x(sizes, dim, x, device): t = torch.randn(sizes, device=device) actual = torch.cumulative_trapezoid( t, x=torch.tensor(x, device=device), dim=dim ) self.assertEqual(torch.empty(actual.shape), actual) test_dx((2,), -1, 1, device) test_dx((3, 3), -1, 1, device) test_dx((4, 2), 0, 1, device) test_dx((2, 3, 4), 1, 1, device) test_dx((10, 2), 0, 0.1, device) test_dx((1, 10), 0, 2.3, device) test_dx((0, 2), 0, 1.0, device) test_dx((0, 2), 1, 1.0, device) test_dx((512, 512), 1, 1.0, device) test_dx((100, 100, 100), 1, 1.0, device) test_x((2,), -1, [100, 50], device) test_x((4, 2), 0, [2, 3, 4, 5], device) test_x((2, 3, 4), 1, [1.0, 2.0, 3.0], device) test_x( (10, 2), 0, [2.0, 3.0, 4.0, 7.0, 11.0, 14.0, 22.0, 26.0, 26.1, 30.3], device ) test_x((1, 10), 0, [1.0], device) test_x((0, 2), 1, [1, 2], device) test_x((2, 3, 4), -1, [1.0, 2.0, 3.0, 4.0], device) test_x((2, 3, 4), 0, [1.0, 2.0], device) test_x((2, 3, 4), 1, [1.0, 2.0, 3.0], device) test_x((2, 3, 4), 2, [1.0, 2.0, 3.0, 4.0], device) test_empty_x( (0, 2), 0, [], device ) # SciPy failing when x == [], but our version returns empty with self.assertRaisesRegex(IndexError, "Dimension out of range"): test_x((2, 3), 2, [], device) test_dx((2, 3), 2, 1.0, device) with self.assertRaisesRegex( RuntimeError, "There must be one `x` value for each sample point" ): test_x((2, 3), 1, [1.0, 2.0], device) test_x((0, 2), 0, [1.0, 2.0], device) test_x((2, 3), 1, [1.0, 2.0, 3.0, 4.0], device) with self.assertRaisesRegex( RuntimeError, "Currently, we only support dx as a real number" ): test_dx((2, 2), -1, complex(1, 1), device) with self.assertRaisesRegex( TypeError, "received an invalid combination of arguments" ): actual = torch.cumulative_trapezoid( torch.randn((3, 3)), x=torch.randn((3, 3)), dx=3 ) @skipMeta @dtypes(torch.double) def test_pow_scalar_overloads_mem_overlap(self, device, dtype): sz = 3 doubles = torch.randn(2 * sz, dtype=dtype, device=device) self.check_internal_mem_overlap(lambda t: t.pow_(42), 1, dtype, device) self.unary_check_input_output_mem_overlap( doubles, sz, lambda input, out: torch.pow(input, 42, out=out) ) self.unary_check_input_output_mem_overlap( doubles, sz, lambda input, out: torch.pow(42, input, out=out) ) @dtypes( *list( product( all_types_and_complex_and(torch.half, torch.bfloat16), all_types_and_complex_and(torch.half, torch.bfloat16), ) ) ) def test_float_power(self, device, dtypes): def to_np(value): if isinstance(value, torch.Tensor) and value.dtype == torch.bfloat16: return value.to(torch.float).cpu().numpy() return value.cpu().numpy() if isinstance(value, torch.Tensor) else value base_dtype = dtypes[0] exp_dtype = dtypes[1] out_dtype = ( torch.complex128 if base_dtype.is_complex or exp_dtype.is_complex else torch.float64 ) base = make_tensor((30,), dtype=base_dtype, device=device, low=1, high=100) # Complex and real results do not agree between PyTorch and NumPy when computing negative and zero power of 0 # Related: https://github.com/pytorch/pytorch/issues/48000 # base[0] = base[3] = base[7] = 0 exp = make_tensor((30,), dtype=exp_dtype, device=device, low=-2, high=2) exp[0] = exp[4] = exp[6] = 0 expected = torch.from_numpy(np.float_power(to_np(base), to_np(exp))) exponents = [-2.8, -2, -1, -0.5, 0.5, 1, 2] complex_exponents = exponents + [ -2.5j, -1.0j, 1.0j, 2.5j, 1.0 + 1.0j, -1.0 - 1.5j, 3.3j, ] for op in ( torch.float_power, torch.Tensor.float_power, torch.Tensor.float_power_, ): # Case of Tensor x Tensor if op is torch.Tensor.float_power_ and base_dtype != out_dtype: with self.assertRaisesRegex( RuntimeError, "operation's result requires dtype" ): op(base.clone(), exp) else: result = op(base.clone(), exp) self.assertEqual(expected, result) if op is torch.float_power: out = torch.empty_like(base).to(device=device, dtype=out_dtype) op(base, exp, out=out) self.assertEqual(expected, out) # Case of Tensor x Scalar for i in complex_exponents if exp_dtype.is_complex else exponents: out_dtype_scalar_exp = ( torch.complex128 if base_dtype.is_complex or type(i) == complex else torch.float64 ) expected_scalar_exp = torch.from_numpy(np.float_power(to_np(base), i)) if ( op is torch.Tensor.float_power_ and base_dtype != out_dtype_scalar_exp ): with self.assertRaisesRegex( RuntimeError, "operation's result requires dtype" ): op(base.clone(), i) else: result = op(base.clone(), i) self.assertEqual(expected_scalar_exp, result) if op is torch.float_power: out = torch.empty_like(base).to( device=device, dtype=out_dtype_scalar_exp ) op(base, i, out=out) self.assertEqual(expected_scalar_exp, out) # Case of Scalar x Tensor for i in complex_exponents if base_dtype.is_complex else exponents: out_dtype_scalar_base = ( torch.complex128 if exp_dtype.is_complex or type(i) == complex else torch.float64 ) expected_scalar_base = torch.from_numpy(np.float_power(i, to_np(exp))) result = torch.float_power(i, exp) self.assertEqual(expected_scalar_base, result) out = torch.empty_like(exp).to(device=device, dtype=out_dtype_scalar_base) torch.float_power(i, exp, out=out) self.assertEqual(expected_scalar_base, out) def test_float_power_exceptions(self, device): def _promo_helper(x, y): for i in (x, y): if type(i) == complex: return torch.complex128 elif type(i) == torch.Tensor and i.is_complex(): return torch.complex128 return torch.double test_cases = ( (torch.tensor([-2, -1, 0, 1, 2], device=device), -0.25), ( torch.tensor([-1.0j, 0j, 1.0j, 1.0 + 1.0j, -1.0 - 1.5j], device=device), 2.0, ), ) for base, exp in test_cases: for out_dtype in (torch.long, torch.float, torch.double, torch.cdouble): out = torch.empty(1, device=device, dtype=out_dtype) required_dtype = _promo_helper(base, exp) if out.dtype == required_dtype: torch.float_power(base, exp, out=out) else: with self.assertRaisesRegex( RuntimeError, "operation's result requires dtype" ): torch.float_power(base, exp, out=out) if base.dtype == required_dtype: torch.Tensor.float_power_(base.clone(), exp) else: with self.assertRaisesRegex( RuntimeError, "operation's result requires dtype" ): torch.Tensor.float_power_(base.clone(), exp) @skipIf(not TEST_SCIPY, "Scipy required for the test.") @dtypes( *product( all_types_and(torch.half, torch.bool), all_types_and(torch.half, torch.bool) ) ) def test_xlogy_xlog1py(self, device, dtypes): x_dtype, y_dtype = dtypes def out_variant_helper(torch_fn, x, y): expected = torch_fn(x, y) out = torch.empty_like(expected) torch_fn(x, y, out=out) self.assertEqual(expected, out) def xlogy_inplace_variant_helper(x, y): if x.dtype in integral_types_and(torch.bool): with self.assertRaisesRegex( RuntimeError, "can't be cast to the desired output type" ): x.clone().xlogy_(y) else: expected = torch.empty_like(x) torch.xlogy(x, y, out=expected) inplace_out = x.clone().xlogy_(y) self.assertEqual(expected, inplace_out) def test_helper(torch_fn, reference_fn, inputs, scalar=None): x, y, z = inputs torch_fn_partial = partial(torch_fn, x) reference_fn_partial = partial(reference_fn, x.cpu().numpy()) self.compare_with_numpy( torch_fn_partial, reference_fn_partial, x, exact_dtype=False ) self.compare_with_numpy( torch_fn_partial, reference_fn_partial, y, exact_dtype=False ) self.compare_with_numpy( torch_fn_partial, reference_fn_partial, z, exact_dtype=False ) val = scalar if scalar is not None else x out_variant_helper(torch_fn, val, x) out_variant_helper(torch_fn, val, y) out_variant_helper(torch_fn, val, z) # Tensor-Tensor Test (tensor of same and different shape) x = make_tensor((3, 2, 4, 5), dtype=x_dtype, device=device, low=0.5, high=1000) y = make_tensor((3, 2, 4, 5), dtype=y_dtype, device=device, low=0.5, high=1000) z = make_tensor((4, 5), dtype=y_dtype, device=device, low=0.5, high=1000) x_1p = make_tensor( (3, 2, 4, 5), dtype=x_dtype, device=device, low=-0.5, high=1000 ) y_1p = make_tensor( (3, 2, 4, 5), dtype=y_dtype, device=device, low=-0.5, high=1000 ) z_1p = make_tensor((4, 5), dtype=y_dtype, device=device, low=-0.5, high=1000) xlogy_fns = torch.xlogy, scipy.special.xlogy xlog1py_fns = torch.special.xlog1py, scipy.special.xlog1py test_helper(*xlogy_fns, (x, y, z)) xlogy_inplace_variant_helper(x, x) xlogy_inplace_variant_helper(x, y) xlogy_inplace_variant_helper(x, z) test_helper(*xlog1py_fns, (x_1p, y_1p, z_1p)) # Scalar-Tensor Test test_helper(*xlogy_fns, (x, y, z), 3.14) test_helper(*xlog1py_fns, (x_1p, y_1p, z_1p), 3.14) # Special Values Tensor-Tensor t = torch.tensor( [-1.0, 0.0, 1.0, 2.0, float("inf"), -float("inf"), float("nan")], device=device, ) zeros = torch.zeros(7, dtype=y_dtype, device=device) def test_zeros_special_helper(torch_fn, reference_fn, scalar=False): zeros_t = 0 if scalar else zeros zeros_np = 0 if scalar else zeros.cpu().numpy() torch_fn_partial = partial(torch_fn, zeros_t) reference_fn_partial = partial(reference_fn, zeros_np) self.compare_with_numpy( torch_fn_partial, reference_fn_partial, t, exact_dtype=False ) out_variant_helper(torch_fn, zeros_t, t) test_zeros_special_helper(*xlogy_fns) xlogy_inplace_variant_helper(zeros, t) test_zeros_special_helper(*xlog1py_fns) # Special Values Scalar-Tensor test_zeros_special_helper(*xlogy_fns, scalar=True) test_zeros_special_helper(*xlog1py_fns, scalar=True) @dtypes(torch.float64) def test_xlogy_xlog1py_gradients(self, device, dtype): make_arg = partial(torch.tensor, dtype=dtype, device=device, requires_grad=True) zeros = torch.zeros((2,), dtype=dtype, device=device) x = make_arg([0.0, 0.0]) y = make_arg([-1.5, 0.0]) torch.special.xlogy(x, y).sum().backward() self.assertEqual(x.grad, zeros) x = make_arg([0.0, 0.0]) y = make_arg([-2.5, -1.0]) torch.special.xlog1py(x, y).sum().backward() self.assertEqual(x.grad, zeros) def test_xlogy_xlog1py_scalar_type_promotion(self, device): # Test that python numbers don't participate in type promotion at the same # priority level as 0-dim tensors t = torch.randn((), dtype=torch.float32, device=device) self.assertEqual(t.dtype, torch.xlogy(t, 5).dtype) self.assertEqual(t.dtype, torch.xlogy(t, 5.0).dtype) self.assertEqual(t.dtype, torch.special.xlog1py(t, 5).dtype) self.assertEqual(t.dtype, torch.special.xlog1py(t, 5.0).dtype) self.assertEqual(t.dtype, torch.xlogy(5, t).dtype) self.assertEqual(t.dtype, torch.xlogy(5.0, t).dtype) self.assertEqual(t.dtype, torch.special.xlog1py(5, t).dtype) self.assertEqual(t.dtype, torch.special.xlog1py(5.0, t).dtype) @skipIf(not TEST_SCIPY, "Scipy required for the test.") def test_xlogy_xlog1py_bfloat16(self, device): def _compare_helper(x, y, torch_fn, reference_fn): x_np = x if isinstance(x, float) else x.cpu().to(torch.float).numpy() y_np = y if isinstance(y, float) else y.cpu().to(torch.float).numpy() expected = torch.from_numpy(reference_fn(x_np, y_np)) actual = torch_fn(x, y) self.assertEqual(expected, actual, exact_dtype=False) x_dtype, y_dtype = torch.bfloat16, torch.bfloat16 # Tensor-Tensor Test (tensor of same and different shape) x = make_tensor((3, 2, 4, 5), dtype=x_dtype, device=device, low=0.5, high=1000) y = make_tensor((3, 2, 4, 5), dtype=y_dtype, device=device, low=0.5, high=1000) z = make_tensor((4, 5), dtype=y_dtype, device=device, low=0.5, high=1000) x_1p = make_tensor( (3, 2, 4, 5), dtype=x_dtype, device=device, low=-0.8, high=1000 ) y_1p = make_tensor( (3, 2, 4, 5), dtype=y_dtype, device=device, low=-0.8, high=1000 ) z_1p = make_tensor((4, 5), dtype=y_dtype, device=device, low=-0.8, high=1000) xlogy_fns = torch.xlogy, scipy.special.xlogy xlog1py_fns = torch.special.xlog1py, scipy.special.xlog1py _compare_helper(x, x, *xlogy_fns) _compare_helper(x, y, *xlogy_fns) _compare_helper(x, z, *xlogy_fns) _compare_helper(x, 3.14, *xlogy_fns) _compare_helper(y, 3.14, *xlogy_fns) _compare_helper(z, 3.14, *xlogy_fns) _compare_helper(x_1p, x_1p, *xlog1py_fns) _compare_helper(x_1p, y_1p, *xlog1py_fns) _compare_helper(x_1p, z_1p, *xlog1py_fns) _compare_helper(x_1p, 3.14, *xlog1py_fns) _compare_helper(y_1p, 3.14, *xlog1py_fns) _compare_helper(z_1p, 3.14, *xlog1py_fns) # Special Values Tensor-Tensor t = torch.tensor( [-1.0, 0.0, 1.0, 2.0, float("inf"), -float("inf"), float("nan")], device=device, ) zeros = torch.tensor(7, dtype=y_dtype, device=device) _compare_helper(t, zeros, *xlogy_fns) _compare_helper(t, 0.0, *xlogy_fns) _compare_helper(t, zeros, *xlog1py_fns) _compare_helper(t, 0.0, *xlog1py_fns) @dtypes(*product(all_types_and(torch.bool), all_types_and(torch.bool))) @skipIf(not TEST_SCIPY, "Scipy required for the test.") @slowTest def test_zeta(self, device, dtypes): x_dtype, q_dtype = dtypes def test_helper(x, q): x_np = x if isinstance(x, float) else x.cpu().numpy() q_np = q if isinstance(q, float) else q.cpu().numpy() expected = torch.from_numpy(scipy.special.zeta(x_np, q_np)) actual = torch.special.zeta(x, q) rtol, atol = None, None if self.device_type == "cpu": rtol, atol = 1e-6, 1e-6 self.assertEqual(expected, actual, rtol=rtol, atol=atol, exact_dtype=False) # x tensor - q tensor same size x = make_tensor((2, 3, 4), dtype=x_dtype, device=device) q = make_tensor((2, 3, 4), dtype=q_dtype, device=device) test_helper(x, q) # x tensor - q tensor broadcast lhs x = make_tensor((2, 1, 4), dtype=x_dtype, device=device) q = make_tensor((2, 3, 4), dtype=q_dtype, device=device) test_helper(x, q) # x tensor - q tensor broadcast rhs x = make_tensor((2, 3, 4), dtype=x_dtype, device=device) q = make_tensor((2, 1, 4), dtype=q_dtype, device=device) test_helper(x, q) # x tensor - q tensor broadcast all x = make_tensor((2, 3, 1), dtype=x_dtype, device=device) q = make_tensor((2, 1, 4), dtype=q_dtype, device=device) test_helper(x, q) # x scalar - q tensor for x in np.linspace(-5, 5, num=10).tolist(): if not q_dtype.is_floating_point: q_dtype = torch.get_default_dtype() q = make_tensor((2, 3, 4), dtype=q_dtype, device=device) test_helper(x, q) # x tensor - q scalar for q in np.linspace(-5, 5, num=10).tolist(): if not x_dtype.is_floating_point: x_dtype = torch.get_default_dtype() x = make_tensor((2, 3, 4), dtype=x_dtype, device=device) test_helper(x, q) @onlyCUDA @dtypes(torch.chalf) def test_mul_chalf_tensor_and_cpu_scalar(self, device, dtype): # Tests that Tensor and CPU Scalar work for `mul` for chalf. # Ideally, this should be covered by `test_complex_half_reference_testing` # from test_ops.py by checking reference_samples from the OpInfo. # But currently that doesn't work as sample generation requires support of # `index_select` which is not implemented for `complex32` at the # time of writing this test. # TODO: Remove this test once above issue is fixed. # Ref: https://github.com/pytorch/pytorch/pull/76364 x = make_tensor((2, 2), device=device, dtype=dtype) self.assertEqual(x * 2.5, x * torch.tensor(2.5, device=device, dtype=dtype)) tensor_binary_ops = [ "__lt__", "__le__", "__gt__", "__ge__", "__eq__", "__ne__", "__add__", "__radd__", "__iadd__", "__sub__", "__rsub__", "__isub__", "__mul__", "__rmul__", "__imul__", "__matmul__", "__rmatmul__", "__truediv__", "__rtruediv__", "__itruediv__", "__floordiv__", "__rfloordiv__", "__ifloordiv__", "__mod__", "__rmod__", "__imod__", "__pow__", "__rpow__", "__ipow__", "__lshift__", "__rlshift__", "__ilshift__", "__rshift__", "__rrshift__", "__irshift__", "__and__", "__rand__", "__iand__", "__xor__", "__rxor__", "__ixor__", "__or__", "__ror__", "__ior__", # Unsupported operators # '__imatmul__', # '__divmod__', '__rdivmod__', '__idivmod__', ] # Test that binary math operations return NotImplemented for unknown types. def generate_not_implemented_tests(cls): class UnknownType: pass # TODO: refactor to inline these _types = [ torch.half, torch.float, torch.double, torch.int8, torch.short, torch.int, torch.long, torch.uint8, ] def create_test_func(op): @dtypes(*_types) def test(self, device, dtype): # Generate the inputs tensor = torch.empty((), device=device, dtype=dtype) # Runs the tensor op on the device result = getattr(tensor, op)(UnknownType()) self.assertEqual(result, NotImplemented) return test for op in tensor_binary_ops: test_name = f"test_{op}_not_implemented" assert not hasattr(cls, test_name), f"{test_name} already in {cls.__name__}" setattr(cls, test_name, create_test_func(op)) generate_not_implemented_tests(TestBinaryUfuncs) instantiate_device_type_tests(TestBinaryUfuncs, globals()) if __name__ == "__main__": run_tests()