Add new autotest (#5562)

* fix upsample nearest bug

* fix upsample nearest bug (#5347)

Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>

* fix upsample bilinear bug

* add new autotest case

* add more tests

* fix bug

* auto format by CI

* add rtol and atol

* add rtol and atol

* add nothing

* auto format by CI

* fix export bug

* fix bug

* fix bug

* fix bug

* fix bug

* auto format by CI

* fix bug

* fix bug

* fix ci error

* fix bug

* auto format by CI

Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>
Co-authored-by: oneflow-ci-bot <ci-bot@oneflow.org>

ci/test/generic_test.sh

@@ -7,7 +7,7 @@ export PYTHONUNBUFFERED=1
 src_dir=${ONEFLOW_SRC_DIR:-"$PWD"}
 test_dir=${ONEFLOW_TEST_DIR:-"$PWD/oneflow/python/test/ops"}
 test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
-
+export ONEFLOW_TEST_UTILS_DIR=$src_dir/oneflow/python/test_utils
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir

oneflow/python/test/modules/test_activation.py

@@ -62,11 +62,15 @@ class TestReLUModule(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_relu_impl(test_case, *arg)
 
+    @autotest
     def test_relu_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case, "nn.ReLU", device=device,
-            )
+        m = torch.nn.ReLU()
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
 
 
 def _test_relu6_impl(test_case, shape, device):
@@ -101,11 +105,15 @@ class TestReLU6Module(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_relu6_impl(test_case, *arg)
 
+    @autotest
     def test_relu6_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case, "nn.ReLU6", device=device,
-            )
+        m = torch.nn.ReLU6()
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
 
 
 def _test_tanh_nn_impl(test_case, shape, device):
@@ -153,23 +161,22 @@ class TestTanh(flow.unittest.TestCase):
             _test_tanh_nn_impl(test_case, *arg)
             _test_tanh_function_impl(test_case, *arg)
 
+    @autotest
     def test_tanh_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case, "nn.Tanh", device=device,
-            )
-
+        m = torch.nn.Tanh()
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
+
+    @autotest
     def test_flow_tanh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "tanh", device=device,
-            )
-
-    def test_tensor_tanh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "tanh", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = flow.tanh(x)
+        return y
 
 
 def _test_elu_function_impl(test_case, shape, device):
@@ -202,15 +209,15 @@ class TestELUModule(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_elu_function_impl(test_case, *arg)
 
+    @autotest
     def test_elu_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case,
-                "nn.ELU",
-                extra_annotations={"alpha": float},
-                extra_generators={"alpha": random(0, 6)},
-                device=device,
-            )
+        m = torch.nn.ELU(alpha=random() | nothing())
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
 
 
 def _np_gelu(x):
@@ -694,16 +701,15 @@ class TestSoftplusModule(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             arg[0](test_case, *arg[1:])
 
-    @unittest.skip("Pytorch Softplus has bug")
+    @autotest
     def test_softplus_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case,
-                "nn.Softplus",
-                extra_annotations={"beta": int, "threshold": int},
-                extra_generators={"beta": random(3, 4), "threshold": random(1, 2)},
-                device=device,
-            )
+        m = torch.nn.Softplus(beta=random() | nothing(), threshold=random() | nothing())
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
 
 
 def _test_hardswish_impl(test_case, shape, device):
@@ -797,16 +803,15 @@ class TestLeakyReLUModule(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_leakyrelu_impl(test_case, *arg)
 
+    @autotest
     def test_leakyrelu_module_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_module_against_pytorch(
-                test_case,
-                "nn.LeakyReLU",
-                extra_annotations={"negative_slope": float},
-                extra_generators={"negative_slope": random(0, 6)},
-                device=device,
-                n=2,
-            )
+        m = torch.nn.LeakyReLU(negative_slope=random() | nothing())
+        m.train(random())
+        device = random_device()
+        m.to(device)
+        x = random_pytorch_tensor().to(device)
+        y = m(x)
+        return y
 
 
 def _test_mish(test_case, shape, device):

oneflow/python/test/modules/test_math_ops.py

@@ -21,7 +21,6 @@ import numpy as np
 import oneflow.experimental as flow
 from test_util import GenArgList, type_name_to_flow_type, type_name_to_np_type
 from automated_test_util import *
-import torch
 
 
 def _test_variance_keepdim(test_case, shape, device):
@@ -101,17 +100,12 @@ class Testsinh(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_sinh_impl(test_case, *arg)
 
+    @autotest()
     def test_flow_sinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "sinh", device=device,
-            )
-
-    def test_flow_tensor_sinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "sinh", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.sinh(x)
+        return y
 
 
 def _test_sin(test_case, shape, device):
@@ -480,29 +474,21 @@ class TestAsin(flow.unittest.TestCase):
             _test_asin(test_case, *arg)
             _test_arcsin(test_case, *arg)
 
+    @unittest.skip("asin has bug")
+    @autotest()
     def test_flow_asin_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "asin", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.asin(x)
+        return y
 
+    @unittest.skip("arcsin has bug")
+    @autotest()
     def test_flow_arcsin_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "arcsin", device=device,
-            )
-
-    def test_flow_tensor_asin_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "asin", device=device,
-            )
-
-    def test_flow_tensor_arcsin_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "arcsin", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.arcsin(x)
+        return y
 
 
 def _test_asinh(test_case, shape, device):
@@ -549,29 +535,19 @@ class TestAsinh(flow.unittest.TestCase):
             _test_asinh(test_case, *arg)
             _test_arcsinh(test_case, *arg)
 
+    @autotest()
     def test_flow_asinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "asinh", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.asinh(x)
+        return y
 
+    @autotest()
     def test_flow_arcsinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case, "arcsinh", device=device,
-            )
-
-    def test_flow_tensor_asinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "asinh", device=device,
-            )
-
-    def test_flow_tensor_arcsinh_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case, "arcsinh", device=device,
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.arcsinh(x)
+        return y
 
 
 def _topk_np(input, k, dim: int = None, largest: bool = True, _sorted: bool = True):
@@ -764,25 +740,13 @@ def arccosh_input_tensor(shape):
 )
 @flow.unittest.skip_unless_1n1d()
 class TestArccosh(flow.unittest.TestCase):
+    @unittest.skip("arccosh has bug")
+    @autotest()
     def test_arccosh_flow_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case,
-                "arccosh",
-                device=device,
-                n=2,
-                extra_generators={"input": arccosh_input_tensor((3, 3))},
-            )
-
-    def test_arccosh_tensor_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case,
-                "arccosh",
-                device=device,
-                n=2,
-                extra_generators={"input": arccosh_input_tensor((3, 3))},
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.arccosh(x)
+        return y
 
 
 def _test_acosh_impl(test_case, shape, device):
@@ -831,25 +795,13 @@ class TestAcosh(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_acosh_impl(test_case, *arg)
 
+    @unittest.skip("acosh has bug")
+    @autotest()
     def test_acosh_flow_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case,
-                "acosh",
-                device=device,
-                n=2,
-                extra_generators={"input": acosh_input_tensor((3, 3))},
-            )
-
-    def test_acosh_tensor_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_tensor_against_pytorch(
-                test_case,
-                "acosh",
-                device=device,
-                n=2,
-                extra_generators={"input": acosh_input_tensor((3, 3))},
-            )
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = torch.acosh(x)
+        return y
 
 
 def _test_atan2_forward(test_case, shape, scalar, device):
@@ -951,18 +903,13 @@ class TestAtan2(flow.unittest.TestCase):
         for arg in GenArgList(arg_dict):
             _test_atan2_backward(test_case, *arg)
 
+    @autotest()
     def test_flow_atan2_with_random_data(test_case):
-        for device in ["cpu", "cuda"]:
-            test_flow_against_pytorch(
-                test_case,
-                "atan2",
-                extra_annotations={"other": flow.Tensor},
-                extra_generators={
-                    "input": random_tensor(ndim=1, dim1=1),
-                    "other": random_tensor(ndim=1, dim1=1),
-                },
-                device=device,
-            )
+        device = random_device()
+        x1 = random_pytorch_tensor(ndim=1, dim0=1).to(device)
+        x2 = random_pytorch_tensor(ndim=1, dim0=1).to(device)
+        y = torch.atan2(x1, x2)
+        return y
 
 
 if __name__ == "__main__":

oneflow/python/test/tensor/automated_test_util.py

@@ -16,8 +16,16 @@ limitations under the License.
 import os
 import sys
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(BASE_DIR)
+test_util_parent_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
 
+oneflow_test_utils_dir_from_env = os.getenv("ONEFLOW_TEST_UTILS_DIR")
+if oneflow_test_utils_dir_from_env:
+    from pathlib import Path
 
+    oneflow_test_utils_dir_from_env = Path(oneflow_test_utils_dir_from_env)
+    test_util_parent_dir = str(oneflow_test_utils_dir_from_env.parent.absolute())
+
+sys.path.append(test_util_parent_dir)
 from test_utils.automated_test_util import *

oneflow/python/test/tensor/test_tensor.py

@@ -21,6 +21,7 @@ import numpy as np
 
 import oneflow.experimental as flow
 import oneflow.typing as oft
+from automated_test_util import *
 
 
 @flow.unittest.skip_unless_1n1d()
@@ -90,7 +91,7 @@ class TestTensor(flow.unittest.TestCase):
         np_ones = np.ones(x.shape)
         np_zeros = np.zeros(x.shape)
 
-        random_fill_val = random.uniform(-100.0, 100.0)
+        random_fill_val = np.random.uniform(-100.0, 100.0)
         x.fill_(random_fill_val)
         test_case.assertTrue(np.allclose(x.numpy(), random_fill_val * np_ones))
 
@@ -114,7 +115,7 @@ class TestTensor(flow.unittest.TestCase):
         np_ones = np.ones(x.shape, dtype=np.int32)
         np_zeros = np.zeros(x.shape, dtype=np.int32)
 
-        random_fill_val = random.randint(-100, 100)
+        random_fill_val = np.random.randint(-100, 100)
         x.fill_(random_fill_val)
         test_case.assertTrue(np.allclose(x.numpy(), random_fill_val * np_ones))
 
@@ -389,33 +390,73 @@ class TestTensor(flow.unittest.TestCase):
         np_out = np.sum(input.numpy(), axis=(2, 1))
         test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
 
-    def test_asinh(test_case):
-        input = flow.Tensor(np.random.randn(4, 5, 6), dtype=flow.float32)
-        of_out = input.asinh()
-        np_out = np.arcsinh(input.numpy())
-        test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
-
-    def test_arcsinh(test_case):
-        input = flow.Tensor(np.random.randn(4, 5, 6), dtype=flow.float32)
-        of_out = input.arcsinh()
-        np_out = np.arcsinh(input.numpy())
-        test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
-
-    def test_asin(test_case):
-        input = flow.Tensor(np.random.random((4, 5, 6)) - 0.5, dtype=flow.float32)
-        of_out = input.asin()
-        np_out = np.arcsin(input.numpy())
-        test_case.assertTrue(
-            np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True)
-        )
-
-    def test_arcsin(test_case):
-        input = flow.Tensor(np.random.random((4, 5, 6)) - 0.5, dtype=flow.float32)
-        of_out = input.arcsin()
-        np_out = np.arcsin(input.numpy())
-        test_case.assertTrue(
-            np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True)
-        )
+    @autotest()
+    def test_tensor_tanh_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.tanh()
+        return y
+
+    @unittest.skip("asin has bug")
+    @autotest()
+    def test_flow_tensor_asin_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.asin()
+        return y
+
+    @unittest.skip("arcsin has bug")
+    @autotest()
+    def test_flow_tensor_arcsin_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.arcsin()
+        return y
+
+    @autotest()
+    def test_flow_tensor_asinh_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.asinh()
+        return y
+
+    @autotest()
+    def test_flow_tensor_arcsinh_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.arcsinh()
+        return y
+
+    @autotest()
+    def test_flow_tensor_sinh_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.sinh()
+        return y
+
+    @autotest()
+    def test_flow_tensor_atan2_with_random_data(test_case):
+        device = random_device()
+        x1 = random_pytorch_tensor(ndim=1, dim0=1).to(device)
+        x2 = random_pytorch_tensor(ndim=1, dim0=1).to(device)
+        y = x1.atan2(x2)
+        return y
+
+    @unittest.skip("arccosh has bug")
+    @autotest()
+    def test_arccosh_tensor_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.arccosh()
+        return y
+
+    @unittest.skip("acosh has bug")
+    @autotest()
+    def test_acosh_tensor_with_random_data(test_case):
+        device = random_device()
+        x = random_pytorch_tensor().to(device)
+        y = x.acosh()
+        return y
 
     def test_mean(test_case):
         input = flow.Tensor(np.random.randn(2, 3), dtype=flow.float32)

oneflow/python/test_utils/automated_test_util/torch_flow_dual_object.py

@@ -217,7 +217,7 @@ def equality_checker(torch_type, flow_type):
     return deco
 
 
-def check_equality(dual_object: DualObject):
+def check_equality(dual_object: DualObject, rtol=1e-4, atol=1e-5):
     checker = torch_type2checker.get(
         (type(dual_object.pytorch), type(dual_object.oneflow)), None
     )
@@ -229,12 +229,12 @@ def check_equality(dual_object: DualObject):
                 checker = value
                 break
     assert checker is not None
-    return checker(dual_object.pytorch, dual_object.oneflow)
+    return checker(dual_object.pytorch, dual_object.oneflow, rtol, atol)
 
 
 @equality_checker(torch_original.Tensor, flow.Tensor)
 @equality_checker(torch_original.Tensor, flow_stable._oneflow_internal.Tensor)
-def check_tensor_equality(torch_tensor, flow_tensor):
+def check_tensor_equality(torch_tensor, flow_tensor, rtol=1e-4, atol=1e-5):
     # TODO: check dtype
     if torch_tensor.grad is not None:
         assert (
@@ -244,7 +244,14 @@ def check_tensor_equality(torch_tensor, flow_tensor):
             torch_tensor.grad.detach().cpu().numpy(), flow_tensor.grad.numpy()
         ):
             return False
-    return np.allclose(torch_tensor.detach().cpu().numpy(), flow_tensor.numpy())
+
+    return np.allclose(
+        torch_tensor.detach().cpu().numpy(),
+        flow_tensor.numpy(),
+        rtol=rtol,
+        atol=atol,
+        equal_nan=True,
+    )
 
 
 def autotest(n=20, auto_backward=True, rtol=1e-4, atol=1e-5):
@@ -283,7 +290,7 @@ def autotest(n=20, auto_backward=True, rtol=1e-4, atol=1e-5):
                             )
                         )
                 for x in dual_objects_to_test:
-                    test_case.assertTrue(check_equality(x))
+                    test_case.assertTrue(check_equality(x, rtol=rtol, atol=atol))
                 if verbose:
                     print("test passed")
                 n -= 1