Dev pure cpu (#3398)

* cmake dont panic when build cuda

* naive changes

* fix cudaMemcpyKind

* fix acc actor

* fix actor

* fix gdb

* fix vm

* fix work type

* fix cuda type

* fix cuda type

* fix collective backend

* fix global scope

* amp

* rm PackKernelUtil gpu

* fix log

* fix rand

* fix sync size

* fix allocator

* fix vm

* fix kernel

* fix kernel

* fix kernels

* fix kernel

* fix softmax

* fix kernels

* fix reshape kernels

* add workaround

* try fix symbol not found

* fix vm

* fix vm

* fix jpeg

* fix broadcast gpu

* fix broadcast like

* fix transpose

* fix matmul

* fix CopyElemOnGpu

* fix sigmoid

* fix sigmoid and softmax

* fix relu

* fix sparse cross entropy

* fix kernels

* fix tanh

* fix same padding

* fix softmax

* fix undefined symbol: gzgets

* fix CopyField

* fix scalar add

* fix CopyNDGpuImpl

* copier

* fix slice boxing

* fix mem copier

* fix zero l...

oneflow/python/test/ops/test_TestDataTypeAttr.py

@@ -19,6 +19,8 @@ import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
 from test_util import GenArgList, type_name_to_flow_type, type_name_to_np_type
+import unittest
+import os
 
 
 def TestDataTypeAttr(input, output_type):
@@ -49,6 +51,7 @@ def RunTest(data_type):
     assert output.dtype == type_name_to_np_type[data_type]
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_data_type_attr(test_case):
     # TODO: fix bugs in ForeignOutputKernel with "float16" and "char" dtype, do not test these two dtypes here
     for data_type in ["float32", "double", "int8", "int32", "int64", "uint8"]:

oneflow/python/test/ops/test_TestDynamicSource.py

@@ -15,6 +15,8 @@ limitations under the License.
 """
 import numpy as np
 import oneflow as flow
+import unittest
+import os
 
 
 def my_test_source(name):
@@ -28,6 +30,7 @@ def my_test_source(name):
     )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_test_dynamic_source(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_data_type(flow.float)

oneflow/python/test/ops/test_TestMultiInputGrad.py

@@ -20,6 +20,7 @@ import numpy as np
 import oneflow as flow
 import test_global_storage
 from test_util import GenArgList
+import unittest
 
 
 def TestMultiInput(x1, x2):
@@ -35,6 +36,7 @@ def TestMultiInput(x1, x2):
     )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_TestMultiInput_grad_mirrored_inplace(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_data_type(flow.float)

oneflow/python/test/ops/test_TestMultiOutputOrder.py

@@ -16,8 +16,11 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def TestMultiOutputOrder(x, name):
     return (
         flow.user_op_builder(name)

oneflow/python/test/ops/test_TestReshape.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 
 def TestReshape(x, shape, name):
@@ -58,12 +60,14 @@ def mirrored_tensor_def_test(test_case, func_config):
     test_case.assertTrue(np.array_equal(x.reshape(5, 4), y))
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_fixed_TestReshape(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_logical_view(flow.scope.consistent_view())
     fixed_tensor_def_test(test_case, func_config)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_mirrored_TestReshape(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_logical_view(flow.scope.mirrored_view())

oneflow/python/test/ops/test_activations.py

@@ -99,5 +99,6 @@ def test_activations(test_case):
     for arg in GenArgList(arg_dict):
         compare_with_tensorflow(*arg)
 
-    for act_type in arg_dict["activation_type"]:
-        compare_with_tensorflow("gpu", act_type, (1024, 1024), flow.float16)
+    if os.getenv("ONEFLOW_TEST_CPU_ONLY") is None:
+        for act_type in arg_dict["activation_type"]:
+            compare_with_tensorflow("gpu", act_type, (1024, 1024), flow.float16)

oneflow/python/test/ops/test_all_reduce_group.py

@@ -18,6 +18,8 @@ from collections import OrderedDict
 import numpy as np
 import oneflow as flow
 from test_util import GenArgList
+import unittest
+import os
 
 
 def do_test(test_case, mirrored):
@@ -45,6 +47,7 @@ def do_test(test_case, mirrored):
     test_case.assertTrue(np.all(r2 == 0.5))
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_variable_as_loss_on_two_device(test_case):
     arg_dict = OrderedDict()
     arg_dict["mirrored"] = [True, False]

oneflow/python/test/ops/test_assign.py

@@ -19,6 +19,7 @@ import numpy as np
 import oneflow as flow
 from test_util import GenArgDict
 import oneflow.typing as oft
+import os
 
 flow_to_np_dtype_dict = {
     flow.int32: np.int32,
@@ -39,7 +40,8 @@ def _random_input(shape, dtype):
 
 def _of_assign_and_relu(value, dtype, device_type):
     flow.clear_default_session()
-    flow.config.gpu_device_num(1)
+    if os.getenv("ONEFLOW_TEST_CPU_ONLY") is None:
+        flow.config.gpu_device_num(1)
     flow.config.cpu_device_num(1)
     func_config = flow.FunctionConfig()
     func_config.default_data_type(dtype)

oneflow/python/test/ops/test_batch_normalization.py

@@ -22,8 +22,10 @@ import tensorflow as tf
 import test_global_storage
 from test_util import Args, GenArgDict, type_name_to_flow_type, type_name_to_np_type
 import oneflow.typing as oft
+import unittest
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_no_watch_scope_consistent(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_logical_view(flow.scope.consistent_view())
@@ -36,6 +38,7 @@ def test_no_watch_scope_consistent(test_case):
     Foo(np.ones((2, 8, 32, 32), dtype=np.float32))
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_train_consistent(test_case):
     flow.config.enable_debug_mode(True)
     func_config = flow.FunctionConfig()
@@ -359,6 +362,7 @@ def test_layer_batchnorm_trainable_without_training(test_case):
         CompareBnWithTensorFlow(**arg, training=False, trainable=True)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_nn_batchnorm(test_case):
     arg_dict = OrderedDict()
     arg_dict["input_shape"] = [(2, 4, 3, 5)]

oneflow/python/test/ops/test_broadcast_maximum.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 func_config = flow.FunctionConfig()
 func_config.default_data_type(flow.float)
@@ -38,12 +40,14 @@ def _run_test(test_case, a, b, dtype, device):
     _check(test_case, a, b, out.numpy())
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_broadcast_maximum_random_gpu(test_case):
     a = np.random.rand(1024, 1024).astype(np.float32)
     b = np.random.rand(1024, 1024).astype(np.float32)
     _run_test(test_case, a, b, flow.float32, "gpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_broadcast_maximum_broadcast_gpu(test_case):
     a = np.random.rand(1024, 1).astype(np.float32)
     b = np.random.rand(1, 1024).astype(np.float32)

oneflow/python/test/ops/test_broadcast_minimum.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 func_config = flow.FunctionConfig()
 func_config.default_data_type(flow.float)
@@ -38,12 +40,14 @@ def _run_test(test_case, a, b, dtype, device):
     _check(test_case, a, b, out.numpy())
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_broadcast_minimum_random_gpu(test_case):
     a = np.random.rand(1024, 1024).astype(np.float32)
     b = np.random.rand(1024, 1024).astype(np.float32)
     _run_test(test_case, a, b, flow.float32, "gpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_broadcast_minimum_broadcast_gpu(test_case):
     a = np.random.rand(1024, 1).astype(np.float32)
     b = np.random.rand(1, 1024).astype(np.float32)

oneflow/python/test/ops/test_categorical_ordinal_encoder.py

@@ -18,6 +18,8 @@ import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
 import typing
+import unittest
+import os
 
 
 def _test_categorical_ordinal_encoder(
@@ -74,6 +76,7 @@ def _test_categorical_ordinal_encoder(
     test_case.assertEqual(len(vk_set), unique_size)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_categorical_ordinal_encoder_gpu_large(test_case):
     _test_categorical_ordinal_encoder(
         test_case=test_case,
@@ -86,6 +89,7 @@ def test_categorical_ordinal_encoder_gpu_large(test_case):
     )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_categorical_ordinal_encoder_gpu_small(test_case):
     _test_categorical_ordinal_encoder(
         test_case=test_case,

oneflow/python/test/ops/test_ccrelu.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 
 def ccrelu(x, name):
@@ -54,18 +56,21 @@ def mirrored_tensor_def_test(test_case, func_config):
     test_case.assertTrue(np.array_equal(y, np.maximum(x, 0)))
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_ccrelu(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_logical_view(flow.scope.consistent_view())
     fixed_tensor_def_test(test_case, func_config)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_mirror_ccrelu(test_case):
     func_config = flow.FunctionConfig()
     func_config.default_logical_view(flow.scope.mirrored_view())
     mirrored_tensor_def_test(test_case, func_config)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_1n2c_mirror_dynamic_ccrelu(test_case):
     flow.config.gpu_device_num(2)
     func_config = flow.FunctionConfig()

oneflow/python/test/ops/test_concat.py

@@ -20,6 +20,8 @@ import tensorflow as tf
 import test_global_storage
 import random
 import math
+import unittest
+import os
 
 from test_util import GenArgList, type_name_to_flow_type
 from collections import OrderedDict
@@ -242,6 +244,7 @@ def _test_dynamic_concat(test_case, shape, axis, device_type, verbose=False):
     test_case.assertTrue(np.array_equal(of_output, output))
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_dynamic_concat_case_0(test_case):
     _test_dynamic_concat(test_case, (64, 4), 0, "gpu")
 
@@ -250,6 +253,7 @@ def test_dynamic_concat_case_1(test_case):
     _test_dynamic_concat(test_case, (2, 10), 1, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_dynamic_concat_case_2(test_case):
     _test_dynamic_concat(test_case, (4, 7, 128), 2, "gpu")
 

oneflow/python/test/ops/test_constant_like.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 
 def _check(test_case, x, y, value, dtype=None):
@@ -36,6 +38,7 @@ def _run_test(test_case, x, value, dtype=None, device="gpu"):
     _check(test_case, x, y.numpy(), value, dtype=dtype)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu_float(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 1.0, flow.float, "gpu")
@@ -46,6 +49,7 @@ def test_constant_like_cpu_float(test_case):
     _run_test(test_case, x, 2.0, flow.float, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu_double(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 3.0, flow.double, "gpu")
@@ -56,6 +60,7 @@ def test_constant_like_cpu_double(test_case):
     _run_test(test_case, x, 4.0, flow.double, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu_int8(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 5.0, flow.int8, "gpu")
@@ -66,6 +71,7 @@ def test_constant_like_cpu_int8(test_case):
     _run_test(test_case, x, 6.0, flow.int8, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu_int32(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 7.0, flow.int32, "gpu")
@@ -76,6 +82,7 @@ def test_constant_like_cpu_int32(test_case):
     _run_test(test_case, x, 8.0, flow.int32, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu_int64(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 9.0, flow.int64, "gpu")
@@ -86,6 +93,7 @@ def test_constant_like_cpu_int64(test_case):
     _run_test(test_case, x, 10.0, flow.int64, "cpu")
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_constant_like_gpu(test_case):
     x = np.random.rand(10, 3, 32, 1024).astype(np.float32)
     _run_test(test_case, x, 11.0, device="gpu")

oneflow/python/test/ops/test_copy_comm_net_pass_empty.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 
 def ccrelu(x, name):
@@ -30,6 +32,7 @@ def ccrelu(x, name):
     )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 @flow.unittest.num_nodes_required(2)
 def test_multi_node_comm_net(test_case):
     func_config = flow.FunctionConfig()
@@ -64,6 +67,7 @@ def test_multi_node_comm_net(test_case):
             )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 @flow.unittest.num_nodes_required(2)
 def test_multi_node_comm_net_dynamic(test_case):
     func_config = flow.FunctionConfig()

oneflow/python/test/ops/test_cpu_only_user_op.py

@@ -16,6 +16,8 @@ limitations under the License.
 import oneflow as flow
 import numpy as np
 import oneflow.typing as oft
+import unittest
+import os
 
 
 def _cpu_only_relu(x):
@@ -33,7 +35,7 @@ def _check_cpu_only_relu_device(test_case, verbose=False):
     flow.clear_default_session()
     func_config = flow.FunctionConfig()
     func_config.default_data_type(flow.float)
-    func_config.default_placement_scope(flow.scope.placement("gpu", "0:0"))
+    func_config.default_placement_scope(flow.scope.placement("cpu", "0:0"))
 
     @flow.global_function(function_config=func_config)
     def cpu_only_relu_job(x_def: oft.Numpy.Placeholder(shape=(2, 5), dtype=flow.float)):
@@ -68,5 +70,6 @@ def test_cpu_only_user_op(test_case):
     _check_cpu_only_relu_device(test_case)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_non_cpu_only_user_op(test_case):
     _check_non_cpu_only_relu_device(test_case)

oneflow/python/test/ops/test_distribute_concat.py

@@ -15,8 +15,11 @@ limitations under the License.
 """
 import numpy as np
 import oneflow as flow
+import unittest
+import os
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_deadlock(test_case):
     flow.config.gpu_device_num(2)
     func_config = flow.FunctionConfig()

oneflow/python/test/ops/test_gather_nd.py

@@ -20,6 +20,8 @@ import oneflow as flow
 import tensorflow as tf
 from test_util import GenArgList
 import oneflow.typing as oft
+import unittest
+import os
 
 gpus = tf.config.experimental.list_physical_devices("GPU")
 for gpu in gpus:
@@ -271,6 +273,7 @@ def test_gather_nd_case_4(test_case):
         _compare_gather_nd_with_tf(test_case, *arg)
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_dynamic_gather_nd(test_case):
     arg_dict = OrderedDict()
     arg_dict["params_shape"] = [(30, 15)]

oneflow/python/test/ops/test_indexed_slices_reduce_sum.py

@@ -16,6 +16,8 @@ limitations under the License.
 import numpy as np
 import oneflow as flow
 import oneflow.typing as oft
+import unittest
+import os
 
 func_config = flow.FunctionConfig()
 func_config.default_data_type(flow.float)
@@ -59,6 +61,7 @@ def _run_test(test_case, indices, values, indices_dtype, values_dtype, device):
     )
 
 
+@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
 def test_indexed_slices_reduce_sum_gpu(test_case):
     indices = np.random.randint(0, 32, 1024).astype(np.int32)
     values = np.random.rand(1024, 8).astype(np.float32)