diff --git a/cmake/functional.cmake b/cmake/functional.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..2590e408c8e4d5eddcbd29496cf3a9bb6f0129b8
--- /dev/null
+++ b/cmake/functional.cmake
@@ -0,0 +1,32 @@
+function(GENERATE_FUNCTIONAL_API_AND_PYBIND11_CPP SRCS HDRS PYBIND_SRCS ROOT_DIR)
+ set(YAML_FILE ${PROJECT_SOURCE_DIR}/oneflow/core/functional/functional_api.yaml)
+ set(GENERATED_API_DIR oneflow/core/functional)
+ set(GENERATED_PYBIND_DIR oneflow/api/python/functional)
+
+ add_custom_target(create_functional_api_dir
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${GENERATED_API_DIR}
+ VERBATIM)
+ add_custom_target(create_functional_pybind_dir
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${GENERATED_PYBIND_DIR}
+ VERBATIM)
+
+ list(APPEND SRCS ${PROJECT_BINARY_DIR}/${GENERATED_API_DIR}/functional_api.yaml.cpp)
+ list(APPEND HDRS ${PROJECT_BINARY_DIR}/${GENERATED_API_DIR}/functional_api.yaml.h)
+ list(APPEND PYBIND_SRCS ${PROJECT_BINARY_DIR}/${GENERATED_PYBIND_DIR}/functional_api.yaml.pybind.cpp)
+
+ add_custom_command(
+ OUTPUT "${PROJECT_BINARY_DIR}/${GENERATED_API_DIR}/functional_api.yaml.cpp"
+ "${PROJECT_BINARY_DIR}/${GENERATED_API_DIR}/functional_api.yaml.h"
+ "${PROJECT_BINARY_DIR}/${GENERATED_PYBIND_DIR}/functional_api.yaml.pybind.cpp"
+ COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/generate_functional_api.py
+ --yaml_file_path ${YAML_FILE} --generate_pybind
+ DEPENDS ${Python_EXECUTABLE} create_functional_api_dir create_functional_pybind_dir
+ ${PROJECT_SOURCE_DIR}/tools/generate_functional_api.py ${YAML_FILE}
+ VERBATIM)
+
+ set_source_files_properties(${${SRCS}} ${${HDRS}} ${${PYBIND_SRCS}} PROPERTIES GENERATED TRUE)
+ set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+ set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+ set(${PYBIND_SRCS} ${${PYBIND_SRCS}} PARENT_SCOPE)
+
+endfunction()
diff --git a/cmake/oneflow.cmake b/cmake/oneflow.cmake
index 512c51551e7468ddec42f1ca5e2f0277f69e1e48..9e2400e87b1dad99e13bf07353b60f5b704a1954 100644
--- a/cmake/oneflow.cmake
+++ b/cmake/oneflow.cmake
@@ -215,7 +215,7 @@ add_dependencies(of_protoobj make_pyproto_dir ${PROTOBUF_COPY_TARGETS})
# cfg obj lib
include(cfg)
-GENERATE_CFG_AND_PYBIND11_CPP(CFG_SRCS CFG_HRCS PYBIND11_SRCS ${PROJECT_SOURCE_DIR})
+GENERATE_CFG_AND_PYBIND11_CPP(CFG_SRCS CFG_HRCS CFG_PYBIND11_SRCS ${PROJECT_SOURCE_DIR})
oneflow_add_library(of_cfgobj ${CFG_SRCS} ${CFG_HRCS})
add_dependencies(of_cfgobj of_protoobj generate_cfg)
if (BUILD_SHARED_LIBS)
@@ -228,6 +228,13 @@ else()
target_link_libraries(of_cfgobj ${oneflow_third_party_libs})
endif()
+include(functional)
+GENERATE_FUNCTIONAL_API_AND_PYBIND11_CPP(
+ FUNCTIONAL_GENERATED_SRCS FUNCTIONAL_GENERATED_HRCS FUNCTIONAL_PYBIND11_SRCS ${PROJECT_SOURCE_DIR})
+list(APPEND of_all_obj_cc ${FUNCTIONAL_GENERATED_SRCS})
+
+set(PYBIND11_SRCS ${CFG_PYBIND11_SRCS} ${FUNCTIONAL_PYBIND11_SRCS})
+
include_directories(${PROJECT_SOURCE_DIR}) # TO FIND: third_party/eigen3/..
include_directories(${PROJECT_BINARY_DIR})
@@ -300,6 +307,8 @@ add_custom_target(of_pyscript_copy ALL
COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/oneflow/python" "${of_pyscript_dir}/oneflow/python"
COMMAND ${CMAKE_COMMAND} -E copy_directory "${of_proto_python_dir}/oneflow/core" "${of_pyscript_dir}/oneflow/core"
COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/core/__init__.py"
+ COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/F"
+ COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/F/__init__.py"
COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/python_gen"
COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/python_gen/__init__.py"
COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/generate_pip_version.py ${gen_pip_args} --src=${PROJECT_SOURCE_DIR}
diff --git a/dev-requirements.txt b/dev-requirements.txt
index c5d610989fbe5681244d4beac62b64755f976ffa..0dcb33fe1201511dd8d09e552f757388a226229b 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -7,3 +7,4 @@ requests
onnx; sys_platform != 'darwin'
jinja2
opencv-python==4.2.0.34; sys_platform != 'darwin'
+PyYAML
diff --git a/oneflow/api/python/framework/throw.h b/oneflow/api/python/framework/throw.h
new file mode 100644
index 0000000000000000000000000000000000000000..7d8a82aa3397b8c363a891d33fa8418ad38dbbbc
--- /dev/null
+++ b/oneflow/api/python/framework/throw.h
@@ -0,0 +1,72 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_PYTHON_FRAMEWORK_THROW_H_
+#define ONEFLOW_API_PYTHON_FRAMEWORK_THROW_H_
+
+#include "oneflow/core/common/error.h"
+
+namespace oneflow {
+
+class Throw final {
+ public:
+ Throw(const Error& error) : error_(error) {}
+ ~Throw() { ThrowError(error_.error_proto()); }
+
+ Error& error() { return error_; }
+
+ private:
+ Error error_;
+};
+
+} // namespace oneflow
+
+#define CHECK_OR_THROW(expr) \
+ if (!(expr)) \
+ Throw(oneflow::Error::CheckFailedError().AddStackFrame(MAYBE_FAILED_LOC, __FUNCTION__)).error() \
+ << " Check failed: " << OF_PP_STRINGIZE(expr) << "\t"
+
+#define CHECK_EQ_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) == (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_GE_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) >= (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_GT_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) > (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_LE_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) <= (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_LT_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) < (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_NE_OR_THROW(lhs, rhs) \
+ CHECK_OR_THROW((lhs) != (rhs)) << "(" << (lhs) << " vs " << (rhs) << ") "
+
+#define CHECK_STREQ_OR_THROW(lhs, rhs) CHECK_EQ_OR_THROW(std::string(lhs), std::string(rhs))
+
+#define CHECK_STRNE_OR_THROW(lhs, rhs) CHECK_NE_OR_THROW(std::string(lhs), std::string(rhs))
+
+#define CHECK_NOTNULL_OR_THROW(ptr) CHECK_OR_THROW(ptr != nullptr)
+
+#define CHECK_ISNULL_OR_THROW(ptr) CHECK_OR_THROW(ptr == nullptr)
+
+#define TODO_THEN_THROW() \
+ oneflow::Throw(oneflow::Error::Todo().AddStackFrame(MAYBE_FAILED_LOC, __FUNCTION__)).error()
+#define UNIMPLEMENTED_THEN_THROW() \
+ Throw(oneflow::Error::Unimplemented().AddStackFrame(MAYBE_FAILED_LOC, __FUNCTION__)).error()
+
+#endif // ONEFLOW_API_PYTHON_FRAMEWORK_THROW_H_
diff --git a/oneflow/api/python/functional/common.h b/oneflow/api/python/functional/common.h
new file mode 100644
index 0000000000000000000000000000000000000000..164986148b3aa24d94e758861cb3a13193e2aedb
--- /dev/null
+++ b/oneflow/api/python/functional/common.h
@@ -0,0 +1,139 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_PYTHON_FUNCTIONAL_COMMON_H_
+#define ONEFLOW_API_PYTHON_FUNCTIONAL_COMMON_H_
+
+#include <string>
+#include <vector>
+#include <pybind11/pybind11.h>
+
+#include "oneflow/core/common/maybe.h"
+#include "oneflow/core/common/preprocessor.h"
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/framework/attr_map.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+namespace detail {
+
+#define ARITHMETIC_TYPE_SEQ \
+ OF_PP_MAKE_TUPLE_SEQ(int32_t) \
+ OF_PP_MAKE_TUPLE_SEQ(uint32_t) \
+ OF_PP_MAKE_TUPLE_SEQ(int64_t) \
+ OF_PP_MAKE_TUPLE_SEQ(uint64_t) \
+ OF_PP_MAKE_TUPLE_SEQ(float) \
+ OF_PP_MAKE_TUPLE_SEQ(double) \
+ OF_PP_MAKE_TUPLE_SEQ(bool)
+
+#define MAKE_LIST_TUPLE_SEQ(T) (MAKE_LIST_TUPLE(T))
+#define MAKE_LIST_TUPLE(T) (std::vector<T>)
+#define ARITHMETIC_LIST_TYPE_SEQ OF_PP_FOR_EACH_TUPLE(MAKE_LIST_TUPLE_SEQ, ARITHMETIC_TYPE_SEQ)
+
+template<typename T>
+inline bool isinstance(py::handle obj) {
+ return py::isinstance<T>(obj);
+}
+
+#define SPECIALIZE_IS_INSTANCE(T) \
+ template<> \
+ inline bool isinstance<T>(py::handle obj) { \
+ static py::object dummy = py::cast(T()); \
+ CHECK_NOTNULL_OR_THROW(dummy.ptr()) << "Pybind has no internal type " << #T; \
+ return py::isinstance(obj, py::type::of(dummy)); \
+ }
+
+OF_PP_FOR_EACH_TUPLE(SPECIALIZE_IS_INSTANCE, ARITHMETIC_TYPE_SEQ OF_PP_MAKE_TUPLE_SEQ(std::string));
+OF_PP_FOR_EACH_TUPLE(SPECIALIZE_IS_INSTANCE,
+ ARITHMETIC_LIST_TYPE_SEQ OF_PP_MAKE_TUPLE_SEQ(std::vector<std::string>));
+
+#undef SPECIALIZE_IS_INSTANCE
+
+template<typename T>
+struct type_caster {
+ static Maybe<T> cast(py::handle src) { return py::cast<T>(src); }
+};
+
+template<typename T>
+inline Maybe<T> cast(py::handle obj) {
+ return type_caster<T>::cast(obj);
+}
+
+template<typename T>
+struct type_caster<std::vector<T>> {
+ static Maybe<std::vector<T>> cast(py::handle src);
+};
+
+#define SPECIALIZE_INTERNAL_IS_INSTANCE_ADN_CAST(T) \
+ template<> \
+ inline bool isinstance<std::shared_ptr<T>>(py::handle obj) { \
+ return detail::isinstance<T>(obj); \
+ } \
+ \
+ template<> \
+ struct type_caster<std::shared_ptr<T>> { \
+ static Maybe<std::shared_ptr<T>> cast(py::handle src) { \
+ CHECK_OR_RETURN(detail::isinstance<T>(src)) \
+ << "Can not cast to " << #T << " from python object whose type is " \
+ << *JUST(detail::cast<std::string>(py::str(py::type::of(src)))); \
+ return py::cast<std::shared_ptr<T>>(src); \
+ } \
+ };
+
+SPECIALIZE_INTERNAL_IS_INSTANCE_ADN_CAST(one::Tensor);
+SPECIALIZE_INTERNAL_IS_INSTANCE_ADN_CAST(one::TensorTuple);
+SPECIALIZE_INTERNAL_IS_INSTANCE_ADN_CAST(MutableCfgAttrMap);
+
+#undef SPECIALIZE_INTERNAL_IS_INSTANCE_ADN_CAST
+
+template<typename T>
+T&& dereference(T&& val) {
+ return std::move(val);
+}
+
+template<typename T>
+T&& dereference(std::shared_ptr<T>&& val) {
+ return std::move(*val);
+}
+
+template<typename T>
+/*static*/ Maybe<std::vector<T>> type_caster<std::vector<T>>::cast(py::handle src) {
+ PyObject* obj = src.ptr();
+ bool is_tuple = PyTuple_Check(obj);
+ CHECK_OR_RETURN(is_tuple || PyList_Check(obj))
+ << "The python object is not list or tuple, but is "
+ << *JUST(detail::cast<std::string>(py::str(py::type::of(src))));
+ size_t size = is_tuple ? PyTuple_GET_SIZE(obj) : PyList_GET_SIZE(obj);
+
+ auto values = std::make_shared<std::vector<T>>(size);
+ for (int i = 0; i < size; ++i) {
+ values->at(i) = detail::dereference<T>(JUST(detail::cast<T>(
+ is_tuple ? py::handle(PyTuple_GET_ITEM(obj, i)) : py::handle(PyList_GET_ITEM(obj, i)))));
+ }
+ return values;
+}
+
+} // namespace detail
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_API_PYTHON_FUNCTIONAL_COMMON_H_
diff --git a/oneflow/api/python/functional/function_def.h b/oneflow/api/python/functional/function_def.h
new file mode 100644
index 0000000000000000000000000000000000000000..69e9e648af6a9221fc4e3bb4245fda05b64a368f
--- /dev/null
+++ b/oneflow/api/python/functional/function_def.h
@@ -0,0 +1,67 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_PYTHON_FUNCTIONAL_FUNCTION_DEF_H_
+#define ONEFLOW_API_PYTHON_FUNCTIONAL_FUNCTION_DEF_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+#include <pybind11/pybind11.h>
+
+#include "oneflow/api/python/functional/python_arg.h"
+#include "oneflow/core/functional/value_types.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+struct ReturnDef {
+ ReturnDef() : type(kINVALID) {}
+ ReturnDef(const ValueType& t) : type(t) {}
+ ValueType type;
+};
+
+struct ArgumentDef {
+ ArgumentDef() : name(""), type(kINVALID), has_default_value(false) {}
+ ArgumentDef(const std::string& n, const ValueType& t)
+ : name(n), type(t), has_default_value(false) {}
+
+ template<typename T>
+ ArgumentDef(const std::string& n, const T& v)
+ : name(n), type(ValueTypeOf<T>()), has_default_value(true) {
+ default_value = std::make_shared<detail::AnyData<T>>(v);
+ }
+
+ std::string name;
+ ValueType type;
+
+ bool has_default_value;
+ std::shared_ptr<const detail::AnyDataBase> default_value;
+};
+
+struct FunctionDef {
+ std::string name;
+ ReturnDef return_def;
+ std::vector<ArgumentDef> argument_def;
+};
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_API_PYTHON_FUNCTIONAL_FUNCTION_DEF_H_
diff --git a/oneflow/api/python/functional/py_function.h b/oneflow/api/python/functional/py_function.h
new file mode 100644
index 0000000000000000000000000000000000000000..3d2b6b07c10014f82ad92873f3b273136c1396b5
--- /dev/null
+++ b/oneflow/api/python/functional/py_function.h
@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include <pybind11/pybind11.h>
+
+#include "oneflow/api/python/functional/python_arg.h"
+#include "oneflow/api/python/functional/unpack_call.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+template<typename SchemaT>
+inline py::object PyFunction(py::args args, py::kwargs kwargs) {
+ // TODO(): Support multiple function signatures.
+ CHECK_LE(args.size(), SchemaT::max_positionals)
+ << "The maximum count of positional arguments is " << SchemaT::max_positionals;
+ CHECK_LE(kwargs.size(), SchemaT::max_keywords)
+ << "The maximum count of keyword arguments is " << SchemaT::max_keywords;
+
+ // TODO(): Check argument types.
+ std::vector<PythonArg> _args(SchemaT::max_args);
+ for (int i = 0; i < args.size(); ++i) { _args[i] = PythonArg(args[i]); }
+ for (int i = args.size(); i < SchemaT::max_args; ++i) {
+ const auto& arg = SchemaT::argument_def.at(i);
+ if (kwargs.contains(arg.name.c_str())) {
+ _args[i] = PythonArg(kwargs[arg.name.c_str()]);
+ } else {
+ CHECK(arg.has_default_value)
+ << "Argument " << arg.name << " is required, and the function def is \""
+ << SchemaT::signature << "\".";
+ _args[i] = PythonArg(arg.default_value);
+ }
+ }
+ using FType = typename SchemaT::FType;
+ using R = typename SchemaT::R;
+ return py::cast(detail::unpack_call<FType, R, PythonArg>::apply(*SchemaT::func, _args));
+}
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
diff --git a/oneflow/api/python/functional/python_arg.cpp b/oneflow/api/python/functional/python_arg.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7c27719f96a966a39058e721c2c083d3e1602705
--- /dev/null
+++ b/oneflow/api/python/functional/python_arg.cpp
@@ -0,0 +1,139 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "oneflow/api/python/functional/python_arg.h"
+
+#include "oneflow/api/python/functional/common.h"
+#include "oneflow/core/common/data_type.cfg.h"
+#include "oneflow/core/framework/attr_map.h"
+#include "oneflow/core/framework/dtype.h"
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/framework/user_op_attr.cfg.h"
+#include "oneflow/core/functional/scalar.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+#define INSTANCE_CAST_OBJECT_AS(T) \
+ template<> \
+ Maybe<T> PythonArg::ObjectAs<T>() const { \
+ return detail::cast<T>(Borrow()); \
+ } \
+ template<> \
+ Maybe<std::vector<T>> PythonArg::ObjectAs<std::vector<T>>() const { \
+ return detail::cast<std::vector<T>>(Borrow()); \
+ }
+
+OF_PP_FOR_EACH_TUPLE(INSTANCE_CAST_OBJECT_AS,
+ ARITHMETIC_TYPE_SEQ OF_PP_MAKE_TUPLE_SEQ(std::string));
+
+#undef INSTANCE_CAST_OBJECT_AS
+
+template<>
+Maybe<Scalar> PythonArg::ObjectAs<Scalar>() const {
+ py::object obj = Borrow();
+ if (detail::isinstance<int32_t>(obj)) {
+ return Scalar(JUST(detail::cast<int32_t>(obj)));
+ } else if (detail::isinstance<int64_t>(obj)) {
+ return Scalar(JUST(detail::cast<int64_t>(obj)));
+ } else if (detail::isinstance<float>(obj)) {
+ return Scalar(JUST(detail::cast<float>(obj)));
+ } else if (detail::isinstance<double>(obj)) {
+ return Scalar(JUST(detail::cast<double>(obj)));
+ } else if (detail::isinstance<bool>(obj)) {
+ return Scalar(JUST(detail::cast<bool>(obj)));
+ } else {
+ UNIMPLEMENTED_THEN_RETURN() << "Can not convert to scalar from python object whose type is "
+ << *JUST(detail::cast<std::string>(py::str(py::type::of(obj))));
+ }
+}
+
+template<>
+Maybe<std::shared_ptr<one::Tensor>> PythonArg::ObjectAs<std::shared_ptr<one::Tensor>>() const {
+ return detail::cast<std::shared_ptr<one::Tensor>>(Borrow());
+}
+
+template<>
+Maybe<std::shared_ptr<one::TensorTuple>> PythonArg::ObjectAs<std::shared_ptr<one::TensorTuple>>()
+ const {
+ py::object obj = Borrow();
+ if (detail::isinstance<one::TensorTuple>(obj)) {
+ return detail::cast<std::shared_ptr<one::TensorTuple>>(obj);
+ }
+
+ const auto& v = JUST(detail::cast<std::vector<std::shared_ptr<one::Tensor>>>(obj));
+ auto values = std::make_shared<one::TensorTuple>(v->size());
+ for (int i = 0; i < v->size(); ++i) { values->at(i) = v->at(i); }
+ return values;
+}
+
+template<>
+Maybe<one::TensorTuple> PythonArg::ObjectAs<one::TensorTuple>() const {
+ return *JUST(ObjectAs<std::shared_ptr<one::TensorTuple>>());
+}
+
+template<>
+Maybe<std::shared_ptr<cfg::AttrValue>> PythonArg::ObjectAs<std::shared_ptr<cfg::AttrValue>>()
+ const {
+ py::object obj = Borrow();
+ if (detail::isinstance<cfg::AttrValue>(obj)) {
+ return detail::cast<std::shared_ptr<cfg::AttrValue>>(obj);
+ }
+ auto attr_value = std::make_shared<cfg::AttrValue>();
+ if (detail::isinstance<int32_t>(obj)) {
+ attr_value->set_at_int32(JUST(detail::cast<int32_t>(obj)));
+ } else if (detail::isinstance<double>(obj)) {
+ attr_value->set_at_double(JUST(detail::cast<double>(obj)));
+ } else {
+ UNIMPLEMENTED_THEN_RETURN() << "The attribute type was not supported which is "
+ << *JUST(detail::cast<std::string>(py::str(py::type::of(obj))));
+ }
+ return attr_value;
+}
+
+template<>
+Maybe<AttrMap> PythonArg::ObjectAs<AttrMap>() const {
+ const auto& attrs = *(JUST(detail::cast<std::shared_ptr<MutableCfgAttrMap>>(Borrow())));
+ return std::make_shared<AttrMap>(*attrs);
+ ;
+}
+
+template<>
+Maybe<DataType> PythonArg::ObjectAs<DataType>() const {
+ py::object obj = Borrow();
+ if (detail::isinstance<cfg::DataType>(obj)) {
+ const auto& dtype = *JUST(detail::cast<std::shared_ptr<cfg::DataType>>(obj));
+ return static_cast<DataType>(*dtype);
+ } else if (detail::isinstance<DType>(obj)) {
+ return JUST(detail::cast<DType&>(obj)).data_type();
+ } else if (detail::isinstance<int32_t>(obj)) {
+ return static_cast<DataType>(JUST(detail::cast<int32_t>(obj)));
+ } else if (detail::isinstance<int64_t>(obj)) {
+ return static_cast<DataType>(JUST(detail::cast<int64_t>(obj)));
+ } else {
+ UNIMPLEMENTED_THEN_RETURN() << "Can not convert object to DataType from "
+ << *JUST(detail::cast<std::string>(py::str(py::type::of(obj))));
+ }
+ return kInvalidDataType;
+}
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
diff --git a/oneflow/api/python/functional/python_arg.h b/oneflow/api/python/functional/python_arg.h
new file mode 100644
index 0000000000000000000000000000000000000000..519f07e07a7eb20150bf863af83cc856f834d2fc
--- /dev/null
+++ b/oneflow/api/python/functional/python_arg.h
@@ -0,0 +1,91 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_ARG_H_
+#define ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_ARG_H_
+
+#include <pybind11/pybind11.h>
+
+#include "oneflow/api/python/framework/throw.h"
+#include "oneflow/core/common/maybe.h"
+#include "oneflow/core/functional/value_types.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+namespace detail {
+
+struct AnyDataBase {
+ virtual ValueType value_type() const = 0;
+ virtual const void* Ptr() const = 0;
+};
+
+template<typename T>
+struct AnyData : public AnyDataBase {
+ T content;
+ explicit AnyData(const T& v) : content(v) {}
+
+ ValueType value_type() const override { return ValueTypeOf<T>(); }
+ const void* Ptr() const override { return &content; }
+};
+
+} // namespace detail
+
+class PythonArg {
+ public:
+ PythonArg() = default;
+ PythonArg(py::object object) : object_(object.ptr()), active_tag_(HAS_OBJECT) {}
+
+ PythonArg(const std::shared_ptr<const detail::AnyDataBase>& value)
+ : immediate_(value), active_tag_(HAS_IMMEDIATE) {}
+
+ template<typename T, typename std::enable_if<!py::detail::is_pyobject<T>::value, int>::type = 0>
+ PythonArg(const T& value)
+ : immediate_(std::make_shared<detail::AnyData<T>>(value)), active_tag_(HAS_IMMEDIATE) {}
+
+ virtual ~PythonArg() = default;
+
+ template<typename T>
+ operator T() const {
+ if (active_tag_ == HAS_IMMEDIATE) {
+ CHECK_EQ_OR_THROW(ValueTypeOf<T>(), immediate_->value_type())
+ << "Could not convert immediate value from type " << immediate_->value_type()
+ << " to type " << ValueTypeOf<T>();
+ return *reinterpret_cast<const T*>(immediate_->Ptr());
+ }
+ CHECK_EQ_OR_THROW(active_tag_, HAS_OBJECT);
+ return this->ObjectAs<oneflow::detail::remove_cvref_t<T>>().GetOrThrow();
+ }
+
+ private:
+ template<typename T>
+ Maybe<T> ObjectAs() const;
+ py::object Borrow() const { return py::reinterpret_borrow<py::object>(object_); }
+
+ PyObject* object_;
+ std::shared_ptr<const detail::AnyDataBase> immediate_;
+
+ enum { HAS_OBJECT, HAS_IMMEDIATE, HAS_NONE } active_tag_;
+};
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_ARG_H_
diff --git a/oneflow/api/python/functional/unpack_call.h b/oneflow/api/python/functional/unpack_call.h
new file mode 100644
index 0000000000000000000000000000000000000000..f564d5b7cc48314c9eb64670b338f5dc389d500c
--- /dev/null
+++ b/oneflow/api/python/functional/unpack_call.h
@@ -0,0 +1,83 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_PYTHON_FUNCTIONAL_UNPACK_CALL_H_
+#define ONEFLOW_API_PYTHON_FUNCTIONAL_UNPACK_CALL_H_
+
+#include "oneflow/api/python/functional/python_arg.h"
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/common/function_traits.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+namespace detail {
+
+template<typename F, typename R, typename T, int nleft, int index>
+struct unpack_call_dispatcher {
+ template<typename... Args>
+ static R apply(const F& f, const std::vector<T>& args, Args&&... unpacked_args) {
+ return unpack_call_dispatcher<F, R, T, nleft - 1, index + 1>::apply(
+ f, args, std::forward<Args>(unpacked_args)..., args[index]);
+ }
+};
+
+template<typename F, typename R, typename T, int index>
+struct unpack_call_dispatcher<F, R, T, 0, index> {
+ template<typename... Args>
+ static R apply(const F& f, const std::vector<T>& args, Args&&... unpacked_args) {
+ return f(std::forward<Args>(unpacked_args)...);
+ }
+};
+
+template<typename F, typename R, typename T>
+struct unpack_call {
+ static R apply(const F& f, const std::vector<T>& args) {
+ constexpr size_t nargs = function_traits<F>::nargs;
+ CHECK_EQ(nargs, args.size()) << "Requires " << nargs << " arguments, but " << args.size()
+ << " is given.";
+ return unpack_call_dispatcher<F, R, T, nargs, 0>::apply(f, args);
+ }
+};
+
+#define INSTANCE_MAYBE_UNPACK_CALL(K, return_fn) \
+ template<typename F, typename T> \
+ struct unpack_call<F, K, T> { \
+ static constexpr auto return_fn_ = (return_fn); \
+ using R = typename function_traits<decltype(return_fn_)>::return_type; \
+ static R apply(const F& f, const std::vector<T>& args) { \
+ constexpr size_t nargs = function_traits<F>::nargs; \
+ CHECK_EQ(nargs, args.size()) \
+ << "Requires " << nargs << " arguments, but " << args.size() << " is given."; \
+ return (return_fn)(unpack_call_dispatcher<F, K, T, nargs, 0>::apply(f, args)); \
+ } \
+ };
+
+INSTANCE_MAYBE_UNPACK_CALL(Maybe<one::Tensor>,
+ ([](const Maybe<one::Tensor>& t) { return t.GetPtrOrThrow(); }));
+INSTANCE_MAYBE_UNPACK_CALL(Maybe<one::TensorTuple>,
+ ([](const Maybe<one::TensorTuple>& t) { return t.GetPtrOrThrow(); }));
+
+#undef INSTANCE_MAYBE_UNPACK_CALL
+
+} // namespace detail
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_API_PYTHON_FUNCTIONAL_UNPACK_CALL_H_
diff --git a/oneflow/core/autograd/gradient_funcs/scalar_add.cpp b/oneflow/core/autograd/gradient_funcs/scalar_add.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8db59221a96034645ed16750207348c1af11eeb
--- /dev/null
+++ b/oneflow/core/autograd/gradient_funcs/scalar_add.cpp
@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "oneflow/core/framework/op_expr_grad_function.h"
+
+namespace oneflow {
+namespace one {
+
+struct ScalarAddInterpState : public OpExprInterpState {
+ bool requires_grad;
+};
+
+class ScalarAdd : public OpExprGradFunction<ScalarAddInterpState> {
+ public:
+ Maybe<void> Init(const OpExpr& op) override { return Maybe<void>::Ok(); }
+
+ Maybe<void> Capture(ScalarAddInterpState* ctx, const TensorTuple& inputs,
+ const TensorTuple& outputs, const AttrMap& attrs) const override {
+ CHECK_EQ_OR_RETURN(inputs.size(), 1);
+ ctx->requires_grad = inputs.at(0)->requires_grad();
+ return Maybe<void>::Ok();
+ }
+
+ Maybe<void> Apply(const ScalarAddInterpState* ctx, const TensorTuple& out_grads,
+ TensorTuple* in_grads) const override {
+ CHECK_EQ_OR_RETURN(out_grads.size(), 1);
+ in_grads->resize(1);
+ if (ctx->requires_grad) { in_grads->at(0) = out_grads.at(0); }
+ return Maybe<void>::Ok();
+ }
+};
+
+REGISTER_OP_EXPR_GRAD_FUNCTION("scalar_add", ScalarAdd);
+
+} // namespace one
+} // namespace oneflow
diff --git a/oneflow/core/common/error.h b/oneflow/core/common/error.h
index 9eaa0641433377f5d6320388ec630513e4d2ecd3..93e4470c4e8450a59442c637e8f9c38d2acb6f0e 100644
--- a/oneflow/core/common/error.h
+++ b/oneflow/core/common/error.h
@@ -90,9 +90,8 @@ class Error final {
void ThrowError(const std::shared_ptr<cfg::ErrorProto>& error);
const std::shared_ptr<cfg::ErrorProto>& ThreadLocalError();
-// r-value reference is used to supporting expressions like `Error() << "invalid value"`
template<typename T>
-Error&& operator<<(Error&& error, const T& x) {
+Error& operator<<(Error& error, const T& x) {
std::ostringstream ss;
ss << x;
if (error->stack_frame().empty()) {
@@ -101,6 +100,13 @@ Error&& operator<<(Error&& error, const T& x) {
auto* stack_frame_top = error->mutable_stack_frame(error->stack_frame_size() - 1);
stack_frame_top->set_error_msg(stack_frame_top->error_msg() + ss.str());
}
+ return error;
+}
+
+// r-value reference is used to supporting expressions like `Error() << "invalid value"`
+template<typename T>
+Error&& operator<<(Error&& error, const T& x) {
+ error << x;
return std::move(error);
}
diff --git a/oneflow/core/common/function_traits.h b/oneflow/core/common/function_traits.h
index fd1d72651e563154f03f74bac858986608945ac2..4b8636601cffdec2dec5e32029e22023f1e07f40 100644
--- a/oneflow/core/common/function_traits.h
+++ b/oneflow/core/common/function_traits.h
@@ -24,22 +24,40 @@ using void_t = void;
template<typename T, typename = void>
struct function_traits;
+template<typename Ret, typename... Args>
+struct function_traits<Ret(Args...)> {
+ using func_type = Ret(Args...);
+ using return_type = Ret;
+ using args_type = std::tuple<Args...>;
+
+ static constexpr size_t nargs = sizeof...(Args);
+};
+
template<typename Ret, typename... Args>
struct function_traits<Ret (*)(Args...)> {
+ using func_type = Ret(Args...);
using return_type = Ret;
using args_type = std::tuple<Args...>;
+
+ static constexpr size_t nargs = sizeof...(Args);
};
template<typename Ret, typename C, typename... Args>
struct function_traits<Ret (C::*)(Args...)> {
+ using func_type = Ret(Args...);
using return_type = Ret;
using args_type = std::tuple<Args...>;
+
+ static constexpr size_t nargs = sizeof...(Args);
};
template<typename Ret, typename C, typename... Args>
struct function_traits<Ret (C::*)(Args...) const> {
+ using func_type = Ret(Args...);
using return_type = Ret;
using args_type = std::tuple<Args...>;
+
+ static constexpr size_t nargs = sizeof...(Args);
};
template<typename F>
diff --git a/oneflow/core/common/type_traits.h b/oneflow/core/common/type_traits.h
index 3d53189aae56aebc3f46c0d5f684d61a1ba1130e..e8d23682026e22c5aae88f297dd7e79ae96d4bc5 100644
--- a/oneflow/core/common/type_traits.h
+++ b/oneflow/core/common/type_traits.h
@@ -105,6 +105,10 @@ struct IsScalarType final {
};
namespace detail {
+
+template<typename T>
+using remove_cvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
template<typename T, typename Enabled = void>
struct ScalarOrConstRef;
diff --git a/oneflow/core/framework/tensor_tuple.h b/oneflow/core/framework/tensor_tuple.h
index acda3f13b81daaad124bd411ea1476ce7a5e1755..b996aa5b080e899ed1cef1aa617960bc4665f8f5 100644
--- a/oneflow/core/framework/tensor_tuple.h
+++ b/oneflow/core/framework/tensor_tuple.h
@@ -28,8 +28,8 @@ class Tensor;
class TensorTuple final : public std::vector<std::shared_ptr<Tensor>>,
public std::enable_shared_from_this<TensorTuple> {
public:
- TensorTuple(const TensorTuple&) = delete;
- TensorTuple(TensorTuple&) = delete;
+ // TensorTuple(const TensorTuple&) = delete;
+ // TensorTuple(TensorTuple&) = delete;
TensorTuple() = default;
TensorTuple(std::vector<std::shared_ptr<Tensor>>::size_type size);
TensorTuple(std::initializer_list<std::shared_ptr<Tensor>> init_list);
diff --git a/oneflow/core/functional/function_library.h b/oneflow/core/functional/function_library.h
new file mode 100644
index 0000000000000000000000000000000000000000..e00168bef5dff9d95aeca09a95a3aeb7fdf59084
--- /dev/null
+++ b/oneflow/core/functional/function_library.h
@@ -0,0 +1,77 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_CORE_FUNCTIONAL_FUNCTION_LIBRARY_H_
+#define ONEFLOW_CORE_FUNCTIONAL_FUNCTION_LIBRARY_H_
+
+#include "oneflow/core/common/util.h"
+#include "oneflow/core/functional/packed_functor.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+class FunctionLibrary {
+ public:
+ virtual ~FunctionLibrary() = default;
+
+ template<typename Func>
+ void add_functor(const std::string& func_name) {
+ Func func;
+ add_functor(func_name, std::move(func));
+ }
+
+ template<typename Func>
+ void add_functor(const std::string& func_name, const Func& func) {
+ auto packed_func = PackedFunctor::Make(func_name, func);
+ functors_.emplace(func_name, packed_func);
+ }
+
+ Maybe<PackedFunctor> find(const std::string& func_name) {
+ const auto& it = functors_.find(func_name);
+ CHECK_OR_RETURN(it != functors_.end())
+ << "Functor was not found for op " << func_name
+ << ", please check whether the functor has been registered correctly or not.";
+ return std::make_shared<PackedFunctor>(it->second);
+ }
+
+ static FunctionLibrary* Global() {
+ static FunctionLibrary global_function_library;
+ return &global_function_library;
+ }
+
+ private:
+ FunctionLibrary() = default;
+
+ // The reason for not using `std::shared_ptr<PackedFunctor>` is that
+ // the functor maybe stateful.
+ HashMap<std::string, PackedFunctor> functors_;
+};
+
+#define ONEFLOW_FUNCTION_LIBRARY(m) ONEFLOW_FUNCTION_LIBRARY_IMPL(m, __COUNTER__)
+#define ONEFLOW_FUNCTION_LIBRARY_IMPL(m, uuid) \
+ static void OF_PP_CAT(_oneflow_function_library_, uuid)(FunctionLibrary & m); \
+ static int OF_PP_CAT(_oneflow_function_library_dummy_, uuid) = []() { \
+ FunctionLibrary* library = FunctionLibrary::Global(); \
+ OF_PP_CAT(_oneflow_function_library_, uuid)(*library); \
+ return 0; \
+ }(); \
+ void OF_PP_CAT(_oneflow_function_library_, uuid)(FunctionLibrary & m)
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_FUNCTION_LIBRARY_H_
diff --git a/oneflow/core/functional/function_signature.h b/oneflow/core/functional/function_signature.h
new file mode 100644
index 0000000000000000000000000000000000000000..1fdd6b82daa9141a04cbdd87d9eb4382382664ba
--- /dev/null
+++ b/oneflow/core/functional/function_signature.h
@@ -0,0 +1,112 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef ONEFLOW_CORE_FUNCTIONAL_FUNCTION_SIGNATURE_H_
+#define ONEFLOW_CORE_FUNCTIONAL_FUNCTION_SIGNATURE_H_
+
+#include <memory>
+
+#include "oneflow/core/functional/value_types.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+struct FunctionSignature {
+ ValueType return_type;
+ std::vector<ValueType> argument_types;
+};
+
+namespace detail {
+
+template<typename T>
+inline ValueType PackType() {
+ return ValueTypeOf<typename std::decay<T>::type>();
+}
+
+template<typename... Args>
+struct PackTypeListImpl;
+
+template<>
+struct PackTypeListImpl<> {
+ static void pack(std::vector<ValueType>* packed_types) {}
+};
+
+template<typename T, typename... Args>
+struct PackTypeListImpl<T, Args...> {
+ static void pack(std::vector<ValueType>* packed_types) {
+ packed_types->emplace_back(PackType<T>());
+ PackTypeListImpl<Args...>::pack(packed_types);
+ }
+};
+
+template<typename... Args>
+inline std::vector<ValueType> PackTypeList() {
+ std::vector<ValueType> packed_types;
+ detail::PackTypeListImpl<Args...>::pack(&packed_types);
+ return packed_types;
+}
+
+template<typename R, typename... Args>
+inline FunctionSignature PackFunctionSignatureImpl() {
+ FunctionSignature signature;
+ signature.return_type = PackType<R>();
+ signature.argument_types = detail::PackTypeList<Args...>();
+ return signature;
+}
+
+template<typename T>
+struct PackFunctionSignature;
+
+template<typename R, typename... Args>
+struct PackFunctionSignature<R(Args...)> {
+ static FunctionSignature pack() {
+ static auto signature = PackFunctionSignatureImpl<R, Args...>();
+ return signature;
+ }
+};
+
+template<typename T>
+class CheckSignature;
+
+template<typename R, typename... Args>
+class CheckSignature<R(Args...)> {
+ public:
+ CheckSignature(const FunctionSignature& signature) { status_ = CheckSignatureImpl(signature); }
+
+ bool Ok() { return status_; }
+
+ private:
+ bool CheckSignatureImpl(const FunctionSignature& signature);
+ bool status_;
+};
+
+template<typename R, typename... Args>
+bool CheckSignature<R(Args...)>::CheckSignatureImpl(const FunctionSignature& signature) {
+ static ValueType return_type = detail::PackType<R>();
+ if (signature.return_type != return_type) { return false; }
+ static std::vector<ValueType> argument_types = detail::PackTypeList<Args...>();
+ if (argument_types != signature.argument_types) { return false; }
+ return true;
+}
+
+} // namespace detail
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_FUNCTION_SIGNATURE_H_
diff --git a/oneflow/core/functional/functional.h b/oneflow/core/functional/functional.h
new file mode 100644
index 0000000000000000000000000000000000000000..6b330a2fda07762a81e0d305ff49cd8697f79e5b
--- /dev/null
+++ b/oneflow/core/functional/functional.h
@@ -0,0 +1,21 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_CORE_FUNCTIONAL_FUNCTIONAL_H_
+#define ONEFLOW_CORE_FUNCTIONAL_FUNCTIONAL_H_
+
+#include "oneflow/core/functional/functional_api.yaml.h"
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_FUNCTIONAL_H_
diff --git a/oneflow/core/functional/functional_api.yaml b/oneflow/core/functional/functional_api.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3e2dca8c370bebbca870ebb30339c39ec39f5f62
--- /dev/null
+++ b/oneflow/core/functional/functional_api.yaml
@@ -0,0 +1,37 @@
+# Copyright 2020 The OneFlow Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The following data types are allowed:
+# {
+# "Tensor", "TensorTuple", "Scalar", "Int", "Int32", "Int64", "Float", "Double", "String", "Bool",
+# "ScalarList", "IntList", "Int32List", "Int64List", "FloatList", "DoubleList", "StringList", "BoolList"
+# }
+
+- name: "add"
+ signature: "Tensor Add(Tensor x, Tensor y)"
+ bind_python: True
+
+- name: "add_n"
+ signature: "Tensor AddN(TensorTuple inputs)"
+ bind_python: True
+
+- name: "add_scalar"
+ signature: "Tensor AddScalar(Tensor x, *, Scalar alpha)"
+ bind_python: True
+
+- name: "normalization"
+ signature: "Tensor Normalization(Tensor x, Tensor moving_mean, Tensor moving_variance,
+ Tensor gamma, Tensor beta, *, Int32 axis=1, Float epsilon=1e-5,
+ Float momentum=0.9, Bool is_training=False)"
+ bind_python: True
diff --git a/oneflow/core/functional/impl/add_functor.cpp b/oneflow/core/functional/impl/add_functor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..522d25dd85d67ba3200cc0ba2f8df03347d660ed
--- /dev/null
+++ b/oneflow/core/functional/impl/add_functor.cpp
@@ -0,0 +1,101 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "oneflow/core/framework/attr_map.h"
+#include "oneflow/core/framework/op_builder.h"
+#include "oneflow/core/framework/op_expr.h"
+#include "oneflow/core/framework/op_interpreter/op_interpreter_util.h"
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/functional/function_library.h"
+#include "oneflow/core/functional/scalar.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+namespace impl {
+
+class AddFunctor {
+ public:
+ AddFunctor() {
+ add_op_ = CHECK_JUST(one::OpBuilder("add_n").Input("in", 2).Output("out").Build());
+ }
+ Maybe<Tensor> operator()(const std::shared_ptr<one::Tensor>& x,
+ const std::shared_ptr<one::Tensor>& y) const {
+ return OpInterpUtil::Dispatch<Tensor>(*add_op_, {x, y});
+ }
+
+ private:
+ std::shared_ptr<OpExpr> add_op_;
+};
+
+class AddNFunctor {
+ public:
+ AddNFunctor() {
+ add_n_op_.resize(128 /*the maximum number of inputs*/);
+ for (int n = 2; n < add_n_op_.size(); ++n) {
+ add_n_op_[n] = CHECK_JUST(one::OpBuilder("add_n").Input("in", n).Output("out").Build());
+ }
+ }
+ Maybe<Tensor> operator()(const TensorTuple& inputs) const {
+ CHECK_GE_OR_RETURN(inputs.size(), 2);
+ CHECK_LT_OR_RETURN(inputs.size(), add_n_op_.size())
+ << "The maximum number supported of inputs is " << add_n_op_.size();
+ return OpInterpUtil::Dispatch<Tensor>(*add_n_op_.at(inputs.size()), inputs);
+ }
+
+ private:
+ std::vector<std::shared_ptr<OpExpr>> add_n_op_;
+};
+
+class AddScalarFunctor {
+ public:
+ AddScalarFunctor() {
+ add_scalar_op_ = CHECK_JUST(one::OpBuilder("scalar_add").Input("in").Output("out").Build());
+ }
+ Maybe<Tensor> operator()(const std::shared_ptr<one::Tensor>& x, const Scalar& scalar) const {
+ MutableAttrMap attrs;
+ if (scalar.IsFloatingPoint()) {
+ JUST(attrs.SetAttr<double>("float_operand", JUST(scalar.As<double>())));
+ JUST(attrs.SetAttr<bool>("has_float_operand", true));
+ JUST(attrs.SetAttr<bool>("has_int_operand", false));
+ return OpInterpUtil::Dispatch<Tensor>(*add_scalar_op_, {x}, attrs);
+ } else if (scalar.IsIntegral()) {
+ JUST(attrs.SetAttr<int64_t>("int_operand", JUST(scalar.As<int64_t>())));
+ JUST(attrs.SetAttr<bool>("has_float_operand", false));
+ JUST(attrs.SetAttr<bool>("has_int_operand", true));
+ return OpInterpUtil::Dispatch<Tensor>(*add_scalar_op_, {x}, attrs);
+ } else {
+ UNIMPLEMENTED_THEN_RETURN();
+ }
+ }
+
+ private:
+ std::shared_ptr<OpExpr> add_scalar_op_;
+};
+
+} // namespace impl
+
+ONEFLOW_FUNCTION_LIBRARY(m) {
+ m.add_functor<impl::AddFunctor>("Add");
+ m.add_functor<impl::AddNFunctor>("AddN");
+ m.add_functor<impl::AddScalarFunctor>("AddScalar");
+};
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
diff --git a/oneflow/core/functional/impl/normalization_functor.cpp b/oneflow/core/functional/impl/normalization_functor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ec3c302a9b781d881f999abc27cb833669a303a1
--- /dev/null
+++ b/oneflow/core/functional/impl/normalization_functor.cpp
@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "oneflow/core/framework/attr_map.h"
+#include "oneflow/core/framework/op_builder.h"
+#include "oneflow/core/framework/op_expr.h"
+#include "oneflow/core/framework/op_interpreter/op_interpreter_util.h"
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/functional/function_library.h"
+#include "oneflow/core/functional/scalar.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+namespace impl {
+
+class NormalizationFunctor {
+ public:
+ NormalizationFunctor() {
+ norm_eval_op_ = CHECK_JUST(one::OpBuilder("normalization")
+ .Input("x")
+ .Input("moving_mean")
+ .Input("moving_variance")
+ .Input("gamma")
+ .Input("beta")
+ .Output("y")
+ .Attr("training", false)
+ .Build());
+ norm_training_op_ = CHECK_JUST(one::OpBuilder("normalization")
+ .Input("x")
+ .Input("moving_mean")
+ .Input("moving_variance")
+ .Input("gamma")
+ .Input("beta")
+ .Output("y")
+ .Output("mean")
+ .Output("inv_variance")
+ .Attr("training", true)
+ .Build());
+ }
+ Maybe<Tensor> operator()(const std::shared_ptr<one::Tensor>& x,
+ const std::shared_ptr<one::Tensor>& moving_mean,
+ const std::shared_ptr<one::Tensor>& moving_variance,
+ const std::shared_ptr<one::Tensor>& gamma,
+ const std::shared_ptr<one::Tensor>& beta, const int32_t& axis,
+ const float& epsilon, const float& momentum,
+ const bool& is_training) const {
+ MutableAttrMap attrs;
+ JUST(attrs.SetAttr<int32_t>("axis", axis));
+ JUST(attrs.SetAttr<float>("epsilon", epsilon));
+ JUST(attrs.SetAttr<float>("momentum", momentum));
+ std::shared_ptr<OpExpr> op;
+ if (is_training) {
+ op = norm_training_op_;
+ } else {
+ op = norm_eval_op_;
+ }
+ return OpInterpUtil::Dispatch<one::Tensor>(*op, {x, moving_mean, moving_variance, gamma, beta},
+ attrs);
+ }
+
+ private:
+ std::shared_ptr<OpExpr> norm_eval_op_;
+ std::shared_ptr<OpExpr> norm_training_op_;
+};
+
+} // namespace impl
+
+ONEFLOW_FUNCTION_LIBRARY(m) { m.add_functor<impl::NormalizationFunctor>("Normalization"); };
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
diff --git a/oneflow/core/functional/packed_functor.h b/oneflow/core/functional/packed_functor.h
new file mode 100644
index 0000000000000000000000000000000000000000..206153d3e431786ebfc1592f23b35b2689ae9b33
--- /dev/null
+++ b/oneflow/core/functional/packed_functor.h
@@ -0,0 +1,113 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef ONEFLOW_CORE_FUNCTIONAL_FUNCTOR_H_
+#define ONEFLOW_CORE_FUNCTIONAL_FUNCTOR_H_
+
+#include <memory>
+
+#include "oneflow/core/common/function_traits.h"
+#include "oneflow/core/common/type_traits.h"
+#include "oneflow/core/functional/value_types.h"
+#include "oneflow/core/functional/function_signature.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+template<typename T>
+using remove_cvref_t = oneflow::detail::remove_cvref_t<T>;
+
+struct FunctionBody {
+ virtual operator void*() = 0;
+};
+
+template<typename T>
+class FunctionBodyImpl;
+
+template<typename R, typename... Args>
+class FunctionBodyImpl<R(Args...)> : public FunctionBody {
+ public:
+ template<typename Func,
+ typename std::enable_if<
+ std::is_same<typename function_traits<Func>::func_type, R(Args...)>::value,
+ int>::type = 0>
+ FunctionBodyImpl(const Func& func) {
+ func_ = [func](const remove_cvref_t<Args>&... args) {
+ return func(std::forward<const remove_cvref_t<Args>&>(args)...);
+ };
+ }
+
+ operator void*() override { return &func_; }
+
+ private:
+ std::function<R(const remove_cvref_t<Args>&...)> func_;
+};
+
+class Functor {
+ public:
+ Functor(const std::shared_ptr<FunctionBody>& body, const FunctionSignature& signatrue)
+ : signatrue_(signatrue), body_(body) {}
+
+ template<typename R, typename... Args>
+ R call(const remove_cvref_t<Args>&... args) const {
+ if (!detail::CheckSignature<R(Args...)>(signatrue_).Ok()) {
+ LOG(FATAL) << "The function was called with wrong arguments.";
+ }
+ using FuncType = std::function<R(const remove_cvref_t<Args>&...)>;
+ auto* func = reinterpret_cast<FuncType*>(body_->operator void*());
+ return (*func)(std::forward<const remove_cvref_t<Args>&>(args)...);
+ }
+
+ private:
+ FunctionSignature signatrue_;
+ std::shared_ptr<FunctionBody> body_;
+};
+
+class PackedFunctor {
+ public:
+ virtual ~PackedFunctor() = default;
+
+ template<typename Func>
+ static PackedFunctor Make(const std::string& func_name, const Func& func);
+
+ template<typename R, typename... Args>
+ R call(Args... args) const {
+ return functor_.call<R, Args...>(std::forward<Args>(args)...);
+ }
+
+ private:
+ PackedFunctor(const std::string& func_name, const Functor& functor)
+ : func_name_(func_name), functor_(functor) {}
+
+ std::string func_name_;
+ Functor functor_;
+};
+
+template<typename Func>
+PackedFunctor PackedFunctor::Make(const std::string& func_name, const Func& func) {
+ using func_type = typename function_traits<Func>::func_type;
+ auto body = std::make_shared<FunctionBodyImpl<func_type>>(func);
+ FunctionSignature signatute = detail::PackFunctionSignature<func_type>::pack();
+ Functor functor(body, signatute);
+ return PackedFunctor(func_name, std::move(functor));
+}
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_FUNCTOR_H_
diff --git a/oneflow/core/functional/scalar.h b/oneflow/core/functional/scalar.h
new file mode 100644
index 0000000000000000000000000000000000000000..89e508926e7f856d364f8febd4a53decbd844cc9
--- /dev/null
+++ b/oneflow/core/functional/scalar.h
@@ -0,0 +1,78 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef ONEFLOW_CORE_FUNCTIONAL_SCALAR_H_
+#define ONEFLOW_CORE_FUNCTIONAL_SCALAR_H_
+
+#include <type_traits>
+
+#include "oneflow/core/common/maybe.h"
+#include "oneflow/core/functional/value_types.h"
+
+namespace oneflow {
+namespace one {
+namespace functional {
+
+class Scalar {
+ public:
+ Scalar() : Scalar(int32_t(0)) {}
+
+ template<typename T, typename std::enable_if<
+ std::is_integral<T>::value && std::is_signed<T>::value, int>::type = 0>
+ explicit Scalar(const T& value)
+ : value_type_(ValueTypeOf<T>()), value_{.s = value}, active_tag_(HAS_S) {}
+
+ template<typename T, typename std::enable_if<
+ std::is_integral<T>::value && std::is_unsigned<T>::value, int>::type = 0>
+ explicit Scalar(const T& value)
+ : value_type_(ValueTypeOf<T>()), value_{.u = value}, active_tag_(HAS_U) {}
+
+ template<typename T, typename std::enable_if<std::is_floating_point<T>::value, int>::type = 0>
+ explicit Scalar(const T& value)
+ : value_type_(ValueTypeOf<T>()), value_{.d = value}, active_tag_(HAS_D) {}
+
+ const ValueType& type() const { return value_type_; }
+
+ template<typename T, typename std::enable_if<std::is_scalar<T>::value, int>::type = 0>
+ Maybe<T> As() const {
+ switch (active_tag_) {
+ case HAS_S: return static_cast<T>(value_.s);
+ case HAS_U: return static_cast<T>(value_.u);
+ case HAS_D: return static_cast<T>(value_.d);
+ default: UNIMPLEMENTED_THEN_RETURN() << "The scalar has not been initialized.";
+ }
+ }
+
+ bool IsIntegral() const { return active_tag_ == HAS_S || active_tag_ == HAS_U; }
+ bool IsFloatingPoint() const { return active_tag_ == HAS_D; }
+ bool IsSigned() const { return active_tag_ == HAS_S || active_tag_ == HAS_D; }
+ bool IsUnsigned() const { return active_tag_ == HAS_U; }
+
+ private:
+ ValueType value_type_;
+ union Value {
+ int64_t s;
+ uint64_t u;
+ double d;
+ } value_;
+ enum { HAS_S, HAS_U, HAS_D, HAS_NONE } active_tag_;
+};
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_SCALAR_H_
diff --git a/oneflow/core/functional/value_types.h b/oneflow/core/functional/value_types.h
new file mode 100644
index 0000000000000000000000000000000000000000..4d0f1b0ad21ab52009e861d43d90ee2d18f3986b
--- /dev/null
+++ b/oneflow/core/functional/value_types.h
@@ -0,0 +1,121 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef ONEFLOW_CORE_FUNCTIONAL_VALUE_TYPES_H_
+#define ONEFLOW_CORE_FUNCTIONAL_VALUE_TYPES_H_
+
+#include <memory>
+
+#include "oneflow/core/common/data_type.pb.h"
+#include "oneflow/core/common/maybe.h"
+
+namespace oneflow {
+class Shape;
+class AttrMap;
+
+namespace cfg {
+class AttrValue;
+} // namespace cfg
+
+namespace one {
+class Tensor;
+class TensorTuple;
+
+namespace functional {
+class Scalar;
+} // namespace functional
+} // namespace one
+
+namespace one {
+namespace functional {
+
+enum ValueType {
+ kINVALID = 0,
+ kVOID,
+ kINT32,
+ kUINT32,
+ kINT64,
+ kUINT64,
+ kFLOAT,
+ kDOUBLE,
+ kBOOL,
+ kSTRING,
+ kINT32_LIST,
+ kUINT32_LIST,
+ kINT64_LIST,
+ kUINT64_LIST,
+ kFLOAT_LIST,
+ kDOUBLE_LIST,
+ kBOOL_LIST,
+ kSTRING_LIST,
+ kSCALAR,
+ kTENSOR,
+ kTENSOR_REF,
+ kTENSOR_MAYBE,
+ kTENSOR_TUPLE,
+ kTENSOR_TUPLE_REF,
+ kTENSOR_TUPLE_MAYBE,
+ kATTR,
+ kATTR_REF,
+ kATTR_MAP,
+ kDTYPE,
+ kSHAPE,
+};
+
+#define VALUE_TYPE_OF_IMPL(cpp_type, value_type) \
+ template<typename T, typename std::enable_if<std::is_same<T, cpp_type>::value, int>::type = 0> \
+ inline ValueType ValueTypeOf() { \
+ return value_type; \
+ }
+
+VALUE_TYPE_OF_IMPL(void, kVOID);
+VALUE_TYPE_OF_IMPL(int32_t, kINT32);
+VALUE_TYPE_OF_IMPL(uint32_t, kUINT32);
+VALUE_TYPE_OF_IMPL(int64_t, kINT64);
+VALUE_TYPE_OF_IMPL(uint64_t, kUINT64);
+VALUE_TYPE_OF_IMPL(float, kFLOAT);
+VALUE_TYPE_OF_IMPL(double, kDOUBLE);
+VALUE_TYPE_OF_IMPL(bool, kBOOL);
+VALUE_TYPE_OF_IMPL(std::string, kSTRING);
+VALUE_TYPE_OF_IMPL(std::vector<int32_t>, kINT32_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<uint32_t>, kUINT32_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<int64_t>, kINT64_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<uint64_t>, kUINT64_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<float>, kFLOAT_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<double>, kDOUBLE_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<bool>, kBOOL_LIST);
+VALUE_TYPE_OF_IMPL(std::vector<std::string>, kSTRING_LIST);
+
+VALUE_TYPE_OF_IMPL(Scalar, kSCALAR);
+VALUE_TYPE_OF_IMPL(one::Tensor, kTENSOR);
+VALUE_TYPE_OF_IMPL(std::shared_ptr<one::Tensor>, kTENSOR_REF);
+VALUE_TYPE_OF_IMPL(Maybe<one::Tensor>, kTENSOR_MAYBE);
+VALUE_TYPE_OF_IMPL(one::TensorTuple, kTENSOR_TUPLE);
+VALUE_TYPE_OF_IMPL(std::shared_ptr<one::TensorTuple>, kTENSOR_TUPLE_REF);
+VALUE_TYPE_OF_IMPL(Maybe<one::TensorTuple>, kTENSOR_TUPLE_MAYBE);
+VALUE_TYPE_OF_IMPL(cfg::AttrValue, kATTR);
+VALUE_TYPE_OF_IMPL(std::shared_ptr<cfg::AttrValue>, kATTR_REF);
+VALUE_TYPE_OF_IMPL(AttrMap, kATTR_MAP);
+VALUE_TYPE_OF_IMPL(DataType, kDTYPE);
+VALUE_TYPE_OF_IMPL(Shape, kSHAPE);
+
+#undef VALUE_TYPE_OF_IMPL
+
+} // namespace functional
+} // namespace one
+} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_VALUE_TYPES_H_
diff --git a/oneflow/python/framework/functional.py b/oneflow/python/framework/functional.py
new file mode 100644
index 0000000000000000000000000000000000000000..af726befbca345249583d82f7bd427de36f04835
--- /dev/null
+++ b/oneflow/python/framework/functional.py
@@ -0,0 +1,69 @@
+"""
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import oneflow as flow
+import oneflow._oneflow_internal
+
+
+def RecursveDetermine(arg):
+ if isinstance(arg, flow.Tensor):
+ if not arg.is_determined:
+ arg.determine()
+ return arg._local_or_consistent_tensor
+ elif isinstance(arg, list) or isinstance(arg, tuple):
+ arg = list(arg)
+ for i in range(len(arg)):
+ arg[i] = RecursveDetermine(arg[i])
+ return arg
+ elif isinstance(arg, dict):
+ for k, v in arg.items():
+ arg[k] = RecursveDetermine(v)
+ else:
+ return arg
+
+
+class Function:
+ def __init__(self, func_name, handle):
+ self.func_name = func_name
+ self.handle = handle
+
+ def __call__(self, *args, **kwargs):
+ args = list(args)
+ for i in range(len(args)):
+ args[i] = RecursveDetermine(args[i])
+ for k, v in kwargs.items():
+ kwargs[k] = RecursveDetermine(v)
+ return self.handle(*args, **kwargs)
+
+
+def RegisterFunctionalApis():
+ import inspect
+ import oneflow.F
+
+ for s in dir(oneflow._oneflow_internal.F):
+ f = getattr(oneflow._oneflow_internal.F, s)
+ if inspect.isbuiltin(f):
+ func_name = s
+ if s in _function_name_aliases:
+ func_name = _function_name_aliases[s]
+ setattr(oneflow.F, func_name, Function(func_name, f))
+ setattr(oneflow.F, s, Function(func_name, f))
+
+ del inspect
+
+
+_function_name_aliases = {
+ "add_scalar": "scalar_add",
+}
diff --git a/oneflow/python/framework/register_class_method_util.py b/oneflow/python/framework/register_class_method_util.py
index 94774f0872369a966a63d366211f96e55f510eaf..93e2fcd8224c38fdf218c53cf7639580fb48eb88 100644
--- a/oneflow/python/framework/register_class_method_util.py
+++ b/oneflow/python/framework/register_class_method_util.py
@@ -15,6 +15,7 @@ limitations under the License.
"""
import oneflow.python.eager.eager_blob_util as eager_blob_util
import oneflow.python.framework.op_expr_util as op_expr_util
+import oneflow.python.framework.functional as functional
import oneflow.python.framework.remote_blob as remote_blob_util
import oneflow.python.framework.blob_trait as blob_trait
import oneflow._oneflow_internal
@@ -22,6 +23,7 @@ import oneflow._oneflow_internal
def RegisterMethod4Class():
op_expr_util.RegisterMethod4UserOpExpr()
+ functional.RegisterFunctionalApis()
eager_blob_util.RegisterMethod4EagerPhysicalBlob()
diff --git a/oneflow/python/nn/modules/batchnorm.py b/oneflow/python/nn/modules/batchnorm.py
index 07657a098831e4c470ba6c4f392d3c10b3f98649..5f4613442b6bcc59387e9713831843158fd8be05 100644
--- a/oneflow/python/nn/modules/batchnorm.py
+++ b/oneflow/python/nn/modules/batchnorm.py
@@ -101,36 +101,6 @@ class _BatchNorm(_NormBase):
track_running_stats=True,
):
super().__init__(num_features, eps, momentum, affine, track_running_stats)
- self._training_op = (
- flow.builtin_op("normalization")
- .Input("x")
- .Input("moving_mean")
- .Input("moving_variance")
- .Input("gamma")
- .Input("beta")
- .Attr("axis", 1)
- .Attr("epsilon", eps)
- .Attr("momentum", momentum)
- .Output("y")
- .Output("mean")
- .Output("inv_variance")
- .Attr("training", True)
- .Build()
- )
- self._testing_op = (
- flow.builtin_op("normalization")
- .Input("x")
- .Input("moving_mean")
- .Input("moving_variance")
- .Input("gamma")
- .Input("beta")
- .Attr("axis", 1)
- .Attr("epsilon", eps)
- .Attr("momentum", momentum)
- .Output("y")
- .Attr("training", False)
- .Build()
- )
def forward(self, x):
self._check_input_dim(x)
@@ -191,15 +161,17 @@ class _BatchNorm(_NormBase):
return affined
else:
- if self.training:
- res = self._training_op(
- x, self.running_mean, self.running_var, self.weight, self.bias
- )[0]
- else:
- res = self._testing_op(
- x, self.running_mean, self.running_var, self.weight, self.bias
- )[0]
- return res
+ return flow.F.normalization(
+ x,
+ self.running_mean,
+ self.running_var,
+ self.weight,
+ self.bias,
+ axis=1,
+ epsilon=self.eps,
+ momentum=self.momentum,
+ is_training=self.training,
+ )
@oneflow_export("nn.BatchNorm1d")
diff --git a/oneflow/python/nn/modules/math_ops.py b/oneflow/python/nn/modules/math_ops.py
index c335359a2c1d2f17767412718517626c6f9f60ff..9a720c555aaf56a4a7d60d2de2a4fca35ec2a400 100644
--- a/oneflow/python/nn/modules/math_ops.py
+++ b/oneflow/python/nn/modules/math_ops.py
@@ -356,31 +356,14 @@ class BroadcastSub(Module):
class ScalarAdd(Module):
- def __init__(self, operand) -> None:
+ def __init__(self, alpha) -> None:
super().__init__()
- self._op = flow.builtin_op("scalar_add").Input("in").Output("out")
-
- if isinstance(operand, int):
- self._op = (
- self._op.Attr("has_int_operand", True)
- .Attr("has_float_operand", False)
- .Attr("int_operand", operand)
- .Attr("float_operand", 0.0)
- .Build()
- )
- elif isinstance(operand, float):
- self._op = (
- self._op.Attr("has_int_operand", False)
- .Attr("has_float_operand", True)
- .Attr("int_operand", 0)
- .Attr("float_operand", operand)
- .Build()
- )
- else:
- raise ValueError("operand type can only be int or float")
+ if not isinstance(alpha, int) and not isinstance(alpha, float):
+ raise ValueError("scalar type can only be int or float")
+ self.alpha = alpha
def forward(self, x):
- return self._op(x)[0]
+ return flow.F.add_scalar(x, self.alpha)
@oneflow_export("sub")
@@ -576,10 +559,9 @@ class ScalarAddByTensor(Module):
class ElementwiseAdd(Module):
def __init__(self) -> None:
super().__init__()
- self._op = flow.builtin_op("add_n").Input("in", 2).Output("out").Build()
def forward(self, x, y):
- return self._op(x, y)[0]
+ return flow.F.add(x, y)
class BroadcastAdd(Module):
diff --git a/tools/generate_functional_api.py b/tools/generate_functional_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..630f5e3eada517bf976ea79a1dacb82d10e0bd58
--- /dev/null
+++ b/tools/generate_functional_api.py
@@ -0,0 +1,465 @@
+"""
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+import re
+import argparse
+import yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--yaml_file_path",
+ type=str,
+ default="oneflow/core/functional/functional_api.yaml",
+ help="The yaml format file that helps to generate the functional api or pybind cpp.",
+)
+parser.add_argument(
+ "--generate_pybind",
+ action="store_true",
+ default=False,
+ help="Enable to generate functional pybind cpp files.",
+)
+args = parser.parse_args()
+
+api_generate_dir = "oneflow/core/functional"
+pybind_generate_dir = "oneflow/api/python/functional"
+
+license = """/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the \"License\");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an \"AS IS\" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Generated from oneflow/core/functional/functional_api.yaml. DO NOT EDIT!"""
+
+header_fmt = (
+ license
+ + """
+
+#ifndef ONEFLOW_CORE_FUNCTIONAL_GENERATED_FUNCTIONAL_API_H_
+#define ONEFLOW_CORE_FUNCTIONAL_GENERATED_FUNCTIONAL_API_H_
+
+#include "oneflow/core/framework/tensor.h"
+#include "oneflow/core/framework/tensor_tuple.h"
+#include "oneflow/core/functional/scalar.h"
+
+namespace oneflow {{
+namespace one {{
+namespace functional {{
+{0}
+}} // namespace functional
+}} // namespace one
+}} // namespace oneflow
+
+#endif // ONEFLOW_CORE_FUNCTIONAL_GENERATED_FUNCTIONAL_API_H_"""
+)
+
+source_fmt = (
+ license
+ + """
+
+#include "{0}/functional_api.yaml.h"
+#include "oneflow/core/functional/function_library.h"
+
+namespace oneflow {{
+namespace one {{
+namespace functional {{
+{1}
+}} // namespace functional
+}} // namespace one
+}} // namespace oneflow
+"""
+)
+
+pybind_fmt = (
+ license
+ + """
+
+#include <vector>
+#include <pybind11/pybind11.h>
+
+#include "oneflow/api/python/of_api_registry.h"
+#include "oneflow/api/python/functional/function_def.h"
+#include "oneflow/api/python/functional/py_function.h"
+#include "oneflow/core/common/maybe.h"
+#include "oneflow/core/functional/functional.h"
+
+namespace oneflow {{
+namespace one {{
+namespace functional {{
+{0}
+}} // namespace functional
+}} // namespace one
+
+namespace functional = one::functional;
+
+ONEFLOW_API_PYBIND11_MODULE("F", m) {{
+{1}
+}}
+
+}} // namespace oneflow
+"""
+)
+
+types_allowed = {
+ "Tensor",
+ "TensorTuple",
+ "Scalar",
+ "Int",
+ "Int32",
+ "Int64",
+ "Float",
+ "Double",
+ "String",
+ "Bool",
+ "ScalarList",
+ "IntList",
+ "Int32List",
+ "Int64List",
+ "FloatList",
+ "DoubleList",
+ "StringList",
+ "BoolList",
+ "DataType",
+ "Shape",
+}
+
+generic_type_aliases = {
+ "Int": "int32_t",
+ "Int32": "int32_t",
+ "Int64": "int64_t",
+ "Float": "float",
+ "Double": "double",
+ "Bool": "bool",
+}
+
+argument_type_aliases = {
+ "Tensor": "const std::shared_ptr<one::Tensor>&",
+ "TensorTuple": "const TensorTuple&",
+ "Scalar": "const Scalar&",
+ "ScalarList": "const std::vector<Scalar>&",
+ "IntList": "const std::vector<int32_t>&",
+ "Int32List": "const std::vector<int32_t>&",
+ "Int64List": "const std::vector<int64_t>&",
+ "FloatList": "const std::vector<float>&",
+ "DoubleList": "const std::vector<double>&",
+ "String": "const std::string&",
+ "StringList": "const std::vector<std::string>&",
+ "BoolList": "const std::vector<bool>&",
+ "DataType": "const DataType&",
+ "Shape": "const Shape&",
+ **generic_type_aliases,
+}
+
+return_type_aliases = {
+ "Tensor": "Maybe<one::Tensor>",
+ "TensorTuple": "Maybe<one::TensorTuple>",
+ "String": "std::string",
+ **generic_type_aliases,
+}
+
+value_aliases = {
+ "True": "true",
+ "False": "false",
+}
+
+
+def _escape_quote(fmt):
+ return re.sub(r"\"|\'", '\\"', fmt)
+
+
+def _normalize(fmt):
+ fmt = fmt.strip()
+ return re.sub(r"\s+", " ", fmt)
+
+
+def _std_decay(fmt):
+ fmt = fmt.strip()
+ fmt = re.sub(r"(const|&)", "", fmt)
+ return _normalize(fmt)
+
+
+def parse_function_params(fmt):
+ params = []
+ fmt = _normalize(fmt)
+ if not fmt.endswith(")"):
+ raise ValueError('Function def should end with ")": ' + fmt)
+ open_paren = fmt.find("(")
+ if open_paren == -1:
+ raise ValueError('Missing "(" in function def: ' + fmt)
+
+ header = fmt[0:open_paren]
+ items = header.split(" ")
+ if (len(items)) != 2:
+ raise ValueError("Missing return type or function name in function def: " + fmt)
+ params.append(items[0])
+ function_name = items[1]
+
+ tail = fmt[open_paren + 1 : -1]
+ # TODO(): Parse the parameter list more comprehensively.
+ items = tail.split(",")
+ for param in items:
+ params.append(_normalize(param))
+ return function_name, params
+
+
+class Argument:
+ def __init__(self, fmt, keyword_allowed=False):
+ self._keyword_allowed = keyword_allowed
+ self._type = None
+ self._name = None
+ self._default_value = None
+
+ fmt = _normalize(fmt)
+ sp = fmt.rfind(" ")
+ if sp == -1:
+ raise ValueError("Missing argument type or name for argument def: " + fmt)
+ self._type = _normalize(fmt[0:sp])
+ assert self._type in types_allowed, "Unknow type: " + self._type
+
+ if self._type in argument_type_aliases:
+ self._cpp_type = argument_type_aliases[self._type]
+ else:
+ self._cpp_type = self._type
+ self._name = _normalize(fmt[sp + 1 :])
+ sp = self._name.find("=")
+ if sp != -1:
+ self._default_value = _normalize(self._name[sp + 1 :])
+ if self._default_value in value_aliases:
+ self._default_cpp_value = value_aliases[self._default_value]
+ else:
+ self._default_cpp_value = self._default_value
+ self._name = _normalize(self._name[0:sp])
+
+ @property
+ def has_default_value(self):
+ return self._default_value is not None
+
+ def to_string(self, to_cpp=False):
+ fmt = "{0} {1}".format(self._cpp_type if to_cpp else self._type, self._name)
+ if not to_cpp and self.has_default_value:
+ fmt += "={0}".format(self._default_value)
+ return fmt
+
+
+class Return:
+ def __init__(self, fmt):
+ self._type = _normalize(fmt)
+ assert self._type in types_allowed, "Unknow type: " + self._type
+
+ if self._type in return_type_aliases:
+ self._cpp_type = return_type_aliases[self._type]
+ else:
+ self._cpp_type = self._type
+
+ @property
+ def type(self):
+ return self._type
+
+ def to_string(self, to_cpp=False):
+ return self._cpp_type if to_cpp else self._type
+
+
+class FunctionSignature:
+ def __init__(self, fmt):
+ self._fmt = fmt
+ self._name, self._params = parse_function_params(fmt)
+ self._ret = Return(self._params[0])
+ keyword_allowed = False
+ self._args = []
+ for arg in self._params[1:]:
+ if arg == "*":
+ keyword_allowed = True
+ continue
+ self._args.append(Argument(arg, keyword_allowed=keyword_allowed))
+
+ self._max_args_count = len(self._args)
+ self._max_positional_args_count = self._max_args_count
+ count = 0
+ for arg in self._args:
+ if arg._keyword_allowed:
+ count += 1
+ self._max_keyword_args_count = count
+
+ @property
+ def num_of_args():
+ return len(self._args)
+
+ def to_string(self, to_cpp=False):
+ fmt = "{0} {1}(".format(self._ret.to_string(to_cpp=to_cpp), self._name)
+ keyword_start = False
+ for i, arg in enumerate(self._args):
+ if i > 0 and i < len(self._args):
+ fmt += ", "
+ if not keyword_start and arg._keyword_allowed:
+ keyword_start = True
+ if not to_cpp:
+ fmt += "*, "
+ fmt += arg.to_string(to_cpp=to_cpp)
+ fmt += ")"
+ return fmt
+
+
+class Block:
+ def __init__(self, name, signature, bind_python):
+ self._name = name
+ self._signature = signature
+ self._bind_python = bind_python
+
+
+class FunctionalGenerator:
+ def __init__(self, input_file):
+ self._blocks = {}
+ with open(input_file) as f:
+ doc = yaml.load(f, Loader=yaml.FullLoader)
+ for block in doc:
+ assert "name" in block
+ assert "signature" in block
+ name = block["name"]
+ signature = block["signature"]
+ bind_python = False
+ if "bind_python" in block:
+ bind_python = block["bind_python"]
+ self._blocks[name] = Block(
+ name, FunctionSignature(signature), bind_python
+ )
+
+ def generate_cpp_header_file(self, target_header_file):
+ fmt = ""
+ for name, block in self._blocks.items():
+ fmt += "\n"
+ fmt += block._signature.to_string(to_cpp=True)
+ fmt += ";\n"
+
+ with open(target_header_file, "w") as f:
+ f.write(header_fmt.format(fmt))
+
+ def generate_cpp_source_file(self, target_source_file):
+ fmt = ""
+ for name, block in self._blocks.items():
+ signature = block._signature
+ fmt += "\n"
+ fmt += signature.to_string(to_cpp=True)
+ fmt += " {\n"
+ fmt += ' static thread_local const auto& op = CHECK_JUST(FunctionLibrary::Global()->find("{0}"));\n'.format(
+ signature._name
+ )
+ fmt += " return op->call<{0}, {1}>({2});\n".format(
+ signature._ret._cpp_type,
+ ", ".join([arg._cpp_type for arg in signature._args]),
+ ", ".join([arg._name for arg in signature._args]),
+ )
+ fmt += "}\n"
+
+ with open(target_source_file, "w") as f:
+ f.write(source_fmt.format(api_generate_dir, fmt))
+
+ def generate_pybind_for_python(self, target_pybind_source_file):
+ schema_fmt = ""
+ module_fmt = ""
+ for name, block in self._blocks.items():
+ if not block._bind_python:
+ continue
+ signature = block._signature
+ return_type = signature._ret._cpp_type
+ schema_fmt += "\n"
+ schema_fmt += "struct {0}Schema {{\n".format(signature._name)
+ schema_fmt += " using FType = decltype(functional::{0});\n".format(
+ signature._name
+ )
+ schema_fmt += " using R = {0};\n".format(return_type)
+ schema_fmt += "\n"
+ schema_fmt += " static constexpr FType* func = &functional::{0};\n".format(
+ signature._name
+ )
+ schema_fmt += " static constexpr size_t max_args = {0};\n".format(
+ signature._max_args_count
+ )
+ schema_fmt += " static constexpr size_t max_positionals = {0};\n".format(
+ signature._max_positional_args_count
+ )
+ schema_fmt += " static constexpr size_t max_keywords = {0};\n".format(
+ signature._max_keyword_args_count
+ )
+ schema_fmt += ' static constexpr char const* signature = "{0}";\n'.format(
+ _escape_quote(signature.to_string())
+ )
+ schema_fmt += " static ReturnDef return_def;\n"
+ schema_fmt += " static std::vector<ArgumentDef> argument_def;\n"
+ schema_fmt += "};\n"
+ schema_fmt += "\n"
+ schema_fmt += "ReturnDef {0}Schema::return_def = ReturnDef(ValueTypeOf<{1}>());\n".format(
+ signature._name, return_type,
+ )
+
+ argument_def = []
+ for arg in signature._args:
+ if arg.has_default_value:
+ argument_def.append(
+ 'ArgumentDef("{0}", {1}({2}))'.format(
+ arg._name, _std_decay(arg._cpp_type), arg._default_cpp_value
+ )
+ )
+ else:
+ argument_def.append(
+ 'ArgumentDef("{0}", ValueTypeOf<{1}>())'.format(
+ arg._name, _std_decay(arg._cpp_type)
+ )
+ )
+
+ schema_fmt += "std::vector<ArgumentDef> {0}Schema::argument_def = {{{1}}};\n".format(
+ signature._name, ", ".join(argument_def)
+ )
+ module_fmt += ' m.def("{0}", &functional::PyFunction<functional::{1}Schema>);\n'.format(
+ name, signature._name
+ )
+
+ with open(target_pybind_source_file, "w") as f:
+ f.write(pybind_fmt.format(schema_fmt, module_fmt))
+
+
+if __name__ == "__main__":
+ assert os.path.isfile(args.yaml_file_path), (
+ "It is not a regular file for the yaml file which is " + args.yaml_file_path
+ )
+ g = FunctionalGenerator(args.yaml_file_path)
+
+ assert os.path.isdir(api_generate_dir), (
+ "Could not locate the api generate directory which is " + api_generate_dir
+ )
+ target_header_file = os.path.join(api_generate_dir, "functional_api.yaml.h")
+ g.generate_cpp_header_file(target_header_file)
+ target_source_file = os.path.join(api_generate_dir, "functional_api.yaml.cpp")
+ g.generate_cpp_source_file(target_source_file)
+
+ if args.generate_pybind:
+ assert os.path.isdir(pybind_generate_dir), (
+ "Could not locate the pybind generate directory which is "
+ + pybind_generate_dir
+ )
+ target_pybind_source_file = os.path.join(
+ pybind_generate_dir, "functional_api.yaml.pybind.cpp"
+ )
+ g.generate_pybind_for_python(target_pybind_source_file)