diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c6ffc8b1853366edf2999d5e918ed3697688bf8..c1fb5f7564b3a88b18178ab9b9e9a2ed6368f427 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ option(WITH_COCOAPI "Option to build with COCO API" ON) option(BUILD_GIT_VERSION "" ON) option(BUILD_PROFILER "" OFF) option(OF_SOFTMAX_USE_FAST_MATH "" ON) +option(BUILD_CPP_LIB "Option to build with c++ bindings" ON) set(RPC_BACKEND "GRPC,LOCAL" CACHE STRING "") set(THIRD_PARTY_MIRROR "" CACHE STRING "") set(PIP_INDEX_MIRROR "" CACHE STRING "") diff --git a/cmake/oneflow.cmake b/cmake/oneflow.cmake index 6d8e92e0e93a31d16f7ebcbd9d2fdde9963af8c6..b5f0393b51900aa35bcf7c9d12fcc8bc95a4b802 100644 --- a/cmake/oneflow.cmake +++ b/cmake/oneflow.cmake @@ -381,5 +381,8 @@ list(APPEND OF_CORE_HDRS "${PROJECT_SOURCE_DIR}/oneflow/core/autograd/autograd_m copy_files("${OF_CORE_HDRS}" "${PROJECT_SOURCE_DIR}" "${ONEFLOW_INCLUDE_DIR}" of_include_copy) add_custom_target(oneflow_py ALL) add_dependencies(oneflow_py of_include_copy) -add_subdirectory(${PROJECT_SOURCE_DIR}/oneflow/api/cpp) -target_link_libraries(oneflow of_ccobj) + +if (BUILD_CPP_LIB) + add_subdirectory(${PROJECT_SOURCE_DIR}/oneflow/api/cpp) + target_link_libraries(oneflow_infer_test ${of_libs} ${oneflow_third_party_libs} ${oneflow_exe_third_party_libs}) +endif() \ No newline at end of file diff --git a/oneflow/api/cpp/CMakeLists.txt b/oneflow/api/cpp/CMakeLists.txt index f81384e6d2a5edc35d08191d14b3bd680718d444..705f5e230b893293b58e0971ec133b8ea10dd19f 100644 --- a/oneflow/api/cpp/CMakeLists.txt +++ b/oneflow/api/cpp/CMakeLists.txt @@ -1 +1,3 @@ -add_library(oneflow SHARED inference_session.cpp) \ No newline at end of file +#add_library(oneflow SHARED inference_session.cpp job_instance.cpp tensor/tensor.cpp) +add_executable(oneflow_infer_test inference_session_test.cpp inference_session.cpp job_instance.cpp tensor/tensor.cpp) +#target_link_libraries(oneflow_infer_test oneflow) \ No newline at end of file diff --git a/oneflow/api/cpp/inference_session.cpp b/oneflow/api/cpp/inference_session.cpp index b49bb346268950228d74d4cb7df2f3051f1a9144..caca0bf7683386231a0a3b819fd5da77883d25f0 100644 --- a/oneflow/api/cpp/inference_session.cpp +++ b/oneflow/api/cpp/inference_session.cpp @@ -47,7 +47,7 @@ inline int FindModelLatestVersion(std::string saved_model_dir) { versions.push_back(std::stoi(f)); } catch (...) {} } - return *(std::max(std::begin(versions), std::end(versions))); + return *(std::max_element(std::begin(versions), std::end(versions))); } inline bool NeedCheckDeviceTag(OperatorConf& op_conf) { @@ -70,11 +70,17 @@ InferenceSession::~InferenceSession() { } Maybe<void> InferenceSession::Init() { + OpenDefaultSession(); + // env init if(!IsEnvInited().GetOrThrow()) { - // TODO: set env - machine id, ctrl port, data port EnvProto env_proto; - // FIXME: multi-client should be true or false? + auto* machine = env_proto.add_machine(); + machine->set_id(0); + machine->set_addr("127.0.0.1"); + env_proto.set_ctrl_port(option_.ctrl_port); + + SetIsMultiClient(false); JUST(InitEnv(env_proto, false)); // scope init @@ -227,10 +233,12 @@ Maybe<void> InferenceSession::LoadModel_(std::string saved_model_dir, SavedModel saved_model_proto; if (std::find(std::begin(subfiles), std::end(subfiles), saved_model_meta_pb_filename) != std::end(subfiles)) { - CHECK_OR_RETURN(TryParseProtoFromPbFile(saved_model_meta_pb_filename, &saved_model_proto)); + std::string file_path = JoinPath(saved_model_path, saved_model_meta_pb_filename); + CHECK_OR_RETURN(TryParseProtoFromPbFile(file_path, &saved_model_proto)); } else if (std::find(std::begin(subfiles), std::end(subfiles), saved_model_meta_prototxt_filename) != std::end(subfiles)) { - CHECK_OR_RETURN(TryParseProtoFromTextFile(saved_model_meta_prototxt_filename, &saved_model_proto)); + std::string file_path = JoinPath(saved_model_path, saved_model_meta_prototxt_filename); + CHECK_OR_RETURN(TryParseProtoFromTextFile(file_path, &saved_model_proto)); } else { CHECK_OR_RETURN(false) << Error::ValueError("saved model meta file" + saved_model_meta_file_basename + " do not exist in " + saved_model_path); @@ -362,7 +370,7 @@ std::shared_ptr<JobConfigProto> InferenceSession::GetJobConf(std::string job_nam Maybe<void> InferenceSession::RunJob(std::shared_ptr<CPPJobInstance> job_inst) { std::shared_ptr<std::promise<void>> job_promise = std::make_shared<std::promise<void>>(); - auto job_finish_cb = [&job_promise](JobInstance*){ job_promise->set_value(); }; + auto job_finish_cb = [job_promise](const JobInstance*){ job_promise->set_value(); }; job_inst->AddPostFinishCallback(job_finish_cb); JUST(LaunchJob(job_inst)); this->job_promises_.push_back(job_promise); @@ -380,7 +388,7 @@ Maybe<void> InferenceSession::RunPushJobs(std::map<std::string, std::shared_ptr< std::shared_ptr<Tensor> input_tensor = input_tensors[input_name]; - auto push_fn = [&input_tensor](OfBlob* ofblob){ + auto push_fn = [input_tensor](OfBlob* ofblob){ ofblob->CopyShapeFrom(input_tensor->shape().dim_vec().data(), input_tensor->num_axes()); int64_t num_elems = input_tensor->num_elems(); DataType dtype = input_tensor->dtype(); @@ -434,7 +442,7 @@ Maybe<void> InferenceSession::RunLoadCheckpointJob() { auto copy_model_load_path = [this](OfBlob* ofblob) -> void { int64_t* shape = new int64_t[1]{this->checkpoint_path_.size()}; ofblob->CopyShapeFrom(shape, 1); - ofblob->AutoMemCopyFrom(this->checkpoint_path_.data(), this->checkpoint_path_.size()); + ofblob->AutoMemCopyFrom((const int8_t*) this->checkpoint_path_.data(), this->checkpoint_path_.size()); delete[] shape; }; diff --git a/oneflow/api/cpp/inference_session.h b/oneflow/api/cpp/inference_session.h index 9d1fd2da12668f0c775df03339ab9c91143d1a7e..6297a91a1c92ea6ab368e3b42c31ffc9e88ed17f 100644 --- a/oneflow/api/cpp/inference_session.h +++ b/oneflow/api/cpp/inference_session.h @@ -17,6 +17,9 @@ limitations under the License. #define ONEFLOW_API_CPP_INFERENCE_SESSION_H_ #include <future> +#include "oneflow/api/cpp/job_instance.h" +#include "oneflow/api/cpp/tensor/tensor.h" +#include "oneflow/core/serving/saved_model.pb.h" #include "oneflow/api/python/job_build/job_build_and_infer.h" #include "oneflow/api/python/job_build/job_build_and_infer_api.h" @@ -29,11 +32,12 @@ struct ModelVersionPolicy { struct SessionOption { SessionOption() : device_tag("gpu"), device_num(1), - is_mirrored_view(false) {} + is_mirrored_view(false), ctrl_port(11235) {} std::string device_tag; int device_num; bool is_mirrored_view; + int ctrl_port; }; class InferenceSession { diff --git a/oneflow/api/cpp/inference_session_test.cpp b/oneflow/api/cpp/inference_session_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5363a215cedc153e4bb681fd0773e18ba7170e59 --- /dev/null +++ b/oneflow/api/cpp/inference_session_test.cpp @@ -0,0 +1,61 @@ +/* +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include <string> +#include <stdlib.h> +#include "oneflow/api/cpp/inference_session.h" + +using namespace oneflow; + +char* load_image(std::string image_path) { + // TODO + return nullptr; +} + +int main() { + float* image_mock_data = new float[28*28]; + for(int i = 0; i < 28*28; i++) { + image_mock_data[i] = std::rand() / double(RAND_MAX); + } + std::string job_name = "mlp_inference"; + SessionOption option; + option.device_tag = "cpu"; + option.ctrl_port = 11235; + option.device_num = 1; + + InferenceSession session(option); + std::string model_path = "/home/allen/projects/oneflow-serving/lenet_models/"; + ModelVersionPolicy ver_policy; + ver_policy.latest = true; + ver_policy.version = 1; + std::string basename = "saved_model"; + session.LoadModel(model_path, ver_policy, basename); + session.Launch(); + + std::map<std::string, std::shared_ptr<Tensor>> tensor_inputs, tensor_outputs; + std::string image_path = "/home/allen/projects/oneflow-serving/7.png"; + //char* image_data = load_image(image_path); + tensor_inputs["Input_21"] = Tensor::fromBlob((char*)image_mock_data, {1,1,28,28}, kFloat); + tensor_outputs = session.Run(job_name, tensor_inputs); + + for (auto& entry : tensor_outputs) { + std::cout << "Output name:" << entry.first << std::endl; + //TODO + } + session.Close(); + + return 0; +} diff --git a/oneflow/api/cpp/job_instance.cpp b/oneflow/api/cpp/job_instance.cpp index 536eddf89a30c58268f2f03f8d1a866335231bd9..cba4e9eaf6170360e04d723a9cffffb2d6bac86a 100644 --- a/oneflow/api/cpp/job_instance.cpp +++ b/oneflow/api/cpp/job_instance.cpp @@ -31,40 +31,40 @@ CPPJobInstance::CPPJobInstance(std::string job_name, pull_cb_(pull_cb), finish_cb_(finish_cb) {} -~CPPJobInstance::CPPJobInstance(){} +CPPJobInstance::~CPPJobInstance(){} -std::string CPPJobInstance::job_name() const override { return this->job_name_; } +std::string CPPJobInstance::job_name() const { return this->job_name_; } -std::string CPPJobInstance::sole_input_op_name_in_user_job() const override { +std::string CPPJobInstance::sole_input_op_name_in_user_job() const { return this->sole_input_op_name_in_user_job_; } -std::string CPPJobInstance::sole_output_op_name_in_user_job() const override { +std::string CPPJobInstance::sole_output_op_name_in_user_job() const { return this->sole_output_op_name_in_user_job_; } -void CPPJobInstance::PushBlob(uint64_t ofblob_ptr) const override { - this->push_cb_(reinterpret_cast<OfBlob*>(of_blob_ptr)); +void CPPJobInstance::PushBlob(uint64_t ofblob_ptr) const { + this->push_cb_(reinterpret_cast<OfBlob*>(ofblob_ptr)); } -void CPPJobInstance::PullBlob(uint64_t ofblob_ptr) const override { - this->pull_cb_(reinterpret_cast<OfBlob*>(of_blob_ptr)); +void CPPJobInstance::PullBlob(uint64_t ofblob_ptr) const { + this->pull_cb_(reinterpret_cast<OfBlob*>(ofblob_ptr)); } -void CPPJobInstance::Finish() const override { +void CPPJobInstance::Finish() const { this->finish_cb_(); for (auto& post_finish_cb : this->post_finish_cbs_) post_finish_cb(this); } -void CPPJobInstance::AddPostFinishCallback(std::function<void(JobInstance*)> cb) { +void CPPJobInstance::AddPostFinishCallback(std::function<void(const JobInstance*)> cb) { this->post_finish_cbs_.push_back(cb); } std::shared_ptr<CPPJobInstance> MakeUserJobInstance( std::string job_name, - std::function<void()> finish_cb = std::function<void()>()) { + std::function<void()> finish_cb) { return std::make_shared<CPPJobInstance>(job_name, "", "", std::function<void(OfBlob*)>(), std::function<void(OfBlob*)>(), finish_cb); @@ -74,7 +74,7 @@ std::shared_ptr<CPPJobInstance> MakePullJobInstance( std::string job_name, std::string op_name, std::function<void(OfBlob*)> pull_cb, - std::function<void()> finish_cb = std::function<void()>()) { + std::function<void()> finish_cb) { return std::make_shared<CPPJobInstance>( job_name, op_name, "", std::function<void(OfBlob*)>(), pull_cb, finish_cb); } @@ -83,16 +83,16 @@ std::shared_ptr<CPPJobInstance> MakePushJobInstance( std::string job_name, std::string op_name, std::function<void(OfBlob*)> push_cb, - std::function<void()> finish_cb = std::function<void()>()) { + std::function<void()> finish_cb) { return std::make_shared<CPPJobInstance>( job_name, "", op_name, push_cb, std::function<void(OfBlob*)>(), finish_cb); } std::shared_ptr<CPPJobInstance> MakeArgPassJobInstance( std::string job_name, - std::string src_op_name; - std::string dst_op_name; - std::function<void()> finish_cb = std::function<void()>()) { + std::string src_op_name, + std::string dst_op_name, + std::function<void()> finish_cb) { return std::make_shared<CPPJobInstance>(job_name, src_op_name, dst_op_name, std::function<void(OfBlob*)>(), std::function<void(OfBlob*)>(), finish_cb); diff --git a/oneflow/api/cpp/job_instance.h b/oneflow/api/cpp/job_instance.h index b54b239cdbcf238ee4082a69b22a338d92ed18bd..fd5ef8f0a1adc0657b74044c456bbc812daa4fee 100644 --- a/oneflow/api/cpp/job_instance.h +++ b/oneflow/api/cpp/job_instance.h @@ -39,7 +39,7 @@ class CPPJobInstance : public JobInstance { void PushBlob(uint64_t ofblob_ptr) const override; void PullBlob(uint64_t ofblob_ptr) const override; void Finish() const override; - void AddPostFinishCallback(std::function<void(JobInstance*)> cb); + void AddPostFinishCallback(std::function<void(const JobInstance*)> cb); private: int thisown; @@ -49,26 +49,26 @@ class CPPJobInstance : public JobInstance { std::function<void(OfBlob*)> push_cb_; std::function<void(OfBlob*)> pull_cb_; std::function<void()> finish_cb_; - std::vector<std::function<void(CPPJobInstance*)>> post_finish_cbs_; + std::vector<std::function<void(const CPPJobInstance*)>> post_finish_cbs_; }; std::shared_ptr<CPPJobInstance> MakeUserJobInstance( std::string job_name, - std::function<void()> finish_cb = std::function<void()>() + std::function<void()> finish_cb = [](){} ); std::shared_ptr<CPPJobInstance> MakePullJobInstance( std::string job_name, std::string op_name, std::function<void(OfBlob*)> pull_cb, - std::function<void()> finish_cb = std::function<void()>() + std::function<void()> finish_cb = [](){} ); std::shared_ptr<CPPJobInstance> MakePushJobInstance( std::string job_name, std::string op_name, std::function<void(OfBlob*)> push_cb, - std::function<void()> finish_cb = std::function<void()>() + std::function<void()> finish_cb = [](){} ); std::shared_ptr<CPPJobInstance> MakeArgPassJobInstance( diff --git a/oneflow/api/cpp/session/session_util.h b/oneflow/api/cpp/session/session_util.h index 0f777c7f5d574901ff97873d1b3a564ffecbdc8d..b518f88fd258c6a2b9e93f2bf0091f0b8a48665a 100644 --- a/oneflow/api/cpp/session/session_util.h +++ b/oneflow/api/cpp/session/session_util.h @@ -10,6 +10,11 @@ namespace oneflow { +inline void OpenDefaultSession() { + int64_t session_id = oneflow::NewSessionId(); + oneflow::RegsiterSession(session_id); +} + ConfigProto GetDefaultConfigProto() { ConfigProto config_proto; config_proto.mutable_resource()->set_machine_num(0); diff --git a/oneflow/api/cpp/tensor/tensor.cpp b/oneflow/api/cpp/tensor/tensor.cpp index 0bc1d72b98ecadd537757b2bee3b479c4b25348a..e16049b40f0286711eebc219f228d17b98cb6ea9 100644 --- a/oneflow/api/cpp/tensor/tensor.cpp +++ b/oneflow/api/cpp/tensor/tensor.cpp @@ -20,24 +20,24 @@ namespace oneflow { Tensor::Tensor(Shape shape, DataType dtype) : shape_(shape), dtype_(dtype){ - int64_t num_elems = shape.elem_cnt(); + int64_t num_elems = shape_.elem_cnt(); size_t num_bytes = DType::Get(dtype).GetOrThrow()->bytes().GetOrThrow(); - this->data_ = new char[num_items*num_bytes]; + this->data_ = new char[num_elems*num_bytes]; } Tensor::~Tensor() { delete this->data_; } -Tensor::CopyFrom(const char* data) { +void Tensor::CopyFrom(const char* data) { int64_t num_elems = this->shape_.elem_cnt(); size_t num_bytes = DType::Get(this->dtype_).GetOrThrow()->bytes().GetOrThrow(); std::copy(data, data + num_elems * num_bytes, this->data_); } -static std::shared_ptr<Tensor> Tensor::fromBlob(char* blob_data, - Shape shape, - DataType dtype) { +std::shared_ptr<Tensor> Tensor::fromBlob(char* blob_data, + Shape shape, + DataType dtype) { std::shared_ptr<Tensor> tensor = std::make_shared<Tensor>(shape, dtype); tensor->CopyFrom(blob_data); return tensor; diff --git a/oneflow/api/cpp/tensor/tensor.h b/oneflow/api/cpp/tensor/tensor.h index cd62fb51047f969979aeef5d3dd27cfa66f13f60..7496d86b6106f042409cb17379cba30a1c09eb1c 100644 --- a/oneflow/api/cpp/tensor/tensor.h +++ b/oneflow/api/cpp/tensor/tensor.h @@ -37,7 +37,7 @@ class Tensor { Shape shape() const { return this->shape_; } DataType dtype() const { return this->dtype_; } - void CopyFrom(char* blob_data); + void CopyFrom(const char* blob_data); static std::shared_ptr<Tensor> fromBlob(char* blob_data, Shape shape,