Generate cfg header and source files in parallel and prevent rebuild from...

Generate cfg header and source files in parallel and prevent rebuild from scratch when Python version changes (#4876) * refine * Update cfg.cmake * Update cfg.cmake Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>

Generate cfg header and source files in parallel and prevent rebuild from...
Generate cfg header and source files in parallel and prevent rebuild from scratch when Python version changes (#4876) * refine * Update cfg.cmake * Update cfg.cmake Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>
72204213 · Shenghang Tsai · GitHub · 373cefce · 72204213 · 72204213
Unverified Commit 72204213 authored 3 years ago by Shenghang Tsai Committed by GitHub 3 years ago
--- a/cmake/cfg.cmake
+++ b/cmake/cfg.cmake
-execute_process( 
+execute_process(
  COMMAND python3 ${PROJECT_SOURCE_DIR}/tools/cfg/generate_cfg_head_dir_and_convert_src.py
    --get_message_type=cfg_include_dir
  OUTPUT_VARIABLE CFG_INCLUDE_DIR)

-execute_process( 
+execute_process(
  COMMAND python3 ${PROJECT_SOURCE_DIR}/tools/cfg/generate_cfg_head_dir_and_convert_src.py
    --get_message_type=template_convert_python_script
  OUTPUT_VARIABLE TEMPLATE_CONVERT_PYTHON_SCRIPT)

-execute_process( 
+execute_process(
  COMMAND python3 ${PROJECT_SOURCE_DIR}/tools/cfg/generate_cfg_head_dir_and_convert_src.py
    --get_message_type=copy_pyproto_python_script
  OUTPUT_VARIABLE COPY_PYPROTO_PYTHON_SCRIPT)

-execute_process( 
+execute_process(
  COMMAND python3 ${PROJECT_SOURCE_DIR}/tools/cfg/generate_cfg_head_dir_and_convert_src.py
    --get_message_type=pybind_registry_cc
  OUTPUT_VARIABLE PYBIND_REGISTRY_CC)

-execute_process( 
+execute_process(
  COMMAND python3 ${PROJECT_SOURCE_DIR}/tools/cfg/generate_cfg_head_dir_and_convert_src.py
    --get_message_type=template_files
  OUTPUT_VARIABLE TEMPLATE_FILES)
@@ -76,6 +76,7 @@ function(GENERATE_CFG_AND_PYBIND11_CPP SRCS HDRS PYBIND_SRCS ROOT_DIR)
    DEPENDS ${Python_EXECUTABLE} of_protoobj
  )

+  set(CFG_ARGS "")
  foreach(FIL ${CFG_SOURCE_FILE_CONVERT_PROTO})
    set(ABS_FIL ${ROOT_DIR}/${FIL})
    get_filename_component(FIL_WE ${FIL} NAME_WE)
@@ -83,22 +84,22 @@ function(GENERATE_CFG_AND_PYBIND11_CPP SRCS HDRS PYBIND_SRCS ROOT_DIR)
    file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR})
    set(CFG_HPP_FIL ${PROJECT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.cfg.h)
    set(CFG_CPP_FIL ${PROJECT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.cfg.cpp)
-    
-    add_custom_command(
-      OUTPUT "${CFG_HPP_FIL}"
-             "${CFG_CPP_FIL}"
-      COMMAND ${Python_EXECUTABLE} ${TEMPLATE_CONVERT_PYTHON_SCRIPT}
-        --of_cfg_proto_python_dir=${of_cfg_proto_python_dir}
-        --project_build_dir=${PROJECT_BINARY_DIR} --proto_file_path=${FIL}
-        --generate_file_type=cfg.cpp
-      DEPENDS copy_pyproto ${Python_EXECUTABLE} ${ABS_FIL} ${TEMPLATE_FILES}
-      COMMENT "Generating cfg.cpp file on ${FIL}"
-      VERBATIM)
-
    list(APPEND ${HDRS} ${CFG_HPP_FIL})
    list(APPEND ${SRCS} ${CFG_CPP_FIL})
+    list(APPEND CFG_ARGS "--proto_file_path=${FIL}")
  endforeach()

+  add_custom_target(
+    generate_cfg
+    COMMAND ${Python_EXECUTABLE} ${TEMPLATE_CONVERT_PYTHON_SCRIPT}
+      ${CFG_ARGS}
+      --of_cfg_proto_python_dir=${of_cfg_proto_python_dir}
+      --project_build_dir=${PROJECT_BINARY_DIR}
+      --generate_file_type=cfg.cpp
+    DEPENDS copy_pyproto ${Python_EXECUTABLE} ${ABS_FIL} ${TEMPLATE_FILES}
+    VERBATIM
+  )
+
  list(APPEND PYBIND11_FILE_CONVERT_PROTO
      oneflow/core/job/job_conf.proto
      oneflow/core/job/placement.proto
@@ -123,26 +124,28 @@ function(GENERATE_CFG_AND_PYBIND11_CPP SRCS HDRS PYBIND_SRCS ROOT_DIR)
      oneflow/core/operator/op_conf.proto
  )

+  set(PY_CFG_ARGS "")
  foreach(FIL ${PYBIND11_FILE_CONVERT_PROTO})
    set(ABS_FIL ${ROOT_DIR}/${FIL})
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${ABS_FIL} PATH)
    file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR})
    set(CFG_PYBIND_FIL ${PROJECT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.cfg.pybind.cpp)
-    
-    add_custom_command(
-      OUTPUT "${CFG_PYBIND_FIL}"
-      COMMAND ${Python_EXECUTABLE} ${TEMPLATE_CONVERT_PYTHON_SCRIPT}
-        --of_cfg_proto_python_dir=${of_cfg_proto_python_dir}
-        --project_build_dir=${PROJECT_BINARY_DIR} --proto_file_path=${FIL}
-        --generate_file_type=cfg.pybind.cpp
-      DEPENDS copy_pyproto ${Python_EXECUTABLE} ${ABS_FIL} ${TEMPLATE_FILES}
-      COMMENT "Generating cfg.pybind.cpp file on ${FIL}"
-      VERBATIM)
-
    list(APPEND ${PYBIND_SRCS} ${CFG_PYBIND_FIL})
+    list(APPEND PY_CFG_ARGS "--proto_file_path=${FIL}")
  endforeach()

+  add_custom_target(
+    generate_py_cfg
+    COMMAND ${Python_EXECUTABLE} ${TEMPLATE_CONVERT_PYTHON_SCRIPT}
+      ${PY_CFG_ARGS}
+      --of_cfg_proto_python_dir=${of_cfg_proto_python_dir}
+      --project_build_dir=${PROJECT_BINARY_DIR}
+      --generate_file_type=cfg.pybind.cpp
+    DEPENDS copy_pyproto ${Python_EXECUTABLE} ${ABS_FIL} ${TEMPLATE_FILES}
+    VERBATIM
+  )
+
  set_source_files_properties(${${SRCS}} ${${HDRS}} ${${PYBIND_SRCS}} PROPERTIES GENERATED TRUE)
  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
  set(${HDRS} ${${HDRS}} PARENT_SCOPE)

--- a/cmake/oneflow.cmake
+++ b/cmake/oneflow.cmake
@@ -221,7 +221,7 @@ add_dependencies(of_protoobj make_pyproto_dir ${PROTOBUF_COPY_TARGETS})
 include(cfg)
 GENERATE_CFG_AND_PYBIND11_CPP(CFG_SRCS CFG_HRCS PYBIND11_SRCS ${PROJECT_SOURCE_DIR})
 oneflow_add_library(of_cfgobj ${CFG_SRCS} ${CFG_HRCS})
-add_dependencies(of_cfgobj of_protoobj)
+add_dependencies(of_cfgobj of_protoobj generate_cfg)
 if (BUILD_SHARED_LIBS)
  target_link_libraries(of_protoobj ${PROTOBUF_STATIC_LIBRARIES})
  target_link_libraries(of_cfgobj ${PROTOBUF_STATIC_LIBRARIES})
@@ -271,7 +271,7 @@ endif()

 pybind11_add_module(oneflow_internal ${PYBIND11_SRCS} ${of_pybind_obj_cc} ${PYBIND_REGISTRY_CC})
 set_property(TARGET oneflow_internal PROPERTY CXX_VISIBILITY_PRESET "default")
-add_dependencies(oneflow_internal of_cfgobj)
+add_dependencies(oneflow_internal of_cfgobj generate_py_cfg)
 set_target_properties(oneflow_internal PROPERTIES PREFIX "_")
 set_target_properties(oneflow_internal PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/python_scripts/oneflow")
 target_link_libraries(oneflow_internal PRIVATE ${of_libs} ${oneflow_third_party_libs} of_pyext_obj ${oneflow_exe_third_party_libs})

--- a/tools/cfg/template_convert.py
+++ b/tools/cfg/template_convert.py
@@ -6,7 +6,7 @@ import util.proto_reflect_util as proto_reflect_util

 parser = argparse.ArgumentParser()
 parser.add_argument("-project_build", "--project_build_dir", type=str, required=True)
-parser.add_argument("-proto_file", "--proto_file_path", type=str, required=True)
+parser.add_argument("--proto_file_path", action="append", default=[])
 parser.add_argument(
    "-of_cfg_proto_python", "--of_cfg_proto_python_dir", type=str, required=True
 )
@@ -108,7 +108,13 @@ def render_template(proto_file):


 def main():
-    render_template(args.proto_file_path)
+    import multiprocessing
+
+    assert len(args.proto_file_path) > 0
+    with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool:
+        pool.map(render_template, args.proto_file_path)
+        pool.close()
+        pool.join()


 if __name__ == "__main__":