From 84d684592096e7a29457e87d70b271347a2cfe05 Mon Sep 17 00:00:00 2001 From: rjy <raojiayi@huawei.com> Date: Mon, 30 May 2022 16:51:39 +0800 Subject: [PATCH] add Auto-deeplab# This is the commit message #2: --- .../cv/Auto-DeepLab/infer/convert/convert.sh | 29 ++ .../Auto-DeepLab/infer/docker_start_infer.sh | 43 +++ .../cv/Auto-DeepLab/infer/export_for_mindx.py | 45 +++ .../Auto-DeepLab/infer/model/autodeeplab.cfg | 4 + .../Auto-DeepLab/infer/mxbase/CMakeLists.txt | 53 ++++ .../cv/Auto-DeepLab/infer/mxbase/build.sh | 74 +++++ .../infer/mxbase/src/AutoDeepLab.cc | 294 ++++++++++++++++++ .../infer/mxbase/src/AutoDeepLab.h | 67 ++++ .../cv/Auto-DeepLab/infer/mxbase/src/main.cc | 92 ++++++ .../cv/Auto-DeepLab/infer/sdk/__init__.py | 0 .../infer/sdk/autodeeplab_opencv.pipeline | 63 ++++ research/cv/Auto-DeepLab/infer/sdk/main.py | 117 +++++++ research/cv/Auto-DeepLab/infer/sdk/utils.py | 134 ++++++++ .../cv/Auto-DeepLab/infer/util/__init__.py | 0 .../Auto-DeepLab/infer/util/mindx_config.py | 90 ++++++ .../cv/Auto-DeepLab/infer/util/mindx_utils.py | 48 +++ .../infer/util/mxbase_postprocess.py | 148 +++++++++ .../cv/Auto-DeepLab/modelarts/train_start.py | 203 ++++++++++++ .../cv/Auto-DeepLab/scripts/docker_start.sh | 36 +++ 19 files changed, 1540 insertions(+) create mode 100644 research/cv/Auto-DeepLab/infer/convert/convert.sh create mode 100644 research/cv/Auto-DeepLab/infer/docker_start_infer.sh create mode 100644 research/cv/Auto-DeepLab/infer/export_for_mindx.py create mode 100644 research/cv/Auto-DeepLab/infer/model/autodeeplab.cfg create mode 100644 research/cv/Auto-DeepLab/infer/mxbase/CMakeLists.txt create mode 100644 research/cv/Auto-DeepLab/infer/mxbase/build.sh create mode 100644 research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.cc create mode 100644 research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.h create mode 100644 research/cv/Auto-DeepLab/infer/mxbase/src/main.cc create mode 100644 research/cv/Auto-DeepLab/infer/sdk/__init__.py create mode 100644 research/cv/Auto-DeepLab/infer/sdk/autodeeplab_opencv.pipeline create mode 100644 research/cv/Auto-DeepLab/infer/sdk/main.py create mode 100644 research/cv/Auto-DeepLab/infer/sdk/utils.py create mode 100644 research/cv/Auto-DeepLab/infer/util/__init__.py create mode 100644 research/cv/Auto-DeepLab/infer/util/mindx_config.py create mode 100644 research/cv/Auto-DeepLab/infer/util/mindx_utils.py create mode 100644 research/cv/Auto-DeepLab/infer/util/mxbase_postprocess.py create mode 100644 research/cv/Auto-DeepLab/modelarts/train_start.py create mode 100644 research/cv/Auto-DeepLab/scripts/docker_start.sh diff --git a/research/cv/Auto-DeepLab/infer/convert/convert.sh b/research/cv/Auto-DeepLab/infer/convert/convert.sh new file mode 100644 index 000000000..1cb986737 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/convert/convert.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +AIR_PATH=$1 +OM_PATH=$2 + +echo "Input path of AIR file: ${AIR_PATH}" +echo "Output path of OM file: ${OM_PATH}" + +atc --framework=1 \ + --model="${AIR_PATH}" \ + --output="${OM_PATH}" \ + --input_format=NHWC \ + --input_shape="actual_input_1:1,1024,2048,3" \ + --output_type=FP32 \ + --soc_version=Ascend310 +exit 0 diff --git a/research/cv/Auto-DeepLab/infer/docker_start_infer.sh b/research/cv/Auto-DeepLab/infer/docker_start_infer.sh new file mode 100644 index 000000000..d54704bd3 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/docker_start_infer.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker_image=$1 +share_dir=$2 +data_dir=$3 +echo "$1" +echo "$2" +if [ -z "${docker_image}" ]; then + echo "please input docker_image" + exit 1 +fi + +if [ ! -d "${share_dir}" ]; then + echo "please input share directory that contains dataset, models and codes" + exit 1 +fi + + +docker run -it -u root\ + --device=/dev/davinci0 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm \ + --device=/dev/hisi_hdc \ + --privileged \ + -v //usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v ${data_dir}:${data_dir} \ + -v ${share_dir}:${share_dir} \ + ${docker_image} \ + /bin/bash diff --git a/research/cv/Auto-DeepLab/infer/export_for_mindx.py b/research/cv/Auto-DeepLab/infer/export_for_mindx.py new file mode 100644 index 000000000..450b15ad2 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/export_for_mindx.py @@ -0,0 +1,45 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""export checkpoint file into mindir models""" +import sys +import os +import numpy as np + +from mindspore import Tensor, context, load_checkpoint, load_param_into_net, export + + +context.set_context(mode=context.GRAPH_MODE, save_graphs=False) + +if __name__ == "__main__": + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + from src.core.model import AutoDeepLab + from util.mindx_config import obtain_autodeeplab_args + from util.mindx_utils import InferWithFlipNetwork + + args = obtain_autodeeplab_args() + args.total_iters = 0 + + # net + autodeeplab = AutoDeepLab(args) + + # load checkpoint + param_dict = load_checkpoint(args.ckpt_name) + + # load the parameter into net + load_param_into_net(autodeeplab, param_dict) + network = InferWithFlipNetwork(autodeeplab, flip=args.infer_flip, input_format=args.input_format) + + input_data = np.random.uniform(0.0, 1.0, size=[1, 1024, 2048, 3]).astype(np.float32) + export(network, Tensor(input_data), file_name='Auto-DeepLab-s_NHWC_BGR', file_format=args.file_format) diff --git a/research/cv/Auto-DeepLab/infer/model/autodeeplab.cfg b/research/cv/Auto-DeepLab/infer/model/autodeeplab.cfg new file mode 100644 index 000000000..34fd7a8ce --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/model/autodeeplab.cfg @@ -0,0 +1,4 @@ +CLASS_NUM=19 +CHECK_MODEL=true +MODEL_TYPE=1 +FRAMEWORK_TYPE=2 diff --git a/research/cv/Auto-DeepLab/infer/mxbase/CMakeLists.txt b/research/cv/Auto-DeepLab/infer/mxbase/CMakeLists.txt new file mode 100644 index 000000000..dd46f9b52 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/mxbase/CMakeLists.txt @@ -0,0 +1,53 @@ +cmake_minimum_required(VERSION 3.14.0) +project(autodeeplab) +set(TARGET autodeeplab) + +add_definitions(-DENABLE_DVPP_INTERFACE) +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +add_definitions(-Dgoogle=mindxsdk_private) +add_compile_options(-std=c++11 -fPIE -fstack-protector-all -fPIC -Wall) +add_link_options(-Wl,-z,relro,-z,now,-z,noexecstack -s -pie) + + +# Check environment variable +if(NOT DEFINED ENV{ASCEND_HOME}) + message(FATAL_ERROR "please define environment variable:ASCEND_HOME") +endif() +if(NOT DEFINED ENV{ASCEND_VERSION}) + message(FATAL_ERROR "please define environment variable:ASCEND_VERSION") +endif() + +if(NOT DEFINED ENV{ARCH_PATTERN}) + message(FATAL_ERROR "please define environment variable:ARCH_PATTERN") +endif() +set(ACL_INC_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/include) +set(ACL_LIB_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/lib64) + +set(MXBASE_ROOT_DIR $ENV{MX_SDK_HOME}) +set(MXBASE_INC ${MXBASE_ROOT_DIR}/include) +set(MXBASE_LIB_DIR ${MXBASE_ROOT_DIR}/lib) +set(MXBASE_POST_LIB_DIR ${MXBASE_ROOT_DIR}/lib/modelpostprocessors) +set(MXBASE_POST_PROCESS_DIR ${MXBASE_ROOT_DIR}/include/MxBase/postprocess/include) + +if(DEFINED ENV{MXSDK_OPENSOURCE_DIR}) + set(OPENSOURCE_DIR $ENV{MXSDK_OPENSOURCE_DIR}) +else() + set(OPENSOURCE_DIR ${MXBASE_ROOT_DIR}/opensource) +endif() + +include_directories(${ACL_INC_DIR}) +include_directories(${OPENSOURCE_DIR}/include) +include_directories(${OPENSOURCE_DIR}/include/opencv4) + +include_directories(${MXBASE_INC}) +include_directories(${MXBASE_POST_PROCESS_DIR}) + +link_directories(${ACL_LIB_DIR}) +link_directories(${OPENSOURCE_DIR}/lib) +link_directories(${MXBASE_LIB_DIR}) +link_directories(${MXBASE_POST_LIB_DIR}) + +add_executable(${TARGET} src/main.cc src/AutoDeepLab.cc) +target_link_libraries(${TARGET} glog cpprest mxbase deeplabv3post opencv_world) + +install(TARGETS ${TARGET} RUNTIME DESTINATION ${PROJECT_SOURCE_DIR}/) diff --git a/research/cv/Auto-DeepLab/infer/mxbase/build.sh b/research/cv/Auto-DeepLab/infer/mxbase/build.sh new file mode 100644 index 000000000..33dd9c7c1 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/mxbase/build.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +function check_env() +{ + # set ASCEND_VERSION to ascend-toolkit/latest when it was not specified by user + if [ ! "${ASCEND_HOME}" ]; then + export ASCEND_HOME=/usr/local/Ascend/ + echo "Set ASCEND_HOME to the default value: ${ASCEND_HOME}" + else + echo "ASCEND_HOME is set to ${ASCEND_HOME} by user" + fi + + if [ ! "${ASCEND_VERSION}" ]; then + export ASCEND_VERSION=nnrt/latest + echo "Set ASCEND_VERSION to the default value: ${ASCEND_VERSION}" + else + echo "ASCEND_VERSION is set to ${ASCEND_VERSION} by user" + fi + + if [ ! "${ARCH_PATTERN}" ]; then + # set ARCH_PATTERN to ./ when it was not specified by user + export ARCH_PATTERN=./ + echo "ARCH_PATTERN is set to the default value: ${ARCH_PATTERN}" + else + echo "ARCH_PATTERN is set to ${ARCH_PATTERN} by user" + fi + + export LD_LIBRARY_PATH=${MX_SDK_HOME}/lib/modelpostprocessors:${LD_LIBRARY_PATH} +} + +function build() +{ + rm -rf build + mkdir -p build + mkdir -p mxbase_result + cd build || exit + + if ! cmake ..; + then + echo "cmake failed." + return 1 + fi + + if ! (make); + then + echo "make failed." + return 1 + fi + + cd - || exit + return 0 +} + +check_env + +if build; +then + echo "INFO: Build successfully." +else + echo "ERROR: Build failed." +fi diff --git a/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.cc b/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.cc new file mode 100644 index 000000000..263a0d6de --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.cc @@ -0,0 +1,294 @@ +/* + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "AutoDeepLab.h" +#include <fstream> +#include <iostream> +#include <algorithm> +#include <vector> +#include <string> +#include <map> +#include "MxBase/DeviceManager/DeviceManager.h" +#include "MxBase/Log/Log.h" + +using std::max; +using std::vector; + +APP_ERROR AutoDeepLab::Init(const InitParam &initParam) { + deviceId_ = initParam.deviceId; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices(); + if (ret != APP_ERR_OK) { + LogError << "Init devices failed, ret=" << ret << "."; + return ret; + } + ret = MxBase::TensorContext::GetInstance()->SetContext(initParam.deviceId); + if (ret != APP_ERR_OK) { + LogError << "Set context failed, ret=" << ret << "."; + return ret; + } + + model_ = std::make_shared<MxBase::ModelInferenceProcessor>(); + ret = model_->Init(initParam.modelPath, modelDesc_); + if (ret != APP_ERR_OK) { + LogError << "ModelInferenceProcessor init failed, ret=" << ret << "."; + return ret; + } + MxBase::ConfigData configData; + configData.SetJsonValue("CLASS_NUM", std::to_string(initParam.classNum)); + configData.SetJsonValue("MODEL_TYPE", std::to_string(initParam.modelType)); + configData.SetJsonValue("CHECK_MODEL", std::to_string(initParam.checkModel)); + configData.SetJsonValue("FRAMEWORK_TYPE", std::to_string(initParam.frameworkType)); + + auto jsonStr = configData.GetCfgJson().serialize(); + std::map<std::string, std::shared_ptr<void>> config; + config["postProcessConfigContent"] = std::make_shared<std::string>(jsonStr); + config["labelPath"] = std::make_shared<std::string>(initParam.labelPath); + post_ = std::make_shared<MxBase::Deeplabv3Post>(); + ret = post_->Init(config); + if (ret != APP_ERR_OK) { + LogError << "Deeplabv3PostProcess init failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::DeInit() { + model_->DeInit(); + post_->DeInit(); + MxBase::DeviceManager::GetInstance()->DestroyDevices(); + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::ReadImage(const std::string &imgPath, cv::Mat &imageMat) { + imageMat = cv::imread(imgPath, cv::IMREAD_COLOR); + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::Normalize(const cv::Mat &srcImageMat, cv::Mat &dstImageMat) { + constexpr size_t ALPHA_AND_BETA_SIZE = 3; + cv::Mat float32Mat; + srcImageMat.convertTo(float32Mat, CV_32FC3); + + std::vector<cv::Mat> tmp; + cv::split(float32Mat, tmp); + + const std::vector<double> mean = {123.675, 116.28, 103.53}; + const std::vector<double> std = {58.395, 57.12, 57.375}; + for (size_t i = 0; i < ALPHA_AND_BETA_SIZE; ++i) { + tmp[i].convertTo(tmp[i], CV_32FC3, 1 / std[i], - mean[i] / std[i]); + } + cv::merge(tmp, dstImageMat); + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::GetResizeInfo(const cv::Mat &srcImageMat, MxBase::ResizedImageInfo &resizedImageInfo) { + resizedImageInfo.heightOriginal = srcImageMat.rows; + resizedImageInfo.heightResize = srcImageMat.rows; + resizedImageInfo.widthOriginal = srcImageMat.cols; + resizedImageInfo.widthResize = srcImageMat.cols; + resizedImageInfo.resizeType = MxBase::RESIZER_MS_KEEP_ASPECT_RATIO; + + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::CVMatToTensorBase(const cv::Mat &imageMat, MxBase::TensorBase &tensorBase) { + const uint32_t dataSize = imageMat.cols * imageMat.rows * imageMat.elemSize(); + MxBase::MemoryData memoryDataDst(dataSize, MxBase::MemoryData::MEMORY_DEVICE, deviceId_); + MxBase::MemoryData memoryDataSrc(imageMat.data, dataSize, MxBase::MemoryData::MEMORY_HOST); + + APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDataDst, memoryDataSrc); + if (ret != APP_ERR_OK) { + LogError << GetError(ret) << "Memory malloc failed."; + return ret; + } + + std::vector<uint32_t> shape = { + static_cast<uint32_t>(imageMat.rows), + static_cast<uint32_t>(imageMat.cols), + static_cast<uint32_t>(imageMat.channels())}; + tensorBase = MxBase::TensorBase(memoryDataDst, false, shape, MxBase::TENSOR_DTYPE_FLOAT32); + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::Inference(const std::vector<MxBase::TensorBase> &inputs, + std::vector<MxBase::TensorBase> &outputs) { + auto dtypes = model_->GetOutputDataType(); + for (size_t i = 0; i < modelDesc_.outputTensors.size(); ++i) { + std::vector<uint32_t> shape = {}; + for (size_t j = 0; j < modelDesc_.outputTensors[i].tensorDims.size(); ++j) { + shape.push_back((uint32_t)modelDesc_.outputTensors[i].tensorDims[j]); + } + MxBase::TensorBase tensor(shape, dtypes[i], MxBase::MemoryData::MEMORY_DEVICE, deviceId_); + APP_ERROR ret = MxBase::TensorBase::TensorBaseMalloc(tensor); + if (ret != APP_ERR_OK) { + LogError << "TensorBaseMalloc failed, ret=" << ret << "."; + return ret; + } + outputs.push_back(tensor); + } + MxBase::DynamicInfo dynamicInfo = {}; + dynamicInfo.dynamicType = MxBase::DynamicType::STATIC_BATCH; + APP_ERROR ret = model_->ModelInference(inputs, outputs, dynamicInfo); + if (ret != APP_ERR_OK) { + LogError << "ModelInference failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::PostProcess(const std::vector<MxBase::TensorBase> &inputs, + std::vector<MxBase::SemanticSegInfo> &segInfo, const std::vector<MxBase::ResizedImageInfo> &resizedInfo) { + APP_ERROR ret = post_->Process(inputs, segInfo, resizedInfo); + if (ret != APP_ERR_OK) { + LogError << "Process failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::Process(const std::string &imgPath) { + cv::Mat imageMat; + APP_ERROR ret = ReadImage(imgPath, imageMat); + if (ret != APP_ERR_OK) { + LogError << "ReadImage failed, ret=" << ret << "."; + return ret; + } + + MxBase::ResizedImageInfo resizedImageInfo; + GetResizeInfo(imageMat, resizedImageInfo); + Normalize(imageMat, imageMat); + + MxBase::TensorBase tensorBase; + ret = CVMatToTensorBase(imageMat, tensorBase); + if (ret != APP_ERR_OK) { + LogError << "CVMatToTensorBase failed, ret=" << ret << "."; + return ret; + } + + std::vector<MxBase::TensorBase> inputs = {}; + std::vector<MxBase::TensorBase> outputs = {}; + inputs.push_back(tensorBase); + ret = Inference(inputs, outputs); + if (ret != APP_ERR_OK) { + LogError << "Inference failed, ret=" << ret << "."; + return ret; + } + + std::vector<MxBase::SemanticSegInfo> semanticSegInfos = {}; + std::vector<MxBase::ResizedImageInfo> resizedImageInfos = {resizedImageInfo}; + ret = PostProcess(outputs, semanticSegInfos, resizedImageInfos); + if (ret != APP_ERR_OK) { + LogError << "PostProcess failed, ret=" << ret << "."; + return ret; + } + + std::string resultPath = imgPath; + size_t pos = resultPath.find_last_of("."); + resultPath.replace(resultPath.begin() + pos, resultPath.end(), "_MxBase_infer.png"); + SaveResultToImage(semanticSegInfos[0], resultPath); + return APP_ERR_OK; +} + +APP_ERROR AutoDeepLab::SaveResultToImage(const MxBase::SemanticSegInfo &segInfo, const std::string &filePath) { + std::string homePath = "./mxbase_result"; + int pos = filePath.rfind('/'); + std::string fileName(filePath, pos + 1); + std::string outFileName = homePath + "/" + fileName; + + cv::Mat imageMat(segInfo.pixels.size(), segInfo.pixels[0].size(), CV_8UC1); + for (size_t x = 0; x < segInfo.pixels.size(); ++x) { + for (size_t y = 0; y < segInfo.pixels[x].size(); ++y) { + uint8_t gray = segInfo.pixels[x][y]; + imageMat.at<uchar>(x, y) = gray; + } + } + + cv::imwrite(outFileName, imageMat); + return APP_ERR_OK; +} + +DIR *OpenDir(const std::string &dirName) { + if (dirName.empty()) { + std::cout << " dirName is null ! " << std::endl; + return nullptr; + } + std::string realPath = RealPath(dirName); + struct stat s; + lstat(realPath.c_str(), &s); + if (!S_ISDIR(s.st_mode)) { + std::cout << "dirName is not a valid directory !" << std::endl; + return nullptr; + } + DIR *dir = opendir(realPath.c_str()); + if (dir == nullptr) { + std::cout << "Can not open dir " << dirName << std::endl; + return nullptr; + } + std::cout << "Successfully opened the dir " << dirName << std::endl; + return dir; +} + + +APP_ERROR GetAllImages(const std::string& dirName, std::vector<std::string> *ImagesPath) { + struct dirent *filename; + DIR *dir = OpenDir(dirName); + if (dir == nullptr) { + return {}; + } + std::vector<std::string> dirs; + while ((filename = readdir(dir)) != nullptr) { + std::string dName = std::string(filename->d_name); + if (dName == "." || dName == "..") { + continue; + } else if (filename->d_type == DT_DIR) { + dirs.emplace_back(std::string(dirName) + "/" + filename->d_name); + } else if (filename->d_type == DT_REG) { + (*ImagesPath).emplace_back(std::string(dirName) + "/" + filename->d_name); + } else { + continue; + } + } + + for (auto d : dirs) { + dir = OpenDir(d); + while ((filename = readdir(dir)) != nullptr) { + std::string dName = std::string(filename->d_name); + if (dName == "." || dName == ".." || filename->d_type != DT_REG) { + continue; + } + (*ImagesPath).emplace_back(std::string(d) + "/" + filename->d_name); + } + } + std::sort((*ImagesPath).begin(), (*ImagesPath).end()); + for (auto &f : (*ImagesPath)) { + std::cout << "image file: " << f << std::endl; + } + return APP_ERR_OK; +} + +std::string RealPath(const std::string &path) { + char realPathMem[PATH_MAX] = {0}; + char *realPathRet = nullptr; + realPathRet = realpath(path.data(), realPathMem); + if (realPathRet == nullptr) { + std::cout << "File: " << path << " is not exist."; + return ""; + } + + std::string realPath(realPathMem); + std::cout << path << " realpath is: " << realPath << std::endl; + return realPath; +} diff --git a/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.h b/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.h new file mode 100644 index 000000000..c33ce6b3e --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/mxbase/src/AutoDeepLab.h @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUTODEEPLAB_H +#define AUTODEEPLAB_H + +#include <sys/stat.h> +#include <dirent.h> +#include <memory> +#include <vector> +#include <string> +#include <opencv2/opencv.hpp> + +#include "MxBase/DvppWrapper/DvppWrapper.h" +#include "MxBase/ModelInfer/ModelInferenceProcessor.h" +#include "SegmentPostProcessors/Deeplabv3Post.h" +#include "MxBase/PostProcessBases/PostProcessDataType.h" +#include "MxBase/Tensor/TensorContext/TensorContext.h" + +struct InitParam { + uint32_t deviceId; + std::string labelPath; + std::string modelPath; + uint32_t classNum; + uint32_t modelType; + bool checkModel; + uint32_t frameworkType; +}; + +class AutoDeepLab { + public: + APP_ERROR Init(const InitParam &initParam); + APP_ERROR DeInit(); + APP_ERROR ReadImage(const std::string &imgPath, cv::Mat &imageMat); + APP_ERROR Normalize(const cv::Mat &srcImageMat, cv::Mat &dstImageMat); + APP_ERROR GetResizeInfo(const cv::Mat &srcImageMat, MxBase::ResizedImageInfo &resizedImageInfo); + APP_ERROR CVMatToTensorBase(const cv::Mat &imageMat, MxBase::TensorBase &tensorBase); + APP_ERROR Inference(const std::vector<MxBase::TensorBase> &inputs, std::vector<MxBase::TensorBase> &outputs); + APP_ERROR PostProcess(const std::vector<MxBase::TensorBase> &inputs, + std::vector<MxBase::SemanticSegInfo> &segInfo, const std::vector<MxBase::ResizedImageInfo> &resizedInfo); + APP_ERROR Process(const std::string &imgPath); + APP_ERROR SaveResultToImage(const MxBase::SemanticSegInfo &segInfo, const std::string &filePath); + + private: + std::shared_ptr<MxBase::ModelInferenceProcessor> model_; + std::shared_ptr<MxBase::Deeplabv3Post> post_; + MxBase::ModelDesc modelDesc_; + uint32_t deviceId_ = 0; +}; + +APP_ERROR GetAllImages(const std::string &dirName, std::vector<std::string> *ImagesPath); +DIR *OpenDir(const std::string &dirName); +std::string RealPath(const std::string &path); +#endif diff --git a/research/cv/Auto-DeepLab/infer/mxbase/src/main.cc b/research/cv/Auto-DeepLab/infer/mxbase/src/main.cc new file mode 100644 index 000000000..2f2b352f9 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/mxbase/src/main.cc @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "AutoDeepLab.h" +#include "MxBase/Log/Log.h" + +namespace { + const uint32_t CLASS_NUM = 19; + const uint32_t MODEL_TYPE = 1; + const uint32_t FRAMEWORK_TYPE = 2; +} + +void ShowUsage() { + LogWarn << "Usage : ./build/autodeeplab <--image or --dir> [Option]" << std::endl; + LogWarn << "Options :" << std::endl; + LogWarn << " --image infer_image_path the path of single infer image, such as " + "./build/autodeeplab --image /PATH/TO/cityscapes/test.jpg." << std::endl; + LogWarn << " --dir infer_image_dir the dir of batch infer images, such as " + "./build/autodeeplab --dir /PATH/TO/cityscapes/leftImg8bit/val." << std::endl; + return; +} + +int main(int argc, char *argv[]) { + if (argc < 3) { + LogWarn << "Please use as follows." << std::endl; + ShowUsage(); + return APP_ERR_OK; + } + + std::string option = argv[1]; + std::string imgPath = argv[2]; + if (option != "--image" && option != "--dir") { + LogInfo << "Please use as follows." << std::endl; + ShowUsage(); + return APP_ERR_OK; + } + + InitParam initParam = {}; + initParam.deviceId = 0; + initParam.classNum = CLASS_NUM; + initParam.modelType = MODEL_TYPE; + initParam.labelPath = "../model/autodeeplab.names"; + initParam.modelPath = "../model/Auto-DeepLab-s_NHWC_BGR.om"; + initParam.checkModel = true; + initParam.frameworkType = FRAMEWORK_TYPE; + + AutoDeepLab model; + APP_ERROR ret = model.Init(initParam); + if (ret != APP_ERR_OK) { + LogError << "AutoDeepLab init failed, ret=" << ret << "."; + return ret; + } + + std::vector<std::string> imagesPath; + if (option == "--image") { + imagesPath.emplace_back(imgPath); + } else { + ret = GetAllImages(imgPath, &imagesPath); + } + + if (ret != APP_ERR_OK) { + LogError << "read file failed, ret=" << ret << "."; + return ret; + } + LogInfo << "read file success."; + + for (auto path : imagesPath) { + LogInfo << "read image path " << path; + ret = model.Process(path); + if (ret != APP_ERR_OK) { + LogError << "AutoDeepLab process failed, ret=" << ret << "."; + model.DeInit(); + return ret; + } + } + + model.DeInit(); + return APP_ERR_OK; +} diff --git a/research/cv/Auto-DeepLab/infer/sdk/__init__.py b/research/cv/Auto-DeepLab/infer/sdk/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/research/cv/Auto-DeepLab/infer/sdk/autodeeplab_opencv.pipeline b/research/cv/Auto-DeepLab/infer/sdk/autodeeplab_opencv.pipeline new file mode 100644 index 000000000..9d8fbd849 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/sdk/autodeeplab_opencv.pipeline @@ -0,0 +1,63 @@ +{ + "segmentation": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_imagedecoder0" + }, + "mxpi_imagedecoder0": { + "props": { + "handleMethod": "opencv", + "outputDataFormat": "RGB" + }, + "factory": "mxpi_imagedecoder", + "next": "opencv_normalize0" + }, + "opencv_normalize0": { + "props": { + "alpha": "103.53, 116.28, 123.675", + "beta": "57.375, 57.12, 58.395", + "dataType": "FLOAT32", + "format": "BGR888" + }, + "factory": "mxpi_imagenormalize", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "opencv_normalize0", + "modelPath": "../model/Auto-DeepLab-s_NHWC_BGR.om" + }, + "factory": "mxpi_modelinfer", + "next": "mxpi_semanticsegpostprocessor0" + }, + "mxpi_semanticsegpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "../model/autodeeplab.cfg", + "labelPath": "../model/autodeeplab.names", + "postProcessLibPath": "libdeeplabv3post.so" + }, + "factory": "mxpi_semanticsegpostprocessor", + "next": "mxpi_dataserialize0" + }, + "mxpi_dataserialize0": { + "props": { + "outputDataKeys": "mxpi_semanticsegpostprocessor0" + }, + "factory": "mxpi_dataserialize", + "next": "appsink0" + }, + "appsink0": { + "props": { + "blocksize": "4096000" + }, + "factory": "appsink" + } + } +} diff --git a/research/cv/Auto-DeepLab/infer/sdk/main.py b/research/cv/Auto-DeepLab/infer/sdk/main.py new file mode 100644 index 000000000..a3fa0799c --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/sdk/main.py @@ -0,0 +1,117 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""main.py""" +import argparse +import base64 +import json +import os + +import numpy as np + +from StreamManagerApi import MxDataInput +from StreamManagerApi import StreamManagerApi + +import utils + + +def sdk_args(): + """sdk_args""" + parser = argparse.ArgumentParser("Auto-DeepLab SDK Inference") + parser.add_argument('--pipeline', type=str, default='', help='path to pipeline file') + # val data + parser.add_argument('--data_root', type=str, default='', + help='root path of val data') + parser.add_argument('--result_path', type=str, default='', + help='output_path') + parser.add_argument('--num_classes', type=int, default=19, + help='number of classes') + args = parser.parse_args() + return args + + +def _init_stream(pipeline_path): + """_init_stream""" + stream_manager_api = StreamManagerApi() + ret = stream_manager_api.InitManager() + if ret != 0: + raise RuntimeError(f"Failed to init stream manager, ret={ret}") + + with open(pipeline_path, 'rb') as f: + pipeline_str = f.read() + + ret = stream_manager_api.CreateMultipleStreams(pipeline_str) + if ret != 0: + raise RuntimeError(f"Failed to create stream, ret={ret}") + return stream_manager_api + + +def _do_infer(stream_manager_api, data_input): + """_do_infer""" + stream_name = b'segmentation' + unique_id = stream_manager_api.SendDataWithUniqueId( + stream_name, 0, data_input) + if unique_id < 0: + raise RuntimeError("Failed to send data to stream.") + + timeout = 7000 + infer_result = stream_manager_api.GetResultWithUniqueId( + stream_name, unique_id, timeout) + if infer_result.errorCode != 0: + raise RuntimeError( + "GetResultWithUniqueId error, errorCode=%d, errorMsg=%s" % ( + infer_result.errorCode, infer_result.data.decode())) + + load_dict = json.loads(infer_result.data.decode()) + image_mask = load_dict["MxpiImageMask"][0] + data_str = base64.b64decode(image_mask['dataStr']) + shape = image_mask['shape'] + return np.frombuffer(data_str, dtype=np.uint8).reshape(shape) + +def main(): + """main""" + args = sdk_args() + + # init stream manager + stream_manager_api = _init_stream(args.pipeline) + if not stream_manager_api: + exit(1) + + os.makedirs(args.result_path, exist_ok=True) + data_input = MxDataInput() + dataset = utils.CityscapesDataLoader(args.data_root) + hist = np.zeros((args.num_classes, args.num_classes)) + for data_item in dataset: + print(f"start infer {data_item['file_name']}") + data_input.data = data_item['img'] + gtFine = utils.encode_segmap(data_item['gt'], 255) + pred = _do_infer(stream_manager_api, data_input) + + hist += utils.fast_hist(pred.copy().flatten(), gtFine.flatten(), args.num_classes) + color_mask_res = utils.label_to_color_image(pred) + + folder_path = os.path.join(args.result_path, data_item['file_path'].split(os.sep)[-1]) + if not os.path.isdir(folder_path): + os.makedirs(folder_path) + result_file = os.path.join(folder_path, data_item['file_name'].replace('leftImg8bit', 'pred_color')) + color_mask_res.save(result_file) + iou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist) + 1e-10) + print("per-class IOU", iou) + print("mean IOU", round(np.nanmean(iou) * 100, 2)) + + stream_manager_api.DestroyAllStreams() + + +if __name__ == "__main__": + main() diff --git a/research/cv/Auto-DeepLab/infer/sdk/utils.py b/research/cv/Auto-DeepLab/infer/sdk/utils.py new file mode 100644 index 000000000..4288fd6a9 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/sdk/utils.py @@ -0,0 +1,134 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""sdk utils""" +import os +import numpy as np +from PIL import Image + + +cityspallete = [ + 128, 64, 128, + 244, 35, 232, + 70, 70, 70, + 102, 102, 156, + 190, 153, 153, + 153, 153, 153, + 250, 170, 30, + 220, 220, 0, + 107, 142, 35, + 152, 251, 152, + 0, 130, 180, + 220, 20, 60, + 255, 0, 0, + 0, 0, 142, + 0, 0, 70, + 0, 60, 100, + 0, 80, 100, + 0, 0, 230, + 119, 11, 32, + ] + + +def fast_hist(predict, label, n): + """ + fast_hist + inputs: + - predict (ndarray) + - label (ndarray) + - n (int) - number of classes + outputs: + - fast histogram + """ + k = (label >= 0) & (label < n) + return np.bincount(n * label[k].astype(np.int32) + predict[k], minlength=n ** 2).reshape(n, n) + + +def encode_segmap(lbl, ignore_label): + """encode segmap""" + mask = np.uint8(lbl) + + num_classes = 19 + void_classes = [0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1] + valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33] + + class_map = dict(zip(valid_classes, range(num_classes))) + for _voidc in void_classes: + mask[mask == _voidc] = ignore_label + for _validc in valid_classes: + mask[mask == _validc] = class_map[_validc] + + return mask + + +def label_to_color_image(npimg) -> Image: + """label_to_color_image""" + img = Image.fromarray(npimg.astype('uint8'), "P") + img.putpalette(cityspallete) + out_img = img.convert('RGB') + return out_img + + +class CityscapesDataLoader(): + """CityscapesDataLoader""" + def __init__(self, data_root, split='val', ignore_label=255): + """__init__""" + super(CityscapesDataLoader, self).__init__() + images_base = os.path.join(data_root, 'leftImg8bit', split) + annotations_base = os.path.join(data_root, 'gtFine', split) + self.img_id = [] + self.img_path = [] + self.img_name = [] + self.images = [] + self.gtFiles = [] + for root, _, files in os.walk(images_base): + for filename in files: + if filename.endswith('.png'): + self.img_path.append(root) + self.img_name.append(filename) + folder_name = root.split(os.sep)[-1] + gtFine_name = filename.replace('leftImg8bit', 'gtFine_labelIds') + _img = os.path.join(root, filename) + _gt = os.path.join(annotations_base, folder_name, gtFine_name) + self.images.append(_img) + self.gtFiles.append(_gt) + self.len = len(self.images) + self.cur_index = 0 + print(f"Found {self.len} images") + + def __len__(self): + """__len__""" + return self.len + + def __iter__(self): + """__iter__""" + return self + + def __next__(self) -> dict: + """__next__""" + if self.cur_index == self.len: + raise StopIteration() + + with open(self.images[self.cur_index], 'rb') as f: + image = f.read() + gtFine = Image.open(self.gtFiles[self.cur_index]) + gtFine = np.array(gtFine).astype(np.uint8) + dataItem = { + 'file_path': self.img_path[self.cur_index], + 'file_name': self.img_name[self.cur_index], + 'img': image, + 'gt': gtFine, + } + self.cur_index += 1 + return dataItem diff --git a/research/cv/Auto-DeepLab/infer/util/__init__.py b/research/cv/Auto-DeepLab/infer/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/research/cv/Auto-DeepLab/infer/util/mindx_config.py b/research/cv/Auto-DeepLab/infer/util/mindx_config.py new file mode 100644 index 000000000..3a65385ca --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/util/mindx_config.py @@ -0,0 +1,90 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""mindx_config""" +import argparse +import ast + + +def obtain_autodeeplab_args(): + """obtain_autodeeplab_args""" + parser = argparse.ArgumentParser(description="MindSpore Auto-deeplab Training") + parser.add_argument('--epochs', type=int, default=4000, help='num of training epochs') + parser.add_argument('--batch_size', type=int, default=16, help='batch size') + parser.add_argument('--seed', type=int, default=0, metavar='S', help='random seed (default: 0)') + parser.add_argument('--device_target', type=str, default='Ascend', help='"Ascend", "GPU"') + parser.add_argument('--modelArts', type=ast.literal_eval, default=True, + help='train on modelArts or not, default: True') + parser.add_argument('--ms_infer', type=ast.literal_eval, default=False) + parser.add_argument('--eval_flip', type=ast.literal_eval, default=True) + + # learning rate & optimizer + parser.add_argument('--base_lr', type=float, default=0.05, help='base learning rate') + parser.add_argument('--warmup_start_lr', type=float, default=5e-6, help='warm up learning rate') + parser.add_argument('--warmup_iters', type=int, default=1000) + parser.add_argument('--min_lr', type=float, default=None) + parser.add_argument('--weight_decay', type=float, default=1e-4) + + # dataset & preprocess + parser.add_argument('--num_classes', type=int, default=19) + parser.add_argument('--split', type=str, default='train') + parser.add_argument('--ignore_label', type=int, default=255) + parser.add_argument('--crop_size', type=int, default=769, help='image crop size') + parser.add_argument('--workers', type=int, default=4, help='number of data loading workers') + + # architecture + parser.add_argument('--filter_multiplier', type=int, default=20) + parser.add_argument('--parallel', type=ast.literal_eval, default=True) + parser.add_argument('--block_multiplier', type=int, default=5) + parser.add_argument('--use_ABN', type=ast.literal_eval, default=True, help='whether use ABN') + parser.add_argument('--affine', type=ast.literal_eval, default=True, help='whether use affine in BN') + parser.add_argument('--drop_path_keep_prob', type=float, default=1.0, + help='drop_path_keep_prob, (0.0, 1.0]') + parser.add_argument('--net_arch', type=str, default=None) + parser.add_argument('--cell_arch', type=str, default=None) + parser.add_argument('--bn_momentum', type=float, default=0.995) + parser.add_argument('--bn_eps', type=float, default=1e-5) + + # loss + parser.add_argument('--criterion', type=str, default='ohemce', help="'ce', 'ohemce'") + parser.add_argument('--ohem_thresh', type=float, default=0.7, + help='top k present pixels used to compute loss') + parser.add_argument('--initial-fm', default=None, type=int) + + # resume + parser.add_argument('--ckpt_name', type=str, default=None) + parser.add_argument('--start_epoch', type=int, default=0) + + # training path + parser.add_argument('--data_path', type=str, default='', help="path to dataset") + parser.add_argument('--out_path', type=str, default='', help="path to store output") + + # export format + parser.add_argument('--infer_flip', type=ast.literal_eval, default=True, + help="Use flip during infer or not. default: True") + parser.add_argument('--input_format', type=str, default='NHWC', help="input format of mindx sdk & mxbase") + parser.add_argument('--file_format', type=str, default='AIR', help="MINDIR or AIR") + parser.add_argument('--filter_weight', type=ast.literal_eval, default=False, + help="unload final layer default: False") + + # ModelArts + parser.add_argument('--data_url', type=str, default='') + parser.add_argument('--train_url', type=str, default='') + parser.add_argument('--ckpt_url', type=str, default='') + parser.add_argument('--img_url', type=str, default='') + + parser.add_argument('--save_epochs', type=int, default=100) + + args = parser.parse_args() + return args diff --git a/research/cv/Auto-DeepLab/infer/util/mindx_utils.py b/research/cv/Auto-DeepLab/infer/util/mindx_utils.py new file mode 100644 index 000000000..53997db37 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/util/mindx_utils.py @@ -0,0 +1,48 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""mindx_utils""" +import sys +import os + +import mindspore.nn as nn +import mindspore.numpy as msnp +import mindspore.ops as ops + +from src.utils.utils import BuildEvalNetwork + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + + +class InferWithFlipNetwork(nn.Cell): + """InferWithFlipNetwork""" + def __init__(self, network, flip=True, input_format="NCHW"): + super(InferWithFlipNetwork, self).__init__() + self.eval_net = BuildEvalNetwork(network) + self.transpose = ops.Transpose() + self.flip = flip + self.format = input_format + + def construct(self, input_data): + """construct""" + if self.format == "NHWC": + input_data = self.transpose(input_data, (0, 3, 1, 2)) + output = self.eval_net(input_data) + + if self.flip: + flip_input = msnp.flip(input_data, 3) + flip_output = self.eval_net(flip_input) + output += msnp.flip(flip_output, 3) + + return output diff --git a/research/cv/Auto-DeepLab/infer/util/mxbase_postprocess.py b/research/cv/Auto-DeepLab/infer/util/mxbase_postprocess.py new file mode 100644 index 000000000..902e0d089 --- /dev/null +++ b/research/cv/Auto-DeepLab/infer/util/mxbase_postprocess.py @@ -0,0 +1,148 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluate mIOU and Pixel accuracy""" +import os +import argparse +import ast + +import cv2 +from PIL import Image +import numpy as np + + +def fast_hist(predict, label, n): + """ + fast_hist + inputs: + - predict (ndarray) + - label (ndarray) + - n (int) - number of classes + outputs: + - fast histogram + """ + k = (label >= 0) & (label < n) + return np.bincount(n * label[k].astype(np.int32) + predict[k], minlength=n ** 2).reshape(n, n) + +def encode_segmap(lbl, ignore_label): + """encode segmap""" + mask = np.uint8(lbl) + + num_classes = 19 + void_classes = [0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1] + valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33] + + class_map = dict(zip(valid_classes, range(num_classes))) + for _voidc in void_classes: + mask[mask == _voidc] = ignore_label + for _validc in valid_classes: + mask[mask == _validc] = class_map[_validc] + + return mask + +def decode_segmap(pred): + """decode_segmap""" + mask = np.uint8(pred) + + num_classes = 19 + valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33] + rank_classes = range(num_classes) + + class_map = dict(zip(rank_classes, valid_classes)) + + for _rank in rank_classes: + mask[mask == _rank] = class_map[_rank] + + return mask + +def get_color(npimg): + """get_color""" + cityspallete = [ + 128, 64, 128, + 244, 35, 232, + 70, 70, 70, + 102, 102, 156, + 190, 153, 153, + 153, 153, 153, + 250, 170, 30, + 220, 220, 0, + 107, 142, 35, + 152, 251, 152, + 0, 130, 180, + 220, 20, 60, + 255, 0, 0, + 0, 0, 142, + 0, 0, 70, + 0, 60, 100, + 0, 80, 100, + 0, 0, 230, + 119, 11, 32, + ] + img = Image.fromarray(npimg.astype('uint8'), "P") + img.putpalette(cityspallete) + out_img = np.array(img.convert('RGB')) + return out_img + +def infer(args): + """infer""" + images_base = os.path.join(args.dataset_path, 'leftImg8bit/val') + annotations_base = os.path.join(args.dataset_path, 'gtFine/val') + hist = np.zeros((args.num_classes, args.num_classes)) + for root, _, files in os.walk(images_base): + for filename in files: + if filename.endswith('.png'): + print("start post ", filename) + file_name = filename.split('.')[0] + + pred_file = os.path.join(args.result_path, file_name + "_MxBase_infer.png") + + pred = np.array(Image.open(pred_file), dtype=np.uint8) + folder_name = root.split(os.sep)[-1] + + if args.cal_acc: + gtFine_name = filename.replace('leftImg8bit', 'gtFine_labelIds') + label_file = os.path.join(annotations_base, folder_name, gtFine_name) + label = np.array(cv2.imread(label_file, cv2.IMREAD_GRAYSCALE), np.uint8) + label = encode_segmap(label, 255) + hist = hist + fast_hist(pred.copy().flatten(), label.flatten(), args.num_classes) + + if args.save_img: + + # colorful segmentation image + colorImg_name = filename.replace('leftImg8bit', 'predImg_colorful') + colorImg_root = args.output_path + colorImg_root = os.path.join(colorImg_root.replace('output', 'output_img'), folder_name) + colorImg_file = os.path.join(colorImg_root, colorImg_name) + if not os.path.isdir(colorImg_root): + os.makedirs(colorImg_root) + color_pred = get_color(pred.copy()) + color_pred = cv2.cvtColor(np.asarray(color_pred), cv2.COLOR_RGB2BGR) + cv2.imwrite(colorImg_file, color_pred, [cv2.IMWRITE_PNG_COMPRESSION]) + + if args.cal_acc: + miou = np.diag(hist) / (hist.sum(0) + hist.sum(1) - np.diag(hist) + 1e-10) + miou = round(np.nanmean(miou) * 100, 2) + print("mIOU = ", miou, "%") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Auto-DeepLab Inference post-process") + parser.add_argument("--dataset_path", type=str, default="", help="dataset path for evaluation") + parser.add_argument("--num_classes", type=int, default=19) + parser.add_argument("--result_path", type=str, default="", help="Pred png file path.") + parser.add_argument("--output_path", type=str, default="", help="Output path.") + parser.add_argument("--save_img", type=ast.literal_eval, default=True, help="Whether save pics after inference.") + parser.add_argument("--cal_acc", type=ast.literal_eval, default=True, help="Calculate mIOU or not.") + Args = parser.parse_args() + infer(Args) diff --git a/research/cv/Auto-DeepLab/modelarts/train_start.py b/research/cv/Auto-DeepLab/modelarts/train_start.py new file mode 100644 index 000000000..a79cca5b4 --- /dev/null +++ b/research/cv/Auto-DeepLab/modelarts/train_start.py @@ -0,0 +1,203 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +"""Train Auto-DeepLab""" +import sys +import os + +import math +import numpy as np + +import moxing as mox + +import mindspore +import mindspore.nn as nn +from mindspore import context +from mindspore.context import ParallelMode +from mindspore import Model +from mindspore import load_checkpoint, load_param_into_net, export +from mindspore.train.callback import TimeMonitor, LossMonitor, ModelCheckpoint, CheckpointConfig +from mindspore.train.loss_scale_manager import DynamicLossScaleManager +from mindspore.communication import init + + +device_id = int(os.getenv('DEVICE_ID')) +device_num = int(os.getenv('RANK_SIZE')) +local_data_url = "/cache/data" +local_train_url = "/cache/train" + + +def get_last_ckpt(ckpt_dir): + """get_last_ckpt""" + ckpt_files = [ckpt_file for ckpt_file in os.listdir(ckpt_dir) + if ckpt_file.endswith('.ckpt')] + if not ckpt_files: + print("No ckpt file found.") + return None + + return os.path.join(ckpt_dir, sorted(ckpt_files)[-1]) + + +def export_air(args, ckpt_dir): + """export_air""" + args.parallel = False + ckpt_file = get_last_ckpt(ckpt_dir) + air_name = os.path.join(ckpt_dir, 'Auto-DeepLab-s_NHWC_BGR') + if not ckpt_file: + return 1 + + # net + autodeeplab = AutoDeepLab(args) + + # load checkpoint + print('start export ', ckpt_file) + param_dict = load_checkpoint(ckpt_file) + + # load the parameter into net + load_param_into_net(autodeeplab, param_dict) + network = InferWithFlipNetwork(autodeeplab, flip=args.infer_flip, input_format=args.input_format) + + input_data = np.random.uniform(0.0, 1.0, size=[1, 1024, 2048, 3]).astype(np.float32) + export(network, mindspore.Tensor(input_data), file_name=air_name, file_format=args.file_format) + + return 0 + + +def filter_weight(param_dict): + """filter_weight""" + for key in list(param_dict.keys()): + if 'decoder.last_conv.6' in key: + print('filter {}'.format(key)) + del param_dict[key] + return param_dict + + +def train(): + """train""" + args = obtain_autodeeplab_args() + prepare_seed(args.seed) + + context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target=args.device_target, + device_id=int(os.getenv('DEVICE_ID'))) + ckpt_file = args.ckpt_name + + init() + context.set_auto_parallel_context(device_num=device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + gradients_mean=True) + shard_id = device_id + num_shards = device_num + device_data_url = os.path.join(local_data_url, "device{0}".format(device_id)) + device_train_url = os.path.join(local_train_url, "device{0}".format(device_id)) + local_train_file = os.path.join(device_data_url, 'cityscapes_train.mindrecord') + if args.ckpt_name is not None: + ckpt_file = os.path.join(device_data_url, args.ckpt_name) + + mox.file.make_dirs(local_data_url) + mox.file.make_dirs(local_train_url) + mox.file.make_dirs(device_data_url) + mox.file.make_dirs(device_train_url) + mox.file.copy_parallel(src_url=args.data_url, dst_url=device_data_url) + + # define dataset + batch_size = int(args.batch_size // device_num) if args.parallel else args.batch_size + crop = args.crop_size + + train_ds = CityScapesDataset(local_train_file, 'train', args.ignore_label, (crop, crop), + num_shards, shard_id, shuffle=True) + train_ds = train_ds.batch(batch_size=batch_size, drop_remainder=True) + + # counting steps + iter_per_epoch = train_ds.get_dataset_size() + total_iters = iter_per_epoch * args.epochs + args.total_iters = total_iters + + # define model + print(args) + net = AutoDeepLab(args) + + # loading checkpoints + if args.ckpt_name is not None: + print('=> loading checkpoint {0}'.format(args.ckpt_name)) + param_dict = load_checkpoint(ckpt_file) + param_dict = filter_weight(param_dict) if args.filter_weight else param_dict + load_param_into_net(net, param_dict) + print('=> successfully loaded checkpoint {0}'.format(args.ckpt_name)) + + # loss + if args.criterion == 'ohemce': + args.thresh = -math.log(args.ohem_thresh) + args.n_min = int((batch_size * args.crop_size * args.crop_size) // 16) + criterion = build_criterion(args) + + # learning rate + min_lr = args.min_lr if args.min_lr is not None else 0.0 + lr = warmup_poly_lr(args.warmup_start_lr, args.base_lr, min_lr, args.warmup_iters, total_iters) + + # recover training + current_iter = args.start_epoch * iter_per_epoch + epochs = args.epochs - args.start_epoch + lr = lr[current_iter::] + lr = mindspore.Tensor(np.array(lr).astype(np.float32)) + + # optimizer + bias_params = list(filter(lambda x: ('gamma' in x.name) or ('beta' in x.name), net.trainable_params())) + no_bias_params = list(filter(lambda x: ('gamma' not in x.name) and ('beta' not in x.name), net.trainable_params())) + group_params = [{'params': bias_params, 'weight_decay': 0.0}, + {'params': no_bias_params, 'weight_decay': args.weight_decay}, + {'order_params': net.trainable_params()}] + optimizer = nn.Momentum(params=group_params, + learning_rate=lr, + momentum=0.9, + weight_decay=args.weight_decay) + loss_scale_manager = DynamicLossScaleManager() + + # model + model = Model(net, criterion, loss_scale_manager=loss_scale_manager, optimizer=optimizer) + + # callback for loss & time cost + loss_cb = LossMonitor() + time_cb = TimeMonitor(data_size=iter_per_epoch) + cbs = [loss_cb, time_cb] + + # callback for saving ckpt + config_ckpt = CheckpointConfig(save_checkpoint_steps=args.save_epochs * iter_per_epoch, keep_checkpoint_max=40) + ckpt_cb = ModelCheckpoint(prefix='autodeeplab-paral', directory=device_train_url, config=config_ckpt) + cbs += [ckpt_cb] + + model.train(epochs, train_ds, callbacks=cbs, dataset_sink_mode=True) + + export_air(args, device_train_url) + + if args.modelArts and device_id == 0: + mox.file.copy_parallel(src_url="/cache/train", dst_url=args.train_url) + mox.file.copy_parallel(src_url='/tmp', dst_url=args.train_url) + + return 0 + + +if __name__ == "__main__": + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + + from src.utils.loss import build_criterion + from src.core.model import AutoDeepLab + from src.utils.cityscapes import CityScapesDataset + from src.utils.dynamic_lr import warmup_poly_lr + from src.utils.utils import prepare_seed + from infer.util.mindx_config import obtain_autodeeplab_args + from infer.util.mindx_utils import InferWithFlipNetwork + + train() diff --git a/research/cv/Auto-DeepLab/scripts/docker_start.sh b/research/cv/Auto-DeepLab/scripts/docker_start.sh new file mode 100644 index 000000000..85554ca44 --- /dev/null +++ b/research/cv/Auto-DeepLab/scripts/docker_start.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== + +docker_image=$1 \ +data_dir=$2 \ +model_dir=$3 \ + +docker run -it -u root --ipc=host \ + --device=/dev/davinci0 \ + --device=/dev/davinci1 \ + --device=/dev/davinci2 \ + --device=/dev/davinci3 \ + --device=/dev/davinci4 \ + --device=/dev/davinci5 \ + --device=/dev/davinci6 \ + --device=/dev/davinci7 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm --device=/dev/hisi_hdc \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v ${model_dir}:${model_dir} \ + -v ${data_dir}:${data_dir} \ + -v /root/ascend/log:/root/ascend/log ${docker_image} /bin/bash -- GitLab