Skip to content
Snippets Groups Projects
Commit e2ae2e6e authored by i-robot's avatar i-robot Committed by Gitee
Browse files

!1051 [东北大学][高校贡献][Mindspore][SqueezeNetResidual]-高性能预训练模型提交+功能

Merge pull request !1051 from 赵立阳/master
parents 8f8241d9 a735dfba
No related branches found
No related tags found
No related merge requests found
Showing
with 2187 additions and 0 deletions
......@@ -41,3 +41,7 @@
"models/research/cv/inception_resnet_v2/infer/mxbase/InceptionResnetV2Classify.h" "runtime/references"
"models/official/cv/LearningToSeeInTheDark/infer/mxbase/src/LearningToSeeInTheDark.h" "runtime/references"
"models/official/cv/squeezenet/infer/mxbase/src/squeezenetResidual.h" "runtime/references"
"models/official/cv/squeezenet/infer/mxbase/src/main.cpp" "runtime/references"
aipp_op{
aipp_mode : static
input_format : RGB888_U8
rbuv_swap_switch : true
mean_chn_0 : 0
mean_chn_1 : 0
mean_chn_2 : 0
min_chn_0 : 123.675
min_chn_1 : 116.28
min_chn_2 : 103.53
var_reci_chn_0 : 0.0171247538316637
var_reci_chn_1 : 0.0175070028011204
var_reci_chn_2 : 0.0174291938997821
}
#!/bin/bash
if [ $# != 2 ]
then
echo "Wrong parameter format."
echo "Usage:"
echo " bash $0 [INPUT_AIR_PATH] [OUTPUT_OM_PATH_NAME]"
echo "Example"
echo " bash convert_om.sh xxx.air xx"
exit 1
fi
input_air_path=$1
output_om_path=$2
export install_path=/usr/local/Ascend
export ASCEND_ATC_PATH=${install_path}/atc
export LD_LIBRARY_PATH=${install_path}/atc/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/python3.7.5/bin:${install_path}/atc/ccec_compiler/bin:${install_path}/atc/bin:$PATH
export PYTHONPATH=${install_path}/atc/python/site-packages:${install_path}/atc/python/site-packages/auto_tune.egg/auto_tune:${install_path}/atc/python/site-packages/schedule_search.egg:$PYTHONPATH
export ASCEND_OPP_PATH=${install_path}/opp
export ASCEND_SLOG_PRINT_TO_STDOUT=1
echo "Input AIR file path: ${input_air_path}"
echo "Output OM file path: ${output_om_path}"
atc --framework=1 \
--input_format=NCHW \
--model="${input_air_path}" \
--output="${output_om_path}" \
--insert_op_conf=./aipp.config \
--soc_version=Ascend310 \
--output_type=FP32 \
--op_select_implmode=high_precision
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import sys
def convert(input_path, output_path):
data = []
with open(input_path, 'r') as f:
for line in f.readlines():
data.append(line[10:-1])
fp = open(output_path, 'a', encoding='utf-8')
for name in data:
fp.write(name+'\n')
if __name__ == '__main__':
if len(sys.argv) != 3:
print("Wrong parameter format.")
print("Usage:")
print(" python3 convert.py [SYN__PATH] [NAMES_OUTPUT_PATH_NAME]")
sys.exit()
input_p = sys.argv[1]
output_p = sys.argv[2]
convert(input_p, output_p)
{
"im_squeezenetResidual": {
"stream_config": {
"deviceId": "0"
},
"appsrc0": {
"propc": {
"blocksize": "409600"
},
"factory": "appsrc",
"next": "mxpi_imagedecoder0"
},
"mxpi_imagedecoder0": {
"props": {
"handleMethod": "opencv"
},
"factory": "mxpi_imagedecoder",
"next": "mxpi_imageresize0"
},
"mxpi_imageresize0": {
"props": {
"handleMethod": "opencv",
"resizeType": "Resizer_Stretch",
"resizeHeight": "256",
"resizeWidth": "256"
},
"factory": "mxpi_imageresize",
"next": "mxpi_opencvcentercrop0"
},
"mxpi_opencvcentercrop0": {
"props": {
"dataSource": "mxpi_imageresize0",
"cropHeight": "227",
"cropWidth": "227"
},
"factory": "mxpi_opencvcentercrop",
"next": "mxpi_tensorinfer0"
},
"mxpi_tensorinfer0": {
"props": {
"dataSource": "mxpi_opencvcentercrop0",
"modelPath": "../data/model/squeezenetResidual.om",
"waitingTime": "2000",
"outputDeviceId": "-1"
},
"factory": "mxpi_tensorinfer",
"next": "mxpi_classpostprocessor0"
},
"mxpi_classpostprocessor0": {
"props": {
"dataSource": "mxpi_tensorinfer0",
"postProcessConfigPath": "../data/config/squeezenetResidual.cfg",
"labelPath": "../data/config/imagenet1000_claidx_to_labels.names",
"postProcessLibPath": "/usr/local/sdk_home/mxManufacture/lib/modelpostprocessors/libresnet50postprocess.so"
},
"factory": "mxpi_classpostprocessor",
"next": "mxpi_dataserialize0"
},
"mxpi_dataserialize0": {
"props": {
"outputDataKeys": "mxpi_classpostprocessor0"
},
"factory": "mxpi_dataserialize",
"next": "appsink0"
},
"appsink0": {
"props": {
"blocksize": "4096000"
},
"factory": "appsink"
}
}
}
CLASS_NUM=1000
SOFTMAX=false
TOP_K=5
#!/usr/bin/env bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
docker_image=$1
model_dir=$2
data_dir=$3
function show_help() {
echo "Usage: docker_start.sh docker_image data_dir"
}
function param_check() {
if [ -z "${docker_image}" ]; then
echo "please input docker_image"
show_help
exit 1
fi
if [ -z "${model_dir}" ]; then
echo "please input model_dir"
show_help
exit 1
fi
if [ -z "${data_dir}" ]; then
echo "please input data_dir"
show_help
exit 1
fi
}
param_check
docker run -it \
--device=/dev/davinci0 \
--device=/dev/davinci_manager \
--device=/dev/devmm_svm \
--device=/dev/hisi_hdc \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v ${model_dir}:${model_dir} \
-v ${data_dir}:${data_dir} \
${docker_image} \
/bin/bash
cmake_minimum_required(VERSION 3.14.0)
project(squeezenetResidual)
set(TARGET squeezenetResidual)
add_definitions(-DENABLE_DVPP_INTERFACE)
add_compile_options(-std=c++11 -fPIE -fstack-protector-all -fPIC -Wall
-Dgoogle=mindxsdk_private -D_GLIBCXX_USE_CXX11_ABI=0)
add_link_options(-Wl,-z,relro,-z,now,-z,noexecstack -pie)
# Check environment variable
if(NOT DEFINED ENV{MX_SDK_HOME})
message(FATAL_ERROR "please define environment variable:MX_SDK_HOME")
endif()
if(NOT DEFINED ENV{ASCEND_HOME})
message(FATAL_ERROR "please define environment variable:ASCEND_HOME")
endif()
if(NOT DEFINED ENV{ASCEND_VERSION})
message(WARNING "please define environment variable:ASCEND_VERSION")
endif()
if(NOT DEFINED ENV{ARCH_PATTERN})
message(WARNING "please define environment variable:ARCH_PATTERN")
endif()
set(ACL_INC_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/include)
set(ACL_LIB_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/lib64)
set(MXBASE_ROOT_DIR $ENV{MX_SDK_HOME})
set(MXBASE_INC ${MXBASE_ROOT_DIR}/include)
set(MXBASE_LIB_DIR ${MXBASE_ROOT_DIR}/lib)
set(MXBASE_POST_LIB_DIR ${MXBASE_ROOT_DIR}/lib/modelpostprocessors)
set(MXBASE_POST_PROCESS_DIR ${MXBASE_ROOT_DIR}/include/MxBase/postprocess/include)
if(DEFINED ENV{MXSDK_OPENSOURCE_DIR})
set(OPENSOURCE_DIR $ENV{MXSDK_OPENSOURCE_DIR})
else()
set(OPENSOURCE_DIR ${MXBASE_ROOT_DIR}/opensource)
endif()
include_directories(${ACL_INC_DIR})
include_directories(${OPENSOURCE_DIR}/include)
include_directories(${OPENSOURCE_DIR}/include/opencv4)
include_directories(${MXBASE_INC})
include_directories(${MXBASE_POST_PROCESS_DIR})
link_directories(${ACL_LIB_DIR})
link_directories(${OPENSOURCE_DIR}/lib)
link_directories(${MXBASE_LIB_DIR})
link_directories(${MXBASE_POST_LIB_DIR})
add_executable(${TARGET} ./src/main.cpp ./src/squeezenetResidual.cpp)
target_link_libraries(${TARGET} glog cpprest mxbase resnet50postprocess opencv_world stdc++fs)
install(TARGETS ${TARGET} RUNTIME DESTINATION ${PROJECT_SOURCE_DIR}/)
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
path_cur=$(dirname $0)
function check_env()
{
# set ASCEND_VERSION to ascend-toolkit/latest when it was not specified by user
if [ ! "${ASCEND_VERSION}" ]; then
export ASCEND_VERSION=ascend-toolkit/latest
echo "Set ASCEND_VERSION to the default value: ${ASCEND_VERSION}"
else
echo "ASCEND_VERSION is set to ${ASCEND_VERSION} by user"
fi
if [ ! "${ARCH_PATTERN}" ]; then
# set ARCH_PATTERN to ./ when it was not specified by user
export ARCH_PATTERN=./
echo "ARCH_PATTERN is set to the default value: ${ARCH_PATTERN}"
else
echo "ARCH_PATTERN is set to ${ARCH_PATTERN} by user"
fi
}
function build_resnext101()
{
cd $path_cur
rm -rf build
mkdir -p build
cd build
cmake ..
make
ret=$?
if [ ${ret} -ne 0 ]; then
echo "Failed to build resnext101."
exit ${ret}
fi
make install
}
check_env
build_resnext101
/*
* Copyright (c) 2021. Huawei Technologies Co., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "squeezenetResidual.h"
#include <dirent.h>
#include "MxBase/Log/Log.h"
namespace {
const uint32_t CLASS_NUM = 1000;
} // namespace
APP_ERROR ScanImages(const std::string &path, std::vector<std::string> &imgFiles) {
DIR *dirPtr = opendir(path.c_str());
if (dirPtr == nullptr) {
LogError << "opendir failed. dir:" << path;
return APP_ERR_INTERNAL_ERROR;
}
dirent *direntPtr = nullptr;
while ((direntPtr = readdir(dirPtr)) != nullptr) {
std::string fileName = direntPtr->d_name;
if (fileName == "." || fileName == "..") {
continue;
}
imgFiles.emplace_back(path + "/" + fileName);
}
closedir(dirPtr);
return APP_ERR_OK;
}
int main(int argc, char* argv[]) {
if (argc <= 1) {
LogWarn << "Please input image path, such as './squeezenetResidual image_dir'.";
return APP_ERR_OK;
}
InitParam initParam = {};
initParam.deviceId = 0;
initParam.classNum = CLASS_NUM;
initParam.labelPath = "../../data/config/imagenet1000_claidx_to_labels.names";
initParam.topk = 5;
initParam.softmax = false;
initParam.checkTensor = true;
initParam.modelPath = "../../data/model/squeezenetResidual.om";
auto squeezenet = std::make_shared<squeezenetResidual>();
APP_ERROR ret = squeezenet->Init(initParam);
if (ret != APP_ERR_OK) {
LogError << "SqueezenetResidual init failed, ret=" << ret << ".";
return ret;
}
std::string imgPath = argv[1];
std::vector<std::string> imgFilePaths;
ret = ScanImages(imgPath, imgFilePaths);
if (ret != APP_ERR_OK) {
squeezenet->DeInit();
return ret;
}
auto startTime = std::chrono::high_resolution_clock::now();
for (auto &imgFile : imgFilePaths) {
ret = squeezenet->Process(imgFile);
if (ret != APP_ERR_OK) {
LogError << "SqueezenetResidual process failed, ret=" << ret << ".";
squeezenet->DeInit();
return ret;
}
}
auto endTime = std::chrono::high_resolution_clock::now();
squeezenet->DeInit();
double costMilliSecs = std::chrono::duration<double, std::milli>(endTime - startTime).count();
double fps = 1000.0 * imgFilePaths.size() / squeezenet->GetInferCostMilliSec();
LogInfo << "[Process Delay] cost: " << costMilliSecs << " ms\tfps: " << fps << " imgs/sec";
return APP_ERR_OK;
}
/*
* Copyright (c) 2021. Huawei Technologies Co., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <map>
#include "MxBase/DeviceManager/DeviceManager.h"
#include "MxBase/Log/Log.h"
#include "squeezenetResidual.h"
namespace {
const uint32_t YUV_BYTE_NU = 3;
const uint32_t YUV_BYTE_DE = 2;
const uint32_t VPC_H_ALIGN = 2;
}
APP_ERROR squeezenetResidual::Init(const InitParam &initParam) {
deviceId_ = initParam.deviceId;
APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices();
if (ret != APP_ERR_OK) {
LogError << "Init devices failed, ret=" << ret << ".";
return ret;
}
ret = MxBase::TensorContext::GetInstance()->SetContext(initParam.deviceId);
if (ret != APP_ERR_OK) {
LogError << "Set context failed, ret=" << ret << ".";
return ret;
}
dvppWrapper_ = std::make_shared<MxBase::DvppWrapper>();
ret = dvppWrapper_->Init();
if (ret != APP_ERR_OK) {
LogError << "DvppWrapper init failed, ret=" << ret << ".";
return ret;
}
model_ = std::make_shared<MxBase::ModelInferenceProcessor>();
ret = model_->Init(initParam.modelPath, modelDesc_);
if (ret != APP_ERR_OK) {
LogError << "ModelInferenceProcessor init failed, ret=" << ret << ".";
return ret;
}
MxBase::ConfigData configData;
const std::string softmax = initParam.softmax ? "true" : "false";
const std::string checkTensor = initParam.checkTensor ? "true" : "false";
configData.SetJsonValue("CLASS_NUM", std::to_string(initParam.classNum));
configData.SetJsonValue("TOP_K", std::to_string(initParam.topk));
configData.SetJsonValue("SOFTMAX", softmax);
configData.SetJsonValue("CHECK_MODEL", checkTensor);
auto jsonStr = configData.GetCfgJson().serialize();
std::map<std::string, std::shared_ptr<void>> config;
config["postProcessConfigContent"] = std::make_shared<std::string>(jsonStr);
config["labelPath"] = std::make_shared<std::string>(initParam.labelPath);
post_ = std::make_shared<MxBase::Resnet50PostProcess>();
ret = post_->Init(config);
if (ret != APP_ERR_OK) {
LogError << "Resnet50PostProcess init failed, ret=" << ret << ".";
return ret;
}
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::DeInit() {
dvppWrapper_->DeInit();
model_->DeInit();
post_->DeInit();
MxBase::DeviceManager::GetInstance()->DestroyDevices();
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::ReadImage(const std::string &imgPath, cv::Mat &imageMat) {
imageMat = cv::imread(imgPath, cv::IMREAD_COLOR);
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::ResizeImage(const cv::Mat &srcImageMat, cv::Mat &dstImageMat) {
static constexpr uint32_t resizeHeight = 256;
static constexpr uint32_t resizeWidth = 256;
cv::resize(srcImageMat, dstImageMat, cv::Size(resizeWidth, resizeHeight));
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::CVMatToTensorBase(const cv::Mat &imageMat, MxBase::TensorBase &tensorBase) {
const uint32_t dataSize = imageMat.cols * imageMat.rows * MxBase::YUV444_RGB_WIDTH_NU;
LogInfo << "image size after crop" << imageMat.cols << " " << imageMat.rows;
MxBase::MemoryData memoryDataDst(dataSize, MxBase::MemoryData::MEMORY_DEVICE, deviceId_);
MxBase::MemoryData memoryDataSrc(imageMat.data, dataSize, MxBase::MemoryData::MEMORY_HOST_MALLOC);
APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDataDst, memoryDataSrc);
if (ret != APP_ERR_OK) {
LogError << GetError(ret) << "Memory malloc failed.";
return ret;
}
std::vector<uint32_t> shape = {imageMat.rows * MxBase::YUV444_RGB_WIDTH_NU, static_cast<uint32_t>(imageMat.cols)};
tensorBase = MxBase::TensorBase(memoryDataDst, false, shape, MxBase::TENSOR_DTYPE_UINT8);
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::Crop(const cv::Mat &srcImageMat, cv::Mat &dstImageMat) {
static cv::Rect rectOfImg(16, 16, 227, 227);
dstImageMat = srcImageMat(rectOfImg).clone();
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::Inference(const std::vector<MxBase::TensorBase> &inputs,
std::vector<MxBase::TensorBase> &outputs) {
auto dtypes = model_->GetOutputDataType();
for (size_t i = 0; i < modelDesc_.outputTensors.size(); ++i) {
std::vector<uint32_t> shape = {};
for (size_t j = 0; j < modelDesc_.outputTensors[i].tensorDims.size(); ++j) {
shape.push_back((uint32_t)modelDesc_.outputTensors[i].tensorDims[j]);
}
MxBase::TensorBase tensor(shape, dtypes[i], MxBase::MemoryData::MemoryType::MEMORY_DEVICE, deviceId_);
APP_ERROR ret = MxBase::TensorBase::TensorBaseMalloc(tensor);
if (ret != APP_ERR_OK) {
LogError << "TensorBaseMalloc failed, ret=" << ret << ".";
return ret;
}
outputs.push_back(tensor);
}
MxBase::DynamicInfo dynamicInfo = {};
dynamicInfo.dynamicType = MxBase::DynamicType::STATIC_BATCH;
auto startTime = std::chrono::high_resolution_clock::now();
APP_ERROR ret = model_->ModelInference(inputs, outputs, dynamicInfo);
auto endTime = std::chrono::high_resolution_clock::now();
double costMs = std::chrono::duration<double, std::milli>(endTime - startTime).count(); // save time
inferCostTimeMilliSec += costMs;
if (ret != APP_ERR_OK) {
LogError << "ModelInference failed, ret=" << ret << ".";
return ret;
}
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::PostProcess(const std::vector<MxBase::TensorBase> &inputs,
std::vector<std::vector<MxBase::ClassInfo>> &clsInfos) {
APP_ERROR ret = post_->Process(inputs, clsInfos);
if (ret != APP_ERR_OK) {
LogError << "Process failed, ret=" << ret << ".";
return ret;
}
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::SaveResult(const std::string &imgPath,
const std::vector<std::vector<MxBase::ClassInfo>> &BatchClsInfos) {
LogInfo << "image path" << imgPath;
std::string fileName = imgPath.substr(imgPath.find_last_of("/") + 1);
size_t dot = fileName.find_last_of(".");
std::string resFileName = "../result/" + fileName.substr(0, dot) + "_1.txt";
LogInfo << "file path for saving result" << resFileName;
std::ofstream outfile(resFileName);
if (outfile.fail()) {
LogError << "Failed to open result file: ";
return APP_ERR_COMM_FAILURE;
}
uint32_t batchIndex = 0;
for (auto clsInfos : BatchClsInfos) {
std::string resultStr;
for (auto clsInfo : clsInfos) {
LogDebug << " className:" << clsInfo.className << " confidence:" << clsInfo.confidence <<
" classIndex:" << clsInfo.classId;
resultStr += std::to_string(clsInfo.classId) + " ";
}
outfile << resultStr << std::endl;
batchIndex++;
}
outfile.close();
return APP_ERR_OK;
}
APP_ERROR squeezenetResidual::Process(const std::string &imgPath) {
cv::Mat imageMat;
APP_ERROR ret = ReadImage(imgPath, imageMat);
if (ret != APP_ERR_OK) {
LogError << "ReadImage failed, ret=" << ret << ".";
return ret;
}
cv::Mat resizeImage;
ret = ResizeImage(imageMat, resizeImage);
if (ret != APP_ERR_OK) {
LogError << "Resize failed, ret=" << ret << ".";
return ret;
}
cv::Mat cropImage;
ret = Crop(resizeImage, cropImage);
if (ret != APP_ERR_OK) {
LogError << "Crop failed, ret=" << ret << ".";
return ret;
}
std::vector<MxBase::TensorBase> inputs = {};
std::vector<MxBase::TensorBase> outputs = {};
MxBase::TensorBase tensorBase;
ret = CVMatToTensorBase(cropImage, tensorBase);
if (ret != APP_ERR_OK) {
LogError << "CVMatToTensorBase failed, ret=" << ret << ".";
return ret;
}
inputs.push_back(tensorBase);
auto startTime = std::chrono::high_resolution_clock::now();
ret = Inference(inputs, outputs);
auto endTime = std::chrono::high_resolution_clock::now();
double costMs = std::chrono::duration<double, std::milli>(endTime - startTime).count(); // save time
inferCostTimeMilliSec += costMs;
if (ret != APP_ERR_OK) {
LogError << "Inference failed, ret=" << ret << ".";
return ret;
}
std::vector<std::vector<MxBase::ClassInfo>> BatchClsInfos = {};
ret = PostProcess(outputs, BatchClsInfos);
if (ret != APP_ERR_OK) {
LogError << "PostProcess failed, ret=" << ret << ".";
return ret;
}
ret = SaveResult(imgPath, BatchClsInfos);
if (ret != APP_ERR_OK) {
LogError << "Save infer results into file failed. ret = " << ret << ".";
return ret;
}
return APP_ERR_OK;
}
/*
* Copyright (c) 2021. Huawei Technologies Co., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MXBASE_SQUEEZENETRESIDUAL_H
#define MXBASE_SQUEEZENETRESIDUAL_H
#include <string>
#include <vector>
#include <memory>
#include <opencv2/opencv.hpp>
#include "MxBase/DvppWrapper/DvppWrapper.h"
#include "MxBase/ModelInfer/ModelInferenceProcessor.h"
#include "MxBase/Tensor/TensorContext/TensorContext.h"
#include "ClassPostProcessors/Resnet50PostProcess.h"
struct InitParam {
uint32_t deviceId;
std::string labelPath;
uint32_t classNum;
uint32_t topk;
bool softmax;
bool checkTensor;
std::string modelPath;
};
class squeezenetResidual {
public:
APP_ERROR Init(const InitParam &initParam);
APP_ERROR DeInit();
APP_ERROR ReadImage(const std::string &imgPath, cv::Mat &imageMat);
APP_ERROR ResizeImage(const cv::Mat &srcImageMat, cv::Mat &dstImageMat);
APP_ERROR CVMatToTensorBase(const cv::Mat &imageMat, MxBase::TensorBase &tensorBase);
APP_ERROR Crop(const cv::Mat &srcImageMat, cv::Mat &dstImageMat);
APP_ERROR Inference(const std::vector<MxBase::TensorBase> &inputs, std::vector<MxBase::TensorBase> &outputs);
APP_ERROR PostProcess(const std::vector<MxBase::TensorBase> &inputs,
std::vector<std::vector<MxBase::ClassInfo>> &clsInfos);
APP_ERROR Process(const std::string &imgPath);
// get infer time
double GetInferCostMilliSec() const {return inferCostTimeMilliSec;}
private:
APP_ERROR SaveResult(const std::string &imgPath,
const std::vector<std::vector<MxBase::ClassInfo>> &batchClsInfos);
private:
std::shared_ptr<MxBase::DvppWrapper> dvppWrapper_;
std::shared_ptr<MxBase::ModelInferenceProcessor> model_;
std::shared_ptr<MxBase::Resnet50PostProcess> post_;
MxBase::ModelDesc modelDesc_;
uint32_t deviceId_ = 0;
// infer time
double inferCostTimeMilliSec = 0.0;
};
#endif
#coding = utf-8
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the BSD 3-Clause License (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import sys
import datetime
from StreamManagerApi import StreamManagerApi, MxDataInput
if __name__ == '__main__':
# init stream manager
stream_manager = StreamManagerApi()
ret = stream_manager.InitManager()
if ret != 0:
print("Failed to init Stream manager, ret=%s" % str(ret))
exit()
# create streams by pipeline config file
with open("../data/config/squeezenet.pipeline", 'rb') as f:
pipeline = f.read()
ret = stream_manager.CreateMultipleStreams(pipeline)
if ret != 0:
print("Failed to create Stream, ret=%s" % str(ret))
exit()
# Construct the input of the stream
data_input = MxDataInput()
dir_name = sys.argv[1]
res_dir_name = sys.argv[2]
file_list = os.listdir(dir_name)
if not os.path.exists(res_dir_name):
os.makedirs(res_dir_name)
for file_name in file_list:
file_path = os.path.join(dir_name, file_name)
print(file_path)
if not (file_name.lower().endswith(".jpg")
or file_name.lower().endswith(".jpeg")):
continue
with open(file_path, 'rb') as f:
data_input.data = f.read()
empty_data = []
stream_name = b'im_squeezenetResidual'
# Inputs data to a specified stream based on streamName.
in_plugin_id = 0
unique_id = stream_manager.SendData(stream_name, in_plugin_id, data_input)
if unique_id < 0:
print("Failed to send data to stream.")
exit()
# Obtain the inference result by specifying streamName and uniqueId.
start_time = datetime.datetime.now()
infer_result = stream_manager.GetResult(stream_name, unique_id)
end_time = datetime.datetime.now()
print('sdk run time: {}'.format((end_time - start_time).microseconds))
if infer_result.errorCode != 0:
print("GetResultWithUniqueId error. errorCode=%d, errorMsg=%s" % (
infer_result.errorCode, infer_result.data.decode()))
exit()
# print the infer result
infer_res = infer_result.data.decode()
print("process img: {}, infer result: {}".format(file_name, infer_res))
load_dict = json.loads(infer_result.data.decode())
if load_dict.get('MxpiClass') is None:
with open(res_dir_name + "/" + file_name[:-5] + '.txt',
'w') as f_write:
f_write.write("")
continue
res_vec = load_dict.get('MxpiClass')
with open(res_dir_name + "/" + file_name[:-5] + '_1.txt',
'w') as f_write:
res_list = [str(item.get("classId")) + " " for item in res_vec]
f_write.writelines(res_list)
f_write.write('\n')
# destroy streams
stream_manager.DestroyAllStreams()
\ No newline at end of file
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
image_path=$1
result_dir=$2
set -e
# Simple log helper functions
info() { echo -e "\033[1;34m[INFO ][MxStream] $1\033[1;37m" ; }
warn() { echo >&2 -e "\033[1;31m[WARN ][MxStream] $1\033[1;37m" ; }
export LD_LIBRARY_PATH=${MX_SDK_HOME}/lib:${MX_SDK_HOME}/opensource/lib:${MX_SDK_HOME}/opensource/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:${LD_LIBRARY_PATH}
export GST_PLUGIN_SCANNER=${MX_SDK_HOME}/opensource/libexec/gstreamer-1.0/gst-plugin-scanner
export GST_PLUGIN_PATH=${MX_SDK_HOME}/opensource/lib/gstreamer-1.0:${MX_SDK_HOME}/lib/plugins
#to set PYTHONPATH, import the StreamManagerApi.py
export PYTHONPATH=$PYTHONPATH:${MX_SDK_HOME}/python
python3.7 main.py $image_path $result_dir
#coding = utf-8
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the BSD 3-Clause License (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import json
import numpy as np
def cre_groundtruth_dict_fromtxt(gtfile_path):
"""
:param filename: file contains the imagename and label number
:return: dictionary key imagename, value is label number
"""
img_dict = {}
with open(gtfile_path, 'r') as f:
for line in f.readlines():
temp = line.strip().split(" ")
img_name = temp[0].split(".")[0]
img_dict[img_name] = temp[1]
return img_dict
def load_statistical_predict_result(filepath):
"""
function:
the prediction esult file data extraction
input:
result file:filepath
output:
n_label:numble of label
data_vec: the probabilitie of prediction in the 1000
:return: probabilities, numble of label, in_type, color
"""
with open(filepath, 'r')as f:
temp = f.readline().strip().split(" ")
n_label = len(temp)
data_vec = np.zeros((len(temp)), dtype=np.float32)
if n_label != 0:
for ind, cls_ind in enumerate(temp):
data_vec[ind] = np.int32(cls_ind)
return data_vec, n_label
def create_visualization_statistical_result(prediction_file_path,
result_store_path,
img_dict_2, topn=5):
"""
:param prediction_file_path:
:param result_store_path:
:param json_file_name:
:param img_dict_2:
:param topn:
:return:
"""
writer = open(result_store_path, 'w')
table_dict = {}
table_dict["title"] = "Overall statistical evaluation"
table_dict["value"] = []
count = 0
res_cnt = 0
n_labels = ""
count_hit = np.zeros(topn)
for tfile_name in os.listdir(prediction_file_path):
count += 1
temp = tfile_name.split('.')[0]
index = temp.rfind('_')
img_name = temp[:index]
filepath = os.path.join(prediction_file_path, tfile_name)
prediction, n_labels = load_statistical_predict_result(filepath)
if n_labels == 11:
real_label = int(img_dict_2[img_name]) + 1
else:
real_label = int(img_dict_2[img_name])
res_cnt = min(len(prediction), topn)
for i in range(res_cnt):
if real_label == int(prediction[i]):
count_hit[i] += 1
break
if 'value' not in table_dict.keys():
print("the item value does not exist!")
else:
table_dict["value"].extend(
[{"key": "Number of images", "value": str(count)},
{"key": "Number of classes", "value": str(n_labels)}])
accuracy = np.cumsum(count_hit) / count if count else 0
for i in range(res_cnt):
table_dict["value"].append({"key": "Top" + str(i + 1) + " accuracy",
"value": str(
round(accuracy[i] * 100, 2)) + '%'})
json.dump(table_dict, writer, indent=2)
writer.close()
if __name__ == '__main__':
if len(sys.argv) == 4:
# txt file path
folder_davinci_target = sys.argv[1]
# annotation files path, "val_label.txt"
annotation_file_path = sys.argv[2]
# the path to store the results json path
result_json_path = sys.argv[3]
else:
print("Please enter target file result folder | ground truth label file | result json file folder | "
"result json file name, such as ./result val_label.txt . result.json")
exit(1)
if not os.path.exists(folder_davinci_target):
print("Target file folder does not exist.")
if not os.path.exists(annotation_file_path):
print("Ground truth file does not exist.")
img_dict_1 = cre_groundtruth_dict_fromtxt(annotation_file_path)
create_visualization_statistical_result(folder_davinci_target,
result_json_path,
img_dict_1, topn=5)
This diff is collapsed.
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""train squeezenet."""
import os
import sys
import argparse
import numpy as np
import moxing as mox
from mindspore import context
from mindspore import Tensor
from mindspore import export
from mindspore.nn.optim.momentum import Momentum
from mindspore.train.model import Model
from mindspore.context import ParallelMode
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor
from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits
from mindspore.train.loss_scale_manager import FixedLossScaleManager
from mindspore.train.serialization import load_checkpoint, load_param_into_net
from mindspore.communication.management import init, get_rank, get_group_size
from mindspore.common import set_seed
from model_utils.moxing_adapter import moxing_wrapper
from model_utils.device_adapter import get_device_id
from src.lr_generator import get_lr
from src.CrossEntropySmooth import CrossEntropySmooth
set_seed(1)
_CACHE_DATA_URL = "./cache/data"
_CACHE_TRAIN_URL = "./cache/train"
_CACHE_LOAD_URL = "./cache/checkpoint_path"
_CACHE_PRETRAIN_URL = "./cache/res/tmp.ckpt"
_NONE = "none"
# Transfer learning parameter
parser = argparse.ArgumentParser('mindspore squeezenet_residual training')
parser.add_argument('--train_url', type=str, default='obs://neu-base/squeezenet/cache/train',
help='where training log and ckpts saved')
parser.add_argument('--data_url', type=str, default='obs://neu-base/squeezenet/cache/data',
help='path of dataset')
parser.add_argument('--dataset', type=str, default='imagenet', help='Dataset.')
parser.add_argument('--batch_size', type=int, default=8,
help='batch size')
parser.add_argument('--file_format', type=str, default="AIR",
help='output model formats')
parser.add_argument('--pre_trained', type=str,
default='obs://neu-base/squeezenet/suqeezenet_residual_imagenet-50_3.ckpt',
help='pretrained model')
parser.add_argument('--epoch_size', type=int, default=1,
help='epoch num')
parser.add_argument('--save_checkpoint_epochs', type=int, default=1,
help='how many epochs to save ckpt once')
parser.add_argument('--device_target', type=str, default='Ascend',
choices=['Ascend', 'CPU', 'GPU'],
help='device where the code will be implemented. '
'(Default: Ascend)')
parser.add_argument('--file_name', type=str, default="squeezenet_residual",
help='output file name')
parser.add_argument('--lr_max', type=float, default=0.01,
help='lr_max')
args, _ = parser.parse_known_args()
os.makedirs(_CACHE_TRAIN_URL, exist_ok=True)
os.makedirs(_CACHE_DATA_URL, exist_ok=True)
os.makedirs(_CACHE_LOAD_URL, exist_ok=True)
mox.file.copy_parallel(args.data_url, _CACHE_DATA_URL)
train_url = _CACHE_TRAIN_URL
data_url = _CACHE_DATA_URL
load_url = _CACHE_LOAD_URL
if args.dataset == "cifar10":
from src.config import config_cifar as config
else:
from src.config import config_imagenet as config
print("Dataset: ", config.dataset)
#train
if config.net_name == "squeezenet":
from src.squeezenet import SqueezeNet as squeezenet
if config.dataset == "cifar10":
from src.dataset import create_dataset_cifar as create_dataset
else:
from src.dataset import create_dataset_imagenet as create_dataset
else:
from src.squeezenet import SqueezeNet_Residual as squeezenet
if config.dataset == "cifar10":
from src.dataset import create_dataset_cifar as create_dataset
else:
from src.dataset import create_dataset_imagenet as create_dataset
@moxing_wrapper()
def train_net():
"""train net"""
config.output_path = train_url
config.data_path = data_url
config.load_path = load_url
config.epoch_size = args.epoch_size
config.save_checkpoint_epochs = args.save_checkpoint_epochs
config.device_target = args.device_target
target = config.device_target
ckpt_save_dir = config.output_path
if args.pre_trained != _NONE:
mox.file.copy_parallel(args.pre_trained, _CACHE_PRETRAIN_URL)
else:
config.pre_trained = ""
# init context
context.set_context(mode=context.GRAPH_MODE,
device_target=target)
device_num = 1
if config.run_distribute:
if target == "Ascend":
device_id = get_device_id()
device_num = config.device_num
context.set_context(device_id=device_id)
context.set_auto_parallel_context(
device_num=device_num,
parallel_mode=ParallelMode.DATA_PARALLEL,
gradients_mean=True)
init()
# GPU target
else:
init()
device_num = get_group_size()
context.set_auto_parallel_context(
device_num=device_num,
parallel_mode=ParallelMode.DATA_PARALLEL,
gradients_mean=True)
ckpt_save_dir = ckpt_save_dir + "/ckpt_" + str(
get_rank()) + "/"
# create dataset
dataset = create_dataset(dataset_path=config.data_path,
do_train=True,
repeat_num=1,
batch_size=config.batch_size,
run_distribute=config.run_distribute)
step_size = dataset.get_dataset_size()
# define net
net = squeezenet(num_classes=config.class_num)
# load checkpoint
if config.pre_trained:
param_dict = load_checkpoint(config.pre_trained)
load_param_into_net(net, param_dict)
# init lr
lr = get_lr(lr_init=config.lr_init,
lr_end=config.lr_end,
lr_max=config.lr_max,
total_epochs=args.epoch_size,
warmup_epochs=config.warmup_epochs,
pretrain_epochs=config.pretrain_epoch_size,
steps_per_epoch=step_size,
lr_decay_mode=config.lr_decay_mode)
lr = Tensor(lr)
# define loss
if config.dataset == "imagenet":
if not config.use_label_smooth:
config.label_smooth_factor = 0.0
loss = CrossEntropySmooth(sparse=True,
reduction='mean',
smooth_factor=config.label_smooth_factor,
num_classes=config.class_num)
else:
loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
# define opt, model
if target == "Ascend":
loss_scale = FixedLossScaleManager(config.loss_scale,
drop_overflow_update=False)
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
lr,
config.momentum,
config.weight_decay,
config.loss_scale,
use_nesterov=True)
model = Model(net,
loss_fn=loss,
optimizer=opt,
loss_scale_manager=loss_scale,
metrics={'acc'},
amp_level="O2",
keep_batchnorm_fp32=False)
else:
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
lr,
config.momentum,
config.weight_decay,
use_nesterov=True)
model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'})
# define callbacks
time_cb = TimeMonitor(data_size=step_size)
loss_cb = LossMonitor()
cb = [time_cb, loss_cb]
if config.save_checkpoint:
config_ck = CheckpointConfig(
save_checkpoint_steps=config.save_checkpoint_epochs * step_size,
keep_checkpoint_max=config.keep_checkpoint_max)
ckpt_cb = ModelCheckpoint(prefix=config.net_name + '_' + config.dataset,
directory=ckpt_save_dir,
config=config_ck)
cb += [ckpt_cb]
# train model
model.train(config.epoch_size - config.pretrain_epoch_size,
dataset,
callbacks=cb)
if config.net_name == "squeezenet":
from src.squeezenet import SqueezeNet as squeezenet
else:
from src.squeezenet import SqueezeNet_Residual as squeezenet
if config.dataset == "cifar10":
num_classes = 10
else:
num_classes = 1000
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
if config.device_target == "Ascend":
context.set_context(device_id=config.device_id)
def modelarts_pre_process():
'''modelarts pre process function.'''
config.file_name = os.path.join(config.output_path, config.file_name)
@moxing_wrapper(pre_process=modelarts_pre_process)
def run_export():
#get_last_ckpt
ckpt_dir = train_url
file_dict = {}
lists = os.listdir(ckpt_dir)
for i in lists:
ctime = os.stat(os.path.join(ckpt_dir, i)).st_ctime
file_dict[ctime] = i
max_ctime = max(file_dict.keys())
ckpt_files = [ckpt_file for ckpt_file in os.listdir(ckpt_dir)
if ckpt_file.endswith(file_dict[max_ctime])]
if not ckpt_files:
print("No ckpt file found.")
sys.exit(0)
ckpt_file = os.path.join(ckpt_dir, sorted(ckpt_files)[-1])
# export_air
if not ckpt_file:
sys.exit(0)
config.batch_size = args.batch_size
config.checkpoint_file_path = ckpt_file
config.file_name = os.path.join(_CACHE_TRAIN_URL, args.file_name)
print("\nStart exporting AIR.")
net = squeezenet(num_classes=num_classes)
param_dict = load_checkpoint(config.checkpoint_file_path)
load_param_into_net(net, param_dict)
input_data = Tensor(np.zeros([config.batch_size, 3, config.height, config.width], np.float32))
export(net, input_data, file_name=config.file_name, file_format=config.file_format)
print("\nStart uploading to obs.")
mox.file.copy_parallel(_CACHE_TRAIN_URL, args.train_url)
if __name__ == '__main__':
train_net()
run_export()
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
network config setting, will be used in train.py and eval.py
"""
from easydict import EasyDict as ed
# config for squeezenet, imagenet
config_imagenet = ed({
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
"enable_modelarts": False,
# Url for modelarts
"data_url": "",
"train_url": "",
"checkpoint_url": "",
# Path for local
"run_distribute": False,
"enable_profiling": False,
"data_path": "/cache/data",
"output_path": "/cache/train",
"load_path": "/cache/checkpoint_path/",
"device_num": 1,
"device_id": 0,
"device_target": "Ascend",
"checkpoint_path": "./checkpoint/",
"checkpoint_file_path": "suqeezenet_residual_imagenet-300_5004.ckpt",
# Training options
"net_name": "suqeezenet_residual",
"dataset": "imagenet",
"class_num": 1000,
"batch_size": 32,
"loss_scale": 1024,
"momentum": 0.9,
"weight_decay": 7e-5,
"epoch_size": 300,
"pretrain_epoch_size": 0,
"save_checkpoint": True,
"save_checkpoint_epochs": 1,
"keep_checkpoint_max": 10,
"save_checkpoint_path": "./",
"warmup_epochs": 0,
"lr_decay_mode": "cosine",
"use_label_smooth": True,
"label_smooth_factor": 0.1,
"lr_init": 0,
"lr_end": 0,
"lr_max": 0.01,
"pre_trained": "",
#export
"width": 227,
"height": 227,
"file_name": "squeezenet",
"file_format": "AIR"
})
# config for squeezenet, cifar10
config_cifar = ed({
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
"enable_modelarts": False,
# Url for modelarts
"data_url": "",
"train_url": "",
"checkpoint_url": "",
# Path for local
"run_distribute": False,
"enable_profiling": False,
"data_path": "/cache/data",
"output_path": "/cache/train",
"load_path": "/cache/checkpoint_path/",
"device_num": 1,
"device_id": 0,
"device_target": "Ascend",
"checkpoint_path": "./checkpoint/",
"checkpoint_file_path": "suqeezenet_residual_cifar10-150_195.ckpt",
# Training options
"net_name": "suqeezenet_residual",
"dataset": "cifar10",
"class_num": 10,
"batch_size": 32,
"loss_scale": 1024,
"momentum": 0.9,
"weight_decay": 1e-4,
"epoch_size": 150,
"pretrain_epoch_size": 0,
"save_checkpoint": True,
"save_checkpoint_epochs": 1,
"keep_checkpoint_max": 10,
"save_checkpoint_path": "./",
"warmup_epochs": 5,
"lr_decay_mode": "linear",
"lr_init": 0,
"lr_end": 0,
"lr_max": 0.01,
"pre_trained": "",
#export
"width": 227,
"height": 227,
"file_name": "squeezenet",
"file_format": "AIR"
})
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment