diff --git a/research/cv/sknet/infer/mxbase/CMakeLists.txt b/research/cv/sknet/infer/mxbase/CMakeLists.txt index ccc171aceadef32c17dac28232be56ed73c8d76f..60fd7d0f911acf70aabf0820c777488b4b75ac20 100644 --- a/research/cv/sknet/infer/mxbase/CMakeLists.txt +++ b/research/cv/sknet/infer/mxbase/CMakeLists.txt @@ -37,7 +37,7 @@ endif() include_directories(${ACL_INC_DIR}) -include_directories($ENV{ASCEND_HOME}/ascend-toolkit/5.0.4/x86_64-linux/runtime/include) +include_directories($ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/x86_64-linux/runtime/include) include_directories(${OPENSOURCE_DIR}/include) include_directories(${OPENSOURCE_DIR}/include/opencv4) include_directories(${MXBASE_INC}) diff --git a/research/cv/sknet/infer/mxbase/build.sh b/research/cv/sknet/infer/mxbase/build.sh index f88a7d12f72d117d67af5e7f8ff61515c3f86834..849031eb4d92bfd496bc9bea6fc07b67eed1a522 100644 --- a/research/cv/sknet/infer/mxbase/build.sh +++ b/research/cv/sknet/infer/mxbase/build.sh @@ -20,6 +20,14 @@ mkdir -p build cd build || exit function make_plugin() { + # set ASCEND_VERSION to ascend-toolkit/latest when it was not specified by user + if [ ! "${ASCEND_VERSION}" ]; then + export ASCEND_VERSION=ascend-toolkit/latest + echo "Set ASCEND_VERSION to the default value: ${ASCEND_VERSION}" + else + echo "ASCEND_VERSION is set to ${ASCEND_VERSION} by user" + fi + if ! cmake ..; then echo "cmake failed." diff --git a/research/gnn/dgcn/infer/convert/convert.sh b/research/gnn/dgcn/infer/convert/convert.sh new file mode 100644 index 0000000000000000000000000000000000000000..0f626d31d0fabfe3e79118671d89efb53314dec9 --- /dev/null +++ b/research/gnn/dgcn/infer/convert/convert.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +air_path=$1 +om_path=$2 +echo "Input AIR file path: ${air_path}" +echo "Output OM file path: ${om_path}" +atc --framework=1 --model="${air_path}" \ + --output="${om_path}" \ + --soc_version=Ascend310 \ + --op_select_implmode="high_precision" \ + --output_type=FP32 \ No newline at end of file diff --git a/research/gnn/dgcn/infer/docker_start_infer.sh b/research/gnn/dgcn/infer/docker_start_infer.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c68a790ae82492948129dea1c09e056c35ac4d0 --- /dev/null +++ b/research/gnn/dgcn/infer/docker_start_infer.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker_image=$1 +model_dir=$2 +function show_help() { + echo "Usage: docker_start.sh docker_image model_dir data_dir" +} +function param_check() { + if [ -z "${docker_image}" ]; then + echo "please input docker_image" + show_help + exit 1 + fi + if [ -z "${model_dir}" ]; then + echo "please input model_dir" + show_help + exit 1 + fi +} +param_check +docker run -it -u root \ + --device=/dev/davinci0 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm \ + --device=/dev/hisi_hdc \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v ${model_dir}:${model_dir} \ + ${docker_image} \ + /bin/bash diff --git a/research/gnn/dgcn/infer/mxbase/CMakeLists.txt b/research/gnn/dgcn/infer/mxbase/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0a79cc6e7fdc5f89167dc25b70daf9fc65d1a45a --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.10.0) +project(dgcn) +set(TARGET dgcn) +add_definitions(-DENABLE_DVPP_INTERFACE) +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +add_definitions(-Dgoogle=mindxsdk_private) +add_compile_options(-std=c++11 -fPIE -fstack-protector-all -fPIC -Wall) +add_link_options(-Wl,-z,relro,-z,now,-z,noexecstack -s -pie) +# Check environment variable +if(NOT DEFINED ENV{ASCEND_HOME}) + message(FATAL_ERROR "please define environment variable:ASCEND_HOME") +endif() +if(NOT DEFINED ENV{ASCEND_VERSION}) + message(WARNING "please define environment variable:ASCEND_VERSION") +endif() +if(NOT DEFINED ENV{ARCH_PATTERN}) + message(WARNING "please define environment variable:ARCH_PATTERN") +endif() +set(ACL_INC_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/include) +set(ACL_LIB_DIR $ENV{ASCEND_HOME}/$ENV{ASCEND_VERSION}/$ENV{ARCH_PATTERN}/acllib/lib64) +set(MXBASE_ROOT_DIR $ENV{MX_SDK_HOME}) +set(MXBASE_INC ${MXBASE_ROOT_DIR}/include) +set(MXBASE_LIB_DIR ${MXBASE_ROOT_DIR}/lib) +set(MXBASE_POST_LIB_DIR ${MXBASE_ROOT_DIR}/lib/modelpostprocessors) +set(MXBASE_POST_PROCESS_DIR ${MXBASE_ROOT_DIR}/include/MxBase/postprocess/include) +if(DEFINED ENV{MXSDK_OPENSOURCE_DIR}) + set(OPENSOURCE_DIR $ENV{MXSDK_OPENSOURCE_DIR}) +else() + set(OPENSOURCE_DIR ${MXBASE_ROOT_DIR}/opensource) +endif() +include_directories(${ACL_INC_DIR}) +include_directories(${OPENSOURCE_DIR}/include) +include_directories(${OPENSOURCE_DIR}/include/opencv4) +include_directories(${MXBASE_INC}) +include_directories(${MXBASE_POST_PROCESS_DIR}) +link_directories(${ACL_LIB_DIR}) +link_directories(${OPENSOURCE_DIR}/lib) +link_directories(${MXBASE_LIB_DIR}) +link_directories(${MXBASE_POST_LIB_DIR}) +add_executable(${TARGET} src/main.cpp src/dgcn.cpp) +target_link_libraries(${TARGET} glog cpprest mxbase opencv_world stdc++fs) +install(TARGETS ${TARGET} RUNTIME DESTINATION ${PROJECT_SOURCE_DIR}/) diff --git a/research/gnn/dgcn/infer/mxbase/build.sh b/research/gnn/dgcn/infer/mxbase/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..11d273905d6de2038ccad7f7087c87b4e05b8e20 --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +path_cur=$(dirname $0) +function check_env() +{ + # set ASCEND_VERSION to ascend-toolkit/latest when it was not specified by user + if [ ! "${ASCEND_VERSION}" ]; then + export ASCEND_VERSION=ascend-toolkit/latest + echo "Set ASCEND_VERSION to the default value: ${ASCEND_VERSION}" + else + echo "ASCEND_VERSION is set to ${ASCEND_VERSION} by user" + fi + if [ ! "${ARCH_PATTERN}" ]; then + # set ARCH_PATTERN to ./ when it was not specified by user + export ARCH_PATTERN=./ + echo "ARCH_PATTERN is set to the default value: ${ARCH_PATTERN}" + else + echo "ARCH_PATTERN is set to ${ARCH_PATTERN} by user" + fi +} +function build_bert() +{ + cd $path_cur + rm -rf build + mkdir -p build + cd build + cmake .. + make + ret=$? + if [ ${ret} -ne 0 ]; then + echo "Failed to build bert." + exit ${ret} + fi + make install +} +check_env +build_bert \ No newline at end of file diff --git a/research/gnn/dgcn/infer/mxbase/eval_by_mxbase.py b/research/gnn/dgcn/infer/mxbase/eval_by_mxbase.py new file mode 100644 index 0000000000000000000000000000000000000000..95071e00fe9e78861d09b309200b1230669771fa --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/eval_by_mxbase.py @@ -0,0 +1,64 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +postprocess. +""" +import argparse + +import numpy as np + + +def Accuracy(label, mask, preds): + """Accuracy with masking.""" + preds = preds.astype(np.float32) + correct_prediction = np.equal(np.argmax(preds, axis=1), np.argmax(label, axis=1)) + accuracy_all = correct_prediction.astype(np.float32) + mask = mask.astype(np.float32) + mask_reduce = np.mean(mask) + mask = mask / mask_reduce + accuracy_all *= mask + return np.mean(accuracy_all) + + +def get_acc(): + """get infer Accuracy.""" + parser = argparse.ArgumentParser(description='postprocess') + parser.add_argument('--dataset_name', type=str, default='cora', + choices=['cora', 'citeseer', 'pubmed'], help='dataset name') + parser.add_argument('--label_path', type=str, default='', help='y_test npy Files') + parser.add_argument('--mask_path', type=str, default='', help='test_mask npy Files') + args_opt = parser.parse_args() + + label_onehot = np.load(args_opt.label_path) + test_mask = np.load(args_opt.mask_path) + + if args_opt.dataset_name == 'cora': + pred = np.loadtxt('pred1.txt') + pred = pred.reshape(2708, 7) + if args_opt.dataset_name == 'citeseer': + pred = np.loadtxt('pred2.txt') + pred = pred.reshape(3327, 6) + if args_opt.dataset_name == 'pubmed': + pred = np.loadtxt('pred3.txt') + pred = pred.reshape(19717, 3) + + acc = Accuracy(label_onehot, test_mask, pred) + print("Test set results:", "accuracy=", "{:.5f}".format(acc)) + + +if __name__ == '__main__': + get_acc() + \ No newline at end of file diff --git a/research/gnn/dgcn/infer/mxbase/src/DGCN.h b/research/gnn/dgcn/infer/mxbase/src/DGCN.h new file mode 100644 index 0000000000000000000000000000000000000000..f79134617660e5fa8863ef7e5f2a08de5900378f --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/src/DGCN.h @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MXBASE_BERTBASE_H +#define MXBASE_BERTBASE_H +#include <unistd.h> +#include <memory> +#include <utility> +#include <vector> +#include <string> +#include <map> +#include <opencv2/opencv.hpp> +#include "MxBase/DvppWrapper/DvppWrapper.h" +#include "MxBase/ModelInfer/ModelInferenceProcessor.h" +#include "MxBase/Tensor/TensorContext/TensorContext.h" + +extern std::vector<double> g_inferCost; +extern uint32_t g_TP; +extern uint32_t g_FP; +extern uint32_t g_FN; + +struct InitParam { + uint32_t deviceId; + std::string datasetPath; + std::string modelPath; +}; + + +class DgcnBase { + public: + APP_ERROR Init(const InitParam &initParam); + APP_ERROR DeInit(); + APP_ERROR Inference(const std::vector<MxBase::TensorBase> &inputs, std::vector<MxBase::TensorBase> *outputs); + APP_ERROR Process(const std::string &inferPath, const std::string &fileName, const std::string &dataname); + APP_ERROR PostProcess(std::vector<MxBase::TensorBase> *outputs, std::vector<float> *result); + protected: + APP_ERROR ReadTensorFromFile(const std::string &file, float *data, uint32_t size); + APP_ERROR ReadInputTensor(const std::string &fileName, uint32_t index, std::vector<MxBase::TensorBase> *inputs, + uint32_t size, MxBase::TensorDataType type, const std::string &dataname); + APP_ERROR getAucResult(std::vector<float> *result, uint32_t size, const std::string &dataname); + private: + std::shared_ptr<MxBase::DvppWrapper> dvppWrapper_; + std::shared_ptr<MxBase::ModelInferenceProcessor> model_; + MxBase::ModelDesc modelDesc_ = {}; + std::vector<std::string> labelMap_ = {}; + uint32_t deviceId_ = 0; +}; +#endif diff --git a/research/gnn/dgcn/infer/mxbase/src/dgcn.cpp b/research/gnn/dgcn/infer/mxbase/src/dgcn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f9a6a3cd01ee420dfccf89f3a3a6150745af1d6b --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/src/dgcn.cpp @@ -0,0 +1,266 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DGCN.h" +#include <unistd.h> +#include <sys/stat.h> +#include <map> +#include <fstream> +#include "MxBase/DeviceManager/DeviceManager.h" +#include "MxBase/Log/Log.h" + + +APP_ERROR DgcnBase::Init(const InitParam &initParam) { + deviceId_ = initParam.deviceId; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices(); + if (ret != APP_ERR_OK) { + LogError << "Init devices failed, ret=" << ret << "."; + return ret; + } + ret = MxBase::TensorContext::GetInstance()->SetContext(initParam.deviceId); + if (ret != APP_ERR_OK) { + LogError << "Set context failed, ret=" << ret << "."; + return ret; + } + model_ = std::make_shared<MxBase::ModelInferenceProcessor>(); + ret = model_->Init(initParam.modelPath, modelDesc_); + if (ret != APP_ERR_OK) { + LogError << "ModelInferenceProcessor init failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::DeInit() { + dvppWrapper_->DeInit(); + model_->DeInit(); + MxBase::DeviceManager::GetInstance()->DestroyDevices(); + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::ReadTensorFromFile(const std::string &file, float *data, uint32_t size) { + if (data == NULL) { + LogError << "input data is invalid."; + return APP_ERR_COMM_INVALID_POINTER; + } + // open file + std::ifstream in(file, std::ios::in | std::ios::binary); + // check file validity + if (in.fail()) { + LogError << "Failed to open label file: " << file << "."; + return APP_ERR_COMM_OPEN_FAIL; + } + in.read(reinterpret_cast<char*>(data), sizeof(float) * size); + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::ReadInputTensor(const std::string &fileName, uint32_t index, + std::vector<MxBase::TensorBase> *inputs, uint32_t size, + MxBase::TensorDataType type, const std::string &dataname) { + float* data = new float[size]; + APP_ERROR ret = ReadTensorFromFile(fileName, data, size); + std::cout << "size is" << size << std::endl; + if (ret != APP_ERR_OK) { + LogError << "ReadTensorFromFile failed."; + return ret; + } + const uint32_t dataSize = modelDesc_.inputTensors[index].tensorSize; + LogInfo << "dataSize: " << dataSize; + MxBase::MemoryData memoryDataDst(dataSize, MxBase::MemoryData::MEMORY_DEVICE, deviceId_); + MxBase::MemoryData memoryDataSrc(reinterpret_cast<void*>(data), dataSize, MxBase::MemoryData::MEMORY_HOST_MALLOC); + ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDataDst, memoryDataSrc); + if (ret != APP_ERR_OK) { + LogError << GetError(ret) << "Memory malloc and copy failed."; + return ret; + } + uint32_t length4 = 1; + if (dataname == "cora") { + length4 = 2708; + } + if (dataname == "pubmed") { + length4 = 19717; + } + if (dataname == "citeseer") { + length4 = 3312; + } + std::vector<uint32_t> shape = {length4, size/length4}; + inputs->push_back(MxBase::TensorBase(memoryDataDst, false, shape, type)); + std::cout << "shape con" << shape[0] << shape[1] << std::endl; + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::Inference(const std::vector<MxBase::TensorBase> &inputs, std::vector<MxBase::TensorBase> *outputs) { + auto dtypes = model_->GetOutputDataType(); + for (size_t i = 0; i < modelDesc_.outputTensors.size(); ++i) { + std::vector<uint32_t> shape = {}; + for (size_t j = 0; j < modelDesc_.outputTensors[i].tensorDims.size(); ++j) { + shape.push_back((uint32_t)modelDesc_.outputTensors[i].tensorDims[j]); + } + MxBase::TensorBase tensor(shape, dtypes[i], MxBase::MemoryData::MemoryType::MEMORY_DEVICE, deviceId_); + APP_ERROR ret = MxBase::TensorBase::TensorBaseMalloc(tensor); + if (ret != APP_ERR_OK) { + LogError << "TensorBaseMalloc failed, ret=" << ret << "."; + return ret; + } + outputs->push_back(tensor); + } + MxBase::DynamicInfo dynamicInfo = {}; + dynamicInfo.dynamicType = MxBase::DynamicType::STATIC_BATCH; + auto startTime = std::chrono::high_resolution_clock::now(); + APP_ERROR ret = model_->ModelInference(inputs, *outputs, dynamicInfo); + auto endTime = std::chrono::high_resolution_clock::now(); + double costMs = std::chrono::duration<double, std::milli>(endTime - startTime).count(); + g_inferCost.push_back(costMs); + if (ret != APP_ERR_OK) { + LogError << "ModelInference failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::PostProcess(std::vector<MxBase::TensorBase> *outputs, std::vector<float> *result) { + MxBase::TensorBase &tensor = outputs->at(0); + APP_ERROR ret = tensor.ToHost(); + if (ret != APP_ERR_OK) { + LogError << GetError(ret) << "Tensor deploy to host failed."; + return ret; + } + // check tensor is available + auto outputShape = tensor.GetShape(); + uint32_t length = outputShape[0]; + uint32_t classNum = outputShape[1]; + LogInfo << "output shape is: " << length << " "<< classNum << " " << std::endl; + void* data = tensor.GetBuffer(); + LogInfo << "output shape is: " << data; + for (uint32_t i = 0; i < length; i++) { + for (uint32_t j = 0; j < classNum; j++) { + float value = *(reinterpret_cast<float*>(data) + i * classNum + j); + result->push_back(value); + } + } + return APP_ERR_OK; +} + +APP_ERROR DgcnBase::getAucResult(std::vector<float> *result, uint32_t nice, const std::string &dataname) { + if (dataname == "pubmed") { + std::ofstream fp_r("pred3.txt", std::ofstream::app); + int k = 0; + for (uint32_t i = 0; i < 59151; i++) { + fp_r << result->at(i) << " "; + k++; + if (k != 0&&k%3 == 0) { + fp_r << std::endl; + } + } + fp_r << std::endl; + fp_r.close(); + } + if (dataname == "cora") { + std::ofstream fp_r("pred1.txt", std::ofstream::app); + int k = 0; + for (uint32_t i = 0; i < 18956; i++) { + fp_r << result->at(i) << " "; + k++; + if (k != 0&&k%7 == 0) { + fp_r << std::endl; + } + } + fp_r << std::endl; + fp_r.close(); + } + if (dataname == "citeseer") { + std::ofstream fp_r("pred2.txt", std::ofstream::app); + int k = 0; + for (uint32_t i = 0; i < 19962; i++) { + fp_r << result->at(i) << " "; + k++; + if (k != 0&&k%6 == 0) { + fp_r << std::endl; + } + } + fp_r << std::endl; + fp_r.close(); + } + return APP_ERR_OK; +} + + +APP_ERROR DgcnBase::Process(const std::string &inferPath, const std::string &fileName, const std::string &dataname) { + std::vector<MxBase::TensorBase> inputs = {}; + uint32_t lengtht1 = 0; + uint32_t lengtht2 = 0; + uint32_t lengtht3 = 0; + if (dataname == "cora") { + lengtht1 = 7333264; + lengtht2 = 7333264; + lengtht3 = 3880564; + } + if (dataname == "citeseer") { + lengtht1 = 10969344; + lengtht2 = 10969344; + lengtht3 = 12264336; + } + if (dataname == "pubmed") { + lengtht1 = 388760089; + lengtht2 = 388760089; + lengtht3 = 9858500; + } + std::string filename1 = "diffusions.bin"; + std::string spareFile = inferPath + "/00_data/" + filename1; + LogInfo << "read file name: " << spareFile; + APP_ERROR ret = ReadInputTensor(spareFile, 0, &inputs, lengtht1, MxBase::TENSOR_DTYPE_FLOAT32, dataname); + if (ret != APP_ERR_OK) { + LogError << "Read input spare failed, ret=" << ret << "."; + return ret; + } + std::string filename2 = "ppmi.bin"; + std::string denseFile = inferPath + "/01_data/" + filename2; + LogInfo << "read file name: " << denseFile; + ret = ReadInputTensor(denseFile, 1, &inputs, lengtht2, MxBase::TENSOR_DTYPE_FLOAT32, dataname); + if (ret != APP_ERR_OK) { + LogError << "Read input dense file failed, ret=" << ret << "."; + return ret; + } + std::string filename3 = "feature.bin"; + std::string labelFile = inferPath + "/02_data/" + filename3; + LogInfo << "read file name: " << labelFile; + ret = ReadInputTensor(labelFile, 2, &inputs, lengtht3, MxBase::TENSOR_DTYPE_FLOAT32, dataname); + if (ret != APP_ERR_OK) { + LogError << "Read label file failed, ret=" << ret << "."; + return ret; + } + std::vector<MxBase::TensorBase> outputs = {}; + ret = Inference(inputs, &outputs); + if (ret != APP_ERR_OK) { + LogError << "Inference failed, ret=" << ret << "."; + return ret; + } + std::vector<float> result; + ret = PostProcess(&outputs, &result); + if (ret != APP_ERR_OK) { + LogError << "PostProcess failed, ret=" << ret << "."; + return ret; + } + uint32_t nice = outputs.size(); + std::cout << "nice is" << nice << std::endl; + + ret = getAucResult(&result, nice, dataname); + if (ret != APP_ERR_OK) { + LogError << "CalcF1Score read label failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} diff --git a/research/gnn/dgcn/infer/mxbase/src/main.cpp b/research/gnn/dgcn/infer/mxbase/src/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d14878efdb0dbb50399b09c2605641e43157df96 --- /dev/null +++ b/research/gnn/dgcn/infer/mxbase/src/main.cpp @@ -0,0 +1,92 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <unistd.h> +#include <dirent.h> +#include <iostream> +#include <fstream> +#include <vector> +#include "DGCN.h" +#include "MxBase/Log/Log.h" + +std::vector<double> g_inferCost; + +void InitFFMParam(InitParam* initParam) { + initParam->deviceId = 0; + initParam->modelPath = "../convert/cora.om"; +} + +APP_ERROR ReadFilesFromPath(const std::string &path, std::vector<std::string> *files) { + DIR *dir = NULL; + struct dirent *ptr = NULL; + + if ((dir=opendir(path.c_str())) == NULL) { + LogError << "Open dir error: " << path; + return APP_ERR_COMM_OPEN_FAIL; + } + + while ((ptr=readdir(dir)) != NULL) { + // d_type == 8 is file + if (ptr->d_type == 8) { + files->push_back(ptr->d_name); + } + } + closedir(dir); + // sort ascending order + sort(files->begin(), files->end()); + return APP_ERR_OK; +} + +int main(int argc, char* argv[]) { + if (argc <= 1) { + LogWarn << "Please input dataset path"; + return APP_ERR_OK; + } + + InitParam initParam; + InitFFMParam(&initParam); + auto ffmBase = std::make_shared<DgcnBase>(); + APP_ERROR ret = ffmBase->Init(initParam); + if (ret != APP_ERR_OK) { + LogError << "FFMbase init failed, ret=" << ret << "."; + return ret; + } + std::string inferPath = argv[1]; + std::string dataname = argv[2]; + std::vector<std::string> files; + ret = ReadFilesFromPath(inferPath + "/00_data/", &files); + if (ret != APP_ERR_OK) { + LogError << "Read files from path failed, ret=" << ret << "."; + return ret; + } + for (uint32_t i = 0; i < 1; i++) { + ret = ffmBase->Process(inferPath, files[i], dataname); + if (ret != APP_ERR_OK) { + LogError << "FFMbase process failed, ret=" << ret << "."; + ffmBase->DeInit(); + return ret; + } + LogInfo << "finished " << i << " bin file"; + } + ffmBase->DeInit(); + double costSum = 0; + for (uint32_t i = 0; i < g_inferCost.size(); i++) { + costSum += g_inferCost[i]; + } + LogInfo << "Infer sum " << g_inferCost.size() << ", cost total time: " << costSum << " ms."; + LogInfo << "The throughput: " << g_inferCost.size() * 1000 / costSum << " bin/sec."; + return APP_ERR_OK; +} diff --git a/research/gnn/dgcn/infer/sdk/api/infer.py b/research/gnn/dgcn/infer/sdk/api/infer.py new file mode 100644 index 0000000000000000000000000000000000000000..6e2846a360ecda4e314d1786102370a546a9ece5 --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/api/infer.py @@ -0,0 +1,129 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Model Infer """ +import json +import logging +import numpy as np +import MxpiDataType_pb2 as MxpiDataType +from StreamManagerApi import StreamManagerApi, MxDataInput, InProtobufVector, MxProtobufIn, StringVector +from config import config as cfg + + +class SdkApi: + """ Class SdkApi """ + INFER_TIMEOUT = cfg.INFER_TIMEOUT + STREAM_NAME = cfg.STREAM_NAME + + def __init__(self, pipeline_cfg): + self.pipeline_cfg = pipeline_cfg + self._stream_api = None + self._data_input = None + self._device_id = None + + def init(self): + """ Initialize Stream """ + with open(self.pipeline_cfg, 'r') as fp: + self._device_id = int( + json.loads(fp.read())[self.STREAM_NAME]["stream_config"] + ["deviceId"]) + print(f"The device id: {self._device_id}.") + + # create api + self._stream_api = StreamManagerApi() + + # init stream mgr + ret = self._stream_api.InitManager() + if ret != 0: + print(f"Failed to init stream manager, ret={ret}.") + return False + + # create streams + with open(self.pipeline_cfg, 'rb') as fp: + pipe_line = fp.read() + + ret = self._stream_api.CreateMultipleStreams(pipe_line) + if ret != 0: + print(f"Failed to create stream, ret={ret}.") + return False + + self._data_input = MxDataInput() + return True + + def __del__(self): + if not self._stream_api: + return + + self._stream_api.DestroyAllStreams() + + def _send_protobuf(self, stream_name, plugin_id, element_name, buf_type, + pkg_list): + """ Send Stream """ + protobuf = MxProtobufIn() + protobuf.key = element_name.encode("utf-8") + protobuf.type = buf_type + protobuf.protobuf = pkg_list.SerializeToString() + protobuf_vec = InProtobufVector() + protobuf_vec.push_back(protobuf) + err_code = self._stream_api.SendProtobuf(stream_name, plugin_id, + protobuf_vec) + if err_code != 0: + logging.error( + "Failed to send data to stream, stream_name(%s), plugin_id(%s), element_name(%s), " + "buf_type(%s), err_code(%s).", stream_name, plugin_id, + element_name, buf_type, err_code) + return False + return True + + def send_tensor_input(self, stream_name, plugin_id, element_name, + input_data, input_shape, data_type, number): + """ Send Tensor """ + tensor_list = MxpiDataType.MxpiTensorPackageList() + for i in range(number): + data = np.expand_dims(input_data[i, :], 0) + tensor_pkg = tensor_list.tensorPackageVec.add() + # init tensor vector + tensor_vec = tensor_pkg.tensorVec.add() + tensor_vec.deviceId = self._device_id + tensor_vec.memType = 0 + tensor_vec.tensorShape.extend(data.shape) + tensor_vec.tensorDataType = data_type + tensor_vec.dataStr = data.tobytes() + tensor_vec.tensorDataSize = data.shape[0] + print(type(tensor_list)) + buf_type = b"MxTools.MxpiTensorPackageList" + return self._send_protobuf(stream_name, plugin_id, element_name, + buf_type, tensor_list) + + def get_result(self, stream_name, out_plugin_id=0): + """ Get Result """ + keys = [b"mxpi_tensorinfer0"] + keyVec = StringVector() + for key in keys: + keyVec.push_back(key) + infer_result = self._stream_api.GetProtobuf(stream_name, 0, keyVec) + if infer_result.size() == 0: + print("infer_result is null") + exit() + + if infer_result[0].errorCode != 0: + print("GetProtobuf error. errorCode=%d" % ( + infer_result[0].errorCode)) + exit() + TensorList = MxpiDataType.MxpiTensorPackageList() + TensorList.ParseFromString(infer_result[0].messageBuf) + print('tensorShape', TensorList.tensorPackageVec[0].tensorVec[1].tensorShape) + TensorList.ParseFromString(infer_result[0].messageBuf) + #[2708,2,7] + return TensorList diff --git a/research/gnn/dgcn/infer/sdk/config/config.py b/research/gnn/dgcn/infer/sdk/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..a2bae68f4d20de309fc60c4dee87055c5ad31d6d --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/config/config.py @@ -0,0 +1,23 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Model Config """ +STREAM_NAME = "dgcn" + +INFER_TIMEOUT = 100000 + +TENSOR_DTYPE_FLOAT32 = 0 +TENSOR_DTYPE_FLOAT16 = 1 +TENSOR_DTYPE_INT8 = 2 +TENSOR_DTYPE_INT32 = 3 diff --git a/research/gnn/dgcn/infer/sdk/config/dgcn.pipeline b/research/gnn/dgcn/infer/sdk/config/dgcn.pipeline new file mode 100644 index 0000000000000000000000000000000000000000..f5dc0f09018728420f927cd1a7c18185cc2ccace --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/config/dgcn.pipeline @@ -0,0 +1,53 @@ +{ + "dgcn": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_tensorinfer0:0" + }, + "appsrc1": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_tensorinfer0:1" + }, + "appsrc2": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "queue0" + }, + "queue0":{ + "factory": "queue", + "next": "mxpi_tensorinfer0:2" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "appsrc0,appsrc1,appsrc2", + "modelPath": "../data/input/cora.om" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_dataserialize0" + }, + "mxpi_dataserialize0": { + "props": { + "outputDataKeys": "mxpi_tensorinfer0" + }, + "factory": "mxpi_dataserialize", + "next": "appsink0" + }, + "appsink0": { + "props":{ + "blocksize":"4096000" + }, + "factory":"appsink" + } + } +} diff --git a/research/gnn/dgcn/infer/sdk/eval_by_sdk.py b/research/gnn/dgcn/infer/sdk/eval_by_sdk.py new file mode 100644 index 0000000000000000000000000000000000000000..62ac162c1efb5b93aac173b0e1965bec7558a246 --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/eval_by_sdk.py @@ -0,0 +1,63 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +postprocess. +""" +import argparse + +import numpy as np + + +def Accuracy(label, mask, preds): + """Accuracy with masking.""" + preds = preds.astype(np.float32) + correct_prediction = np.equal(np.argmax(preds, axis=1), np.argmax(label, axis=1)) + accuracy_all = correct_prediction.astype(np.float32) + mask = mask.astype(np.float32) + mask_reduce = np.mean(mask) + mask = mask / mask_reduce + accuracy_all *= mask + return np.mean(accuracy_all) + + +def get_acc(): + """get infer Accuracy.""" + parser = argparse.ArgumentParser(description='postprocess') + parser.add_argument('--dataset_name', type=str, default='cora', + choices=['cora', 'citeseer', 'pubmed'], help='dataset name') + parser.add_argument('--label_path', type=str, default='', help='y_test npy Files') + parser.add_argument('--mask_path', type=str, default='', help='test_mask npy Files') + args_opt = parser.parse_args() + + label_onehot = np.load(args_opt.label_path) + test_mask = np.load(args_opt.mask_path) + + if args_opt.dataset_name == 'cora': + pred = np.loadtxt('pred1.txt') + pred = pred.reshape(2708, 7) + if args_opt.dataset_name == 'citeseer': + pred = np.loadtxt('pred2.txt') + pred = pred.reshape(3327, 6) + if args_opt.dataset_name == 'pubmed': + pred = np.loadtxt('pred3.txt') + pred = pred.reshape(19717, 3) + + acc = Accuracy(label_onehot, test_mask, pred) + print("Test set results:", "accuracy=", "{:.5f}".format(acc)) + + +if __name__ == '__main__': + get_acc() diff --git a/research/gnn/dgcn/infer/sdk/main.py b/research/gnn/dgcn/infer/sdk/main.py new file mode 100644 index 0000000000000000000000000000000000000000..48b2a0e7a398e5153d7894bef4736db64cc53e5e --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/main.py @@ -0,0 +1,91 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Model Main """ +import argparse +import time +import glob +import numpy as np +from api.infer import SdkApi +from config import config as cfg + + +def parse_args(): + """set and check parameters.""" + parser = argparse.ArgumentParser(description="bert process") + parser.add_argument("--pipeline_path", type=str, default="./config/dgcn.pipeline", help="SDK infer pipeline") + parser.add_argument("--dataset", type=str, default="cora", help="dataset name") + parser.add_argument("--data_dir", type=str, default="../data/input/cora_bin/", + help="Dataset contain batch_spare batch_label batch_dense") + args_opt = parser.parse_args() + return args_opt + + +def inference(pipeline_path, stream_name, dataset): + sdk_api = SdkApi(pipeline_path) + if not sdk_api.init(): + exit(-1) + files1 = glob.glob(pathname=args.data_dir + '/00_data/' + '*.bin') + files2 = glob.glob(pathname=args.data_dir + '/01_data/' + '*.bin') + files3 = glob.glob(pathname=args.data_dir + '/02_data/' + '*.bin') + files_length1 = len(files1) + files_length2 = len(files2) + files_length3 = len(files3) + if dataset == "cora": + number = 2708 + number1 = 1433 + number2 = 7 + filename = 'pred1.txt' + if dataset == "citeseer": + number = 3327 + number1 = 3703 + number2 = 6 + filename = 'pred2.txt' + if dataset == "pubmed": + number = 19717 + number1 = 500 + number2 = 3 + filename = 'pred3.txt' + for i in range(files_length1): + start_time = time.time() + input_spare_file = args.data_dir + "/00_data/" + "diffusions.bin" + input_spare = np.fromfile(input_spare_file, dtype=np.float16).reshape(number, number) + sdk_api.send_tensor_input(stream_name, 0, "appsrc0", input_spare, input_spare.shape, cfg.TENSOR_DTYPE_FLOAT16, + number) + for i in range(files_length2): + input_dense_file = args.data_dir + "/01_data/" + "ppmi.bin" + input_dense = np.fromfile(input_dense_file, dtype=np.float16).reshape(number, number) + sdk_api.send_tensor_input(stream_name, 1, "appsrc1", input_dense, input_dense.shape, cfg.TENSOR_DTYPE_FLOAT16, + number) + for i in range(files_length3): + # set label data + input_label_file = args.data_dir + "/02_data/" + "feature.bin" + input_label = np.fromfile(input_label_file, dtype=np.float16).reshape(number, number1) + sdk_api.send_tensor_input(stream_name, 2, "appsrc2", input_label, input_label.shape, cfg.TENSOR_DTYPE_FLOAT16, + number) + result = sdk_api.get_result(stream_name) + end_time = time.time() - start_time + print(f"The criteo({i}) inference time is {end_time}") + pred = np.array( + [np.frombuffer(result.tensorPackageVec[i].tensorVec[0].dataStr, dtype=np.float32) for i in range(number)]) + with open(filename, 'a+') as f: + for i in range(number): + for j in range(number2): + f.write(str(pred[i][j]) + " ") + f.write("\n") + +if __name__ == "__main__": + args = parse_args() + args.stream_name = cfg.STREAM_NAME.encode("utf-8") + inference(args.pipeline_path, args.stream_name, args.dataset) diff --git a/research/gnn/dgcn/infer/sdk/run.sh b/research/gnn/dgcn/infer/sdk/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..281c33cb61bef6749eb2dbbd80454acb4b55382f --- /dev/null +++ b/research/gnn/dgcn/infer/sdk/run.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [[ $# -gt 2 ]]; then + echo "Usage: bash run.sh [DATASET] [DATA_DIR]" +exit 1 +fi + +DATASET=$1 +DATA_DIR=$2 + +python3.9 main.py --dataset=$DATASET --data_dir=$DATA_DIR \ No newline at end of file diff --git a/research/gnn/dgcn/modelarts/src/dgcn.py b/research/gnn/dgcn/modelarts/src/dgcn.py new file mode 100644 index 0000000000000000000000000000000000000000..6ffef31ee2bd6fb24ea4b186bbdce561fbfde897 --- /dev/null +++ b/research/gnn/dgcn/modelarts/src/dgcn.py @@ -0,0 +1,86 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""DGCN Network.""" +import numpy as np +from mindspore import nn +from mindspore.ops import operations as P +from mindspore import Tensor +from mindspore.nn.layer.activation import get_activation + + +def glorot(shape): + """Randomly generated weight.""" + W = np.asarray( + np.random.RandomState(1234).uniform( + low=-np.sqrt(6. / (shape[0]+shape[1])), + high=np.sqrt(6. / (shape[0]+shape[1])), + size=(shape[0], shape[1]) + ), dtype=np.float32) + return Tensor(W) + + +class GraphConvolution(nn.Cell): + """Graph convolutional layer.""" + def __init__(self, + feature_in_dim, + feature_out_dim, + dropout_ratio=None, + activation=None, + ): + super(GraphConvolution, self).__init__() + self.in_dim = feature_in_dim + self.out_dim = feature_out_dim + self.weight_init = glorot([self.out_dim, self.in_dim]) + self.fc = nn.Dense(self.in_dim, + self.out_dim, + weight_init=self.weight_init, + has_bias=False) + self.dropout_flag = False + self.dropout_ratio = dropout_ratio + if self.dropout_ratio is not None: + self.dropout_flag = self.dropout_ratio + self.dropout = nn.Dropout(keep_prob=1-self.dropout_ratio) + self.activation = get_activation(activation) + self.activation_flag = self.activation is not None + self.matmul = P.MatMul() + + def construct(self, adj, input_feature): + """Convolutional operations.""" + dropout = input_feature + if self.dropout_flag: + dropout = self.dropout(dropout) + + fc = self.fc(dropout) + output_feature = self.matmul(adj, fc) + + if self.activation_flag: + output_feature = self.activation(output_feature) + return output_feature + + +class DGCN(nn.Cell): + """Generate DGCN model.""" + def __init__(self, input_dim, hidden_dim, output_dim, dropout): + super(DGCN, self).__init__() + self.layer0 = GraphConvolution(input_dim, hidden_dim, activation='relu', dropout_ratio=dropout) + self.layer1 = GraphConvolution(hidden_dim, output_dim, dropout_ratio=dropout) + + def construct(self, adj, ppmi, feature): + Softmax = nn.Softmax() + diffoutput0 = self.layer0(adj, feature) + diffoutput1 = Softmax(self.layer1(adj, diffoutput0)) + ppmioutput0 = self.layer0(ppmi, feature) + ppmioutput1 = Softmax(self.layer1(ppmi, ppmioutput0)) + return diffoutput1, ppmioutput1 diff --git a/research/gnn/dgcn/modelarts/src/utilities.py b/research/gnn/dgcn/modelarts/src/utilities.py new file mode 100644 index 0000000000000000000000000000000000000000..da9600d83fdc1091a0ac6cb9d5b3d5044987e2cc --- /dev/null +++ b/research/gnn/dgcn/modelarts/src/utilities.py @@ -0,0 +1,212 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""generate diffusion and ppmi matrix""" +import random +import itertools +import math +import scipy +import scipy.sparse as sp +import numpy as np +from numpy import inf + + +def diffusion_fun_sparse(A): + """Function of Calculating diffusion matrix.""" + n, m = A.shape + A_with_selfloop = A + sp.identity(n, format='csc') + diags = A_with_selfloop.sum(axis=1).flatten() + + with scipy.errstate(divide='ignore'):# Context manager for error handling, ignoring exceptions divided by zero + diags_sqrt = 1.0 / scipy.sqrt(diags) + diags_sqrt[scipy.isinf(diags_sqrt)] = 0 + DH = sp.spdiags(diags_sqrt, [0], m, n, format='csc') + d = DH.dot(A_with_selfloop.dot(DH)) + return d + + +def _normalize_diffusion_matrix(A): + """Used to nomalized diffusion matrix.""" + n, m = A.shape + A_with_selfloop = A + diags = A_with_selfloop.sum(axis=1).flatten() + + with scipy.errstate(divide='ignore'): + diags_sqrt = 1.0 / scipy.sqrt(diags) + diags_sqrt[scipy.isinf(diags_sqrt)] = 0 + DH = sp.spdiags(diags_sqrt, [0], m, n, format='csc') + d = DH.dot(A_with_selfloop.dot(DH)) + return d + + +def diffusion_fun_improved(A, sampling_num=100, path_len=3, + self_loop=True, spars=False): + """Return normalized adjacent matrix plus PPMI.""" + print("Do the sampling...") + mat = _diffusion_fun_sampling( + A, sampling_num=sampling_num, path_len=path_len, + self_loop=self_loop, spars=spars) + print("Calculating the PPMI...") + # mat is a sparse lil_matrix + pmi = None + if spars: + pmi = _PPMI_sparse(mat) + else: + pmi = _PPMI(mat) + A_with_selfloop = A + pmi + dig = np.sum(A_with_selfloop, axis=1) + dig = np.squeeze(np.asarray(dig)) + Degree = np.diag(dig) + Degree_normalized = Degree ** (-0.5) + Degree_normalized[Degree_normalized == inf] = 0.0 + Diffusion = np.dot( + np.dot(Degree_normalized, A_with_selfloop), Degree_normalized) + return Diffusion + + +def diffusion_fun_improved_ppmi_dynamic_sparsity(A, sampling_num=100, path_len=2, + self_loop=True, spars=True, k=1.0): + """Do the sampling and calculate the PPMI.""" + print("Do the sampling...") + mat = _diffusion_fun_sampling( + A, sampling_num=sampling_num, path_len=path_len, + self_loop=self_loop, spars=spars) + print("Calculating the PPMI...") + # mat is a sparse dok_matrix + if spars: + pmi = _PPMI_sparse(mat) + else: + pmi = _PPMI(mat) + + pmi = _shift(pmi, k) + ans = _normalize_diffusion_matrix(pmi.tocsc()) + + return ans + + +def _shift(mat, k): + """Shift function.""" + print(k) + r, c = mat.shape + x, y = mat.nonzero() + mat = mat.todok() + offset = np.log(k) + print("Offset: " + str(offset)) + for i, j in zip(x, y): + mat[i, j] = max(mat[i, j] - offset, 0) + + x, y = mat.nonzero() + sparsity = 1.0 - len(x) / float(r * c) + print("Sparsity: " + str(sparsity)) + return mat + + +def _diffusion_fun_sampling(A, sampling_num=100, path_len=3, self_loop=True, spars=False): + """Return diffusion matrix.""" + re = None + if not spars: + re = np.zeros(A.shape) + else: + re = sp.dok_matrix(A.shape, dtype=np.float32) + + if self_loop: + A_with_selfloop = A + sp.identity(A.shape[0], format="csr")#A+I + else: + A_with_selfloop = A + + # record each node's neignbors + dict_nid_neighbors = {} + for nid in range(A.shape[0]): + neighbors = np.nonzero(A_with_selfloop[nid])[1] + dict_nid_neighbors[nid] = neighbors + # for each node + for i in range(A.shape[0]): + # for each sampling iter + for dummy_j in range(sampling_num): + _generate_path(i, dict_nid_neighbors, re, path_len) + return re + + +def _generate_path(node_id, dict_nid_neighbors, re, path_len): + """generate path of each node""" + path_node_list = [node_id] + for dummy_i in range(path_len - 1): + temp = dict_nid_neighbors.get(path_node_list[-1]) + if temp.all() == 0: + break + else: + path_node_list.append(random.choice(temp)) + + # update difussion matrix re + for pair in itertools.combinations(path_node_list, 2): + if pair[0] == pair[1]: + re[pair[0], pair[1]] += 1.0 + else: + re[pair[0], pair[1]] += 1.0 + re[pair[1], pair[0]] += 1.0 + + +def _PPMI(mat): + """generate PPMI matrix""" + (nrows, ncols) = mat.shape + colTotals = mat.sum(axis=0) + rowTotals = mat.sum(axis=1).T + # print rowTotals.shape + N = np.sum(rowTotals) + rowMat = np.ones((nrows, ncols), dtype=np.float32) + for i in range(nrows): + rowMat[i, :] = 0 if rowTotals[i] == 0 else rowMat[i, :] * (1.0 / rowTotals[i]) + colMat = np.ones((nrows, ncols), dtype=np.float) + for j in range(ncols): + colMat[:, j] = 0 if colTotals[j] == 0 else colMat[:, j] * (1.0 / colTotals[j]) + P = N * mat * rowMat * colMat + P = np.fmax(np.zeros((nrows, ncols), dtype=np.float32), np.log(P)) + return P + + +def _PPMI_sparse(mat): + """generate sparse PPMI matrix""" + # mat is a sparse dok_matrix + nrows, ncols = mat.shape + colTotals = mat.sum(axis=0) + rowTotals = mat.sum(axis=1).T + + N = float(np.sum(rowTotals)) + rows, cols = mat.nonzero() + + p = sp.dok_matrix((nrows, ncols)) + for i, j in zip(rows, cols): + _under = rowTotals[0, i] * colTotals[0, j] + if _under != 0.0: + log_r = np.log((N * mat[i, j]) / _under) + if log_r > 0: + p[i, j] = log_r + return p + + +def rampup(epoch, scaled_unsup_weight_max, exp=5.0, rampup_length=80): + """Use to calculate the coefficient of ppmiloss""" + if epoch < rampup_length: + p = max(0.0, float(epoch)) / float(rampup_length) + p = 1.0 - p + ramp = math.exp(-p * p * exp) * scaled_unsup_weight_max + else: + ramp = 1.0 * scaled_unsup_weight_max + return ramp + + +def get_scaled_unsup_weight_max(num_labels, X_train_shape, unsup_weight_max=100.0): + """Use to calculate the coefficient of ppmiloss""" + return unsup_weight_max * 1.0 * num_labels / X_train_shape diff --git a/research/gnn/dgcn/modelarts/train_start.py b/research/gnn/dgcn/modelarts/train_start.py new file mode 100644 index 0000000000000000000000000000000000000000..0308776c2c6a9b1d0ab1bbb00cbea296494a64df --- /dev/null +++ b/research/gnn/dgcn/modelarts/train_start.py @@ -0,0 +1,214 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# coding=utf-8 +"""training the network""" +import os +import ast +import time +import argparse +import shutil +import mindspore +from mindspore import Tensor, load_checkpoint, export, load_param_into_net +from mindspore import context, set_seed, save_checkpoint +from mindspore.communication.management import init +from mindspore.context import ParallelMode +import numpy as np +import src.data_process as dp +from src.dgcn import DGCN +from src.metrics import LossAccuracyWrapper, TrainNetWrapper +from src.utilities import diffusion_fun_sparse, \ + diffusion_fun_improved_ppmi_dynamic_sparsity, get_scaled_unsup_weight_max, rampup +from src.config import ConfigDGCN + +parser = argparse.ArgumentParser(description="run DGCN") +parser.add_argument("--device_id", type=int, default=0, help="Device id") +parser.add_argument("--epoch", type=int, default=1, help="epoch num") +parser.add_argument("--seed", type=int, default=1024, help="random seed") +parser.add_argument("--lr", type=float, default=0.01, help="learning rate") +parser.add_argument("--dataset", type=str, default='pubmed', choices=['cora', 'citeseer', 'pubmed'], help="Dataset.") +parser.add_argument('--is_modelarts', type=ast.literal_eval, default=True, help='Train in Modelarts.') +parser.add_argument('--data_url', default=None, help='Location of data.') +parser.add_argument('--train_url', default=None, help='Location of training outputs.') +parser.add_argument("--device_target", type=str, default="Ascend", + choices=["Ascend", "GPU"], help="device target (default: Ascend)") +parser.add_argument("--file_name", type=str, default="dgcn", help="output file name.") +parser.add_argument("--file_format", type=str, choices=["AIR", "MINDIR"], default="AIR", help="file format") +parser.add_argument("--distributed", type=bool, default=False, help="Device id") +args = parser.parse_args() +config = ConfigDGCN() +if args.dataset == "citeseer": + args.seed = 1235 + args.lr = 0.0153 + +if args.device_target == "GPU": + if args.dataset == "cora": + args.seed = 1024 + args.lr = 0.03430959018564401 + if args.dataset == "citeseer": + args.seed = 852 + args.lr = 0.02229329021215099 + if args.dataset == "pubmed": + args.seed = 829 + args.lr = 0.029074859603424454 + +set_seed(args.seed) + + +def run_train(learning_rate=0.01, n_epochs=args.epoch, dataset=None, dropout_rate=0.5, + hidden_size=36): + """run train.""" + context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) + if args.device_target == "Ascend" and not args.distributed: + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id) + if args.distributed: + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id) + init() + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True) + + if args.is_modelarts: + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, + dst_url='/cache/dataset_train/device_' + os.getenv('DEVICE_ID')) + + adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, labels = dp.load_graph_data(dataset) + train_mask = np.reshape(train_mask, (-1, len(train_mask))) + val_mask = np.reshape(val_mask, (-1, len(val_mask))) + test_mask = np.reshape(test_mask, (-1, len(test_mask))) + + #### obtain diffusion and ppmi matrices + diffusions = diffusion_fun_sparse(adj.tocsc()) # Normalized adjacency matrix + ppmi = diffusion_fun_improved_ppmi_dynamic_sparsity(adj, path_len=config.path_len, k=1.0) # ppmi matrix + + adj = Tensor(adj.toarray(), mindspore.float32) + features = Tensor(features.toarray(), mindspore.float32) + diffusions = Tensor(diffusions.toarray(), mindspore.float32) + labels = Tensor(labels, mindspore.float32) + y_test = Tensor(y_test, mindspore.float32) + y_val = Tensor(y_val, mindspore.float32) + y_train = Tensor(y_train, mindspore.float32) + ppmi = Tensor(ppmi.toarray(), mindspore.float32) + + #### construct the classifier model #### + + feature_size = features.shape[1] + label_size = y_train.shape[1] + layer_sizes = [(feature_size, hidden_size), (hidden_size, label_size)] + + print("Convolution Layers:" + str(layer_sizes)) + + dgcn_net = DGCN(input_dim=features.shape[1], hidden_dim=hidden_size, output_dim=y_train.shape[1], + dropout=dropout_rate) + train_net = TrainNetWrapper(dgcn_net, y_train, train_mask, weight_decay=config.weight_decay, + learning_rate=learning_rate) + eval_net = LossAccuracyWrapper(dgcn_net, y_val, val_mask, weight_decay=config.weight_decay) + test_net = LossAccuracyWrapper(dgcn_net, y_test, test_mask, weight_decay=config.weight_decay) + loss_list = [] + dgcn_net.add_flags_recursive(fp16=True) + + #### train model #### + print("...training...") + num_labels = np.sum(train_mask) + X_train_shape = features.shape[0] + accuracys = [] + + for epoch in range(args.epoch): + t = time.time() + + scaled_unsup_weight_max = get_scaled_unsup_weight_max( + num_labels, X_train_shape, unsup_weight_max=15.0) + + ret = rampup(epoch, scaled_unsup_weight_max, exp=5.0, rampup_length=120) + ret = Tensor(ret, mindspore.float32) + + train_net.set_train() + train_result = train_net(diffusions, ppmi, features, ret) + train_loss = train_result[0].asnumpy() + train_accuracy = train_result[1].asnumpy() + + eval_net.set_train(False) + eval_result = eval_net(diffusions, ppmi, features, ret) + eval_loss = eval_result[0].asnumpy() + eval_accuracy = eval_result[1].asnumpy() + loss_list.append(eval_loss) + + print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(train_loss), + "train_acc=", "{:.5f}".format(train_accuracy), "val_loss=", "{:.5f}".format(eval_loss), + "val_acc=", "{:.5f}".format(eval_accuracy), "time=", "{:.5f}".format(time.time() - t)) + if epoch > config.early_stopping and loss_list[-1] > np.mean(loss_list[-(config.early_stopping + 1):-1]): + print("Early stopping...") + break + + t_test = time.time() + test_net.set_train(False) + test_result = test_net(diffusions, ppmi, features, ret) + test_accuracy = test_result[1].asnumpy() + accuracys.append(test_accuracy) + if test_accuracy == max(accuracys): + ckpt_file_name = "/cache/" + ckpt_file_name = os.path.join(ckpt_file_name, dataset) + if not os.path.isdir(ckpt_file_name): + os.makedirs(ckpt_file_name) + ckpt_path = os.path.join(ckpt_file_name, "dgcn.ckpt") + save_checkpoint(dgcn_net, ckpt_path) + + if args.dataset == "cora": + input_dim = 1433 + output_dim = 7 + diffusions = Tensor(np.zeros((2708, 2708), np.float32)) + ppmi = Tensor(np.zeros((2708, 2708), np.float32)) + features = Tensor(np.zeros((2708, 1433), np.float32)) + if args.dataset == "citeseer": + input_dim = 3703 + output_dim = 6 + diffusions = Tensor(np.zeros((3312, 3312), np.float32)) + ppmi = Tensor(np.zeros((3312, 3312), np.float32)) + features = Tensor(np.zeros((3312, 3703), np.float32)) + if args.dataset == "pubmed": + input_dim = 500 + output_dim = 3 + diffusions = Tensor(np.zeros((19717, 19717), np.float32)) + ppmi = Tensor(np.zeros((19717, 19717), np.float32)) + features = Tensor(np.zeros((19717, 500), np.float32)) + save_checkpoint_path = '/cache/' + dgcn_net = DGCN(input_dim=input_dim, hidden_dim=config.hidden1, output_dim=output_dim, dropout=config.dropout) + dgcn_net.set_train(False) + dgcn_net.add_flags_recursive(fp16=False) + ckptname = 'dgcn.ckpt' + ckpt_path = '/cache/' + args.dataset + '/' + param_dict = load_checkpoint(os.path.join(ckpt_path, ckptname)) + load_param_into_net(dgcn_net, param_dict) + export(dgcn_net, diffusions, ppmi, features, file_name=args.file_name, file_format=args.file_format) + print("==========================================") + print(args.file_name + ".air exported successfully!") + print("==========================================") + shutil.copy('dgcn.air', save_checkpoint_path) + mox.file.copy_parallel(src_url='/cache/', dst_url=args.train_url) + print("Test set results:", "accuracy=", "{:.5f}".format(test_accuracy), + "time=", "{:.5f}".format(time.time() - t_test)) + print("Best test accuracy = ", "{:.5f}".format(max(accuracys))) + return max(accuracys) + + +if __name__ == "__main__": + data = args.dataset + print("Testing on the dataset: " + data) + if data in ['cora', 'citeseer', 'pubmed']: + run_train(dataset=data, learning_rate=args.lr, + dropout_rate=config.dropout, n_epochs=config.epochs, hidden_size=config.hidden1) + print("Finished") + else: + print("No such a dataset: " + data) diff --git a/research/gnn/dgcn/script/docker_start.sh b/research/gnn/dgcn/script/docker_start.sh new file mode 100644 index 0000000000000000000000000000000000000000..eff8f5260602cd3ed58ee589d4123e4048dedd50 --- /dev/null +++ b/research/gnn/dgcn/script/docker_start.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.mitations under the License. + +docker_image=$1 +data_dir=$2 +model_dir=$3 + +docker run -it -u root --ipc=host \ + --device=/dev/davinci0 \ + --device=/dev/davinci1 \ + --device=/dev/davinci2 \ + --device=/dev/davinci3 \ + --device=/dev/davinci4 \ + --device=/dev/davinci5 \ + --device=/dev/davinci6 \ + --device=/dev/davinci7 \ + --device=/dev/davinci_manager \ + --device=/dev/devmm_svm --device=/dev/hisi_hdc \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \ + -v ${model_dir}:${model_dir} \ + -v ${data_dir}:${data_dir} \ + -v /root/ascend/log:/root/ascend/log ${docker_image} /bin/bash