diff --git a/research/cv/PatchCore/README_CN.md b/official/cv/patchcore/README_CN.md similarity index 96% rename from research/cv/PatchCore/README_CN.md rename to official/cv/patchcore/README_CN.md index 954a70d0b2b640329ad1658a434a80f3abd4d92e..98018773ca7560afd24bc0649d49ecc0fa21e0d8 100644 --- a/research/cv/PatchCore/README_CN.md +++ b/official/cv/patchcore/README_CN.md @@ -140,6 +140,7 @@ PatchCore使用预训练的WideResNet50作为Encoder, 并去除layer3之后的 │ ├── run_310_infer.sh // 推理脚本 │ ├── run_eval.sh // 评估脚本 │ └── run_train.sh // 训练脚本 + | └── run_all_mvtec.sh // 训练所有的Mvtec数据集 ├── src │ ├── dataset.py // 数据集加载 │ ├── model.py // 模型加载 @@ -180,15 +181,21 @@ python src/pthtockpt.py --pth_path /path/wide_resnet50_2-95faca4d.pth - Ascend处理器环境运行 ```bash - python train.py --dataset_path ../data/ --device_id 0 --pre_ckpt_path ../pretrain/PatchCore_pretrain.ckpt --category screw > train.log 2>&1 & + python train.py --dataset_path ../data/ --pre_ckpt_path ../pretrain/PatchCore_pretrain.ckpt --category screw --device_id 0 > train.log 2>&1 & 或 - bash run_train.sh [dataset_path] [device_id] [pre_ckpt_path] [category] + bash run_train.sh [dataset_path] [pre_ckpt_path] [category] [device_id] ``` 上述python命令将在后台运行,您可以通过train.log文件查看结果。 训练结束后,您可在生成的相应类别embeddings目录下找到faiss.index文件,该文件在评估或推理阶段使用,不需要移动。在相应类别的sample目录下可找到异常图文件。 + 对于Mvtec数据集,可以通过执行以下命令,进行Mvtec中全部类别数据的训练与推理。 + + ```bash + bash run_all_mvtec.sh [dataset_path] [pre_ckpt_path] [device_id] + ``` + ## 评估过程 ### 评估 diff --git a/research/cv/PatchCore/ascend_310_infer/CMakeLists.txt b/official/cv/patchcore/ascend_310_infer/CMakeLists.txt similarity index 92% rename from research/cv/PatchCore/ascend_310_infer/CMakeLists.txt rename to official/cv/patchcore/ascend_310_infer/CMakeLists.txt index d3fa58018b176831ab537342555b0552c5ceeb5d..0b6753852e9c63bf329dd0ef1d305201b31531d7 100644 --- a/research/cv/PatchCore/ascend_310_infer/CMakeLists.txt +++ b/official/cv/patchcore/ascend_310_infer/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14.1) project(Ascend310Infer) add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -std=c++17 -Werror -Wall -fPIE -Wl,--allow-shlib-undefined") -set(PROJECT_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR}/) +set(PROJECT_SRC_ROOT .) option(MINDSPORE_PATH "mindspore install path" "") include_directories(${MINDSPORE_PATH}) include_directories(${MINDSPORE_PATH}/include) diff --git a/research/cv/PatchCore/ascend_310_infer/build.sh b/official/cv/patchcore/ascend_310_infer/build.sh similarity index 100% rename from research/cv/PatchCore/ascend_310_infer/build.sh rename to official/cv/patchcore/ascend_310_infer/build.sh diff --git a/research/cv/PatchCore/ascend_310_infer/inc/utils.h b/official/cv/patchcore/ascend_310_infer/inc/utils.h similarity index 100% rename from research/cv/PatchCore/ascend_310_infer/inc/utils.h rename to official/cv/patchcore/ascend_310_infer/inc/utils.h diff --git a/research/cv/PatchCore/ascend_310_infer/src/main.cc b/official/cv/patchcore/ascend_310_infer/src/main.cc similarity index 100% rename from research/cv/PatchCore/ascend_310_infer/src/main.cc rename to official/cv/patchcore/ascend_310_infer/src/main.cc diff --git a/research/cv/PatchCore/ascend_310_infer/src/utils.cc b/official/cv/patchcore/ascend_310_infer/src/utils.cc similarity index 100% rename from research/cv/PatchCore/ascend_310_infer/src/utils.cc rename to official/cv/patchcore/ascend_310_infer/src/utils.cc diff --git a/research/cv/PatchCore/eval.py b/official/cv/patchcore/eval.py similarity index 96% rename from research/cv/PatchCore/eval.py rename to official/cv/patchcore/eval.py index f6588e46188ea18e1524ffccad88a50a4bfe5743..059d827155df9270954b33bfd4310ff5d6cafa3a 100644 --- a/research/cv/PatchCore/eval.py +++ b/official/cv/patchcore/eval.py @@ -43,6 +43,7 @@ parser.add_argument('--pre_ckpt_path', type=str, required=True, help='Pretrain c args = parser.parse_args() if __name__ == '__main__': + current_path = os.path.abspath(os.path.dirname(__file__)) context.set_context(mode=context.GRAPH_MODE, device_target='Ascend', device_id=args.device_id) # dataset @@ -55,7 +56,7 @@ if __name__ == '__main__': data_iter = test_dataset.create_dict_iterator() step_size = test_dataset.get_dataset_size() - embedding_dir_path, sample_path = prep_dirs('./', args.category) + embedding_dir_path, sample_path = prep_dirs(current_path, args.category) index = faiss.read_index(os.path.join(embedding_dir_path, 'index.faiss')) # network diff --git a/research/cv/PatchCore/export.py b/official/cv/patchcore/export.py similarity index 100% rename from research/cv/PatchCore/export.py rename to official/cv/patchcore/export.py diff --git a/research/cv/PatchCore/picture/PatchCore.png b/official/cv/patchcore/picture/PatchCore.png similarity index 100% rename from research/cv/PatchCore/picture/PatchCore.png rename to official/cv/patchcore/picture/PatchCore.png diff --git a/research/cv/PatchCore/postprocess.py b/official/cv/patchcore/postprocess.py similarity index 100% rename from research/cv/PatchCore/postprocess.py rename to official/cv/patchcore/postprocess.py diff --git a/research/cv/PatchCore/preprocess.py b/official/cv/patchcore/preprocess.py similarity index 100% rename from research/cv/PatchCore/preprocess.py rename to official/cv/patchcore/preprocess.py diff --git a/research/cv/PatchCore/requirements.txt b/official/cv/patchcore/requirements.txt similarity index 100% rename from research/cv/PatchCore/requirements.txt rename to official/cv/patchcore/requirements.txt diff --git a/research/cv/PatchCore/scripts/run_310_infer.sh b/official/cv/patchcore/scripts/run_310_infer.sh similarity index 100% rename from research/cv/PatchCore/scripts/run_310_infer.sh rename to official/cv/patchcore/scripts/run_310_infer.sh diff --git a/official/cv/patchcore/scripts/run_all_mvtec.sh b/official/cv/patchcore/scripts/run_all_mvtec.sh new file mode 100644 index 0000000000000000000000000000000000000000..5d4634d8e99307c34180d937229b6912df6c3e34 --- /dev/null +++ b/official/cv/patchcore/scripts/run_all_mvtec.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 3 ] +then + echo "==============================================================================================================" + echo "Please run the script as: " + echo "bash run_all_mvtec.sh DATA_PATH DEVICE_ID PRETRAINED_PATH CATEGORY" + echo "For example: bash run_all_mvtec.sh /path/dataset /path/pretrained_path 0" + echo "It is better to use the absolute path." + echo "==============================================================================================================" +exit 1 +fi +set -e + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} +DATA_PATH=$(get_real_path $1) +CKPT_APTH=$(get_real_path $2) +export DATA_PATH=$DATA_PATH + +arr=("bottle" "cable" "capsule" "capsule" "carpet" "grid" "hazelnut" "leather" "metal_nut" "pill" "screw" "tile" "toothbrush" "transistor" "wood" "zipper") + +for value in "${arr[@]}" +do + bash run_standalone_train.sh $DATA_PATH $CKPT_APTH $value $3 + bash run_eval.sh $DATA_PATH $CKPT_APTH $value $3 +done + +img_auc=$(grep "auc" eval_*/eval.log | awk -F "img_auc:" '{print $2}' | awk -F "," '{print $1}' | awk '{sum+=$1}END{print sum/NR}' | awk '{printf("%.3f", $1)}') +echo "[INFO] average img_auc = $img_auc" + +pixel_auc=$(grep "auc" eval_*/eval.log | awk -F "img_auc:" '{print $2}' | awk -F "pixel_auc:" '{print $2}' | awk '{sum+=$1}END{print sum/NR}' | awk '{printf("%.3f", $1)}') +echo "[INFO] average pixel_auc = $pixel_auc" diff --git a/official/cv/patchcore/scripts/run_eval.sh b/official/cv/patchcore/scripts/run_eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..8c8b71532e8c02545e272c19f1c51323d9888d8f --- /dev/null +++ b/official/cv/patchcore/scripts/run_eval.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 4 ] +then + echo "==============================================================================================================" + echo "Please run the script as: " + echo "bash run_eval.sh DATA_PATH DEVICE_ID PRETRAINED_PATH CATEGORY" + echo "For example: bash run_eval.sh /path/dataset /path/pretrained_path category 0" + echo "It is better to use the absolute path." + echo "==============================================================================================================" +exit 1 +fi +set -e + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} +DATA_PATH=$(get_real_path $1) +CKPT_APTH=$(get_real_path $2) +export DATA_PATH=$DATA_PATH + +eval_path=eval_$3 +if [ -d $eval_path ]; +then + rm -rf ./$eval_path +fi +mkdir ./$eval_path +cd ./$eval_path +env > env0.log +echo "[INFO] start eval dataset $3." +python ../../eval.py --dataset_path $DATA_PATH --pre_ckpt_path $CKPT_APTH --category $3 --device_id $4 &> eval.log + +if [ $? -eq 0 ];then + echo "[INFO] eval success" +else + echo "[ERROR] eval failed" + exit 2 +fi +echo "[INFO] finish" +cd ../ diff --git a/official/cv/patchcore/scripts/run_standalone_train.sh b/official/cv/patchcore/scripts/run_standalone_train.sh new file mode 100644 index 0000000000000000000000000000000000000000..3fe96d0002278ef59ad24f85423e6f695843aba8 --- /dev/null +++ b/official/cv/patchcore/scripts/run_standalone_train.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 4 ] +then + echo "==============================================================================================================" + echo "Please run the script as: " + echo "bash run_train.sh DATA_PATH DEVICE_ID PRETRAINED_PATH CATEGORY" + echo "For example: bash run_train.sh /path/dataset /path/pretrained_path category 0" + echo "It is better to use the absolute path." + echo "==============================================================================================================" +exit 1 +fi +set -e + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} +DATA_PATH=$(get_real_path $1) +CKPT_APTH=$(get_real_path $2) +export DATA_PATH=$DATA_PATH + +train_path=train_$3 +if [ -d $train_path ]; +then + rm -rf ./$train_path +fi +mkdir ./$train_path +cd ./$train_path +env > env0.log +echo "[INFO] start train dataset $3." +python ../../train.py --dataset_path $DATA_PATH --isModelArts False --pre_ckpt_path $CKPT_APTH --category $3 --device_id $4 &> train.log + +if [ $? -eq 0 ];then + echo "[INFO] training success" +else + echo "[ERROR] training failed" + exit 2 +fi +echo "[INFO] finish" +cd ../ diff --git a/research/cv/PatchCore/src/config.py b/official/cv/patchcore/src/config.py similarity index 100% rename from research/cv/PatchCore/src/config.py rename to official/cv/patchcore/src/config.py diff --git a/research/cv/PatchCore/src/dataset.py b/official/cv/patchcore/src/dataset.py similarity index 100% rename from research/cv/PatchCore/src/dataset.py rename to official/cv/patchcore/src/dataset.py diff --git a/research/cv/PatchCore/src/model.py b/official/cv/patchcore/src/model.py similarity index 100% rename from research/cv/PatchCore/src/model.py rename to official/cv/patchcore/src/model.py diff --git a/research/cv/PatchCore/src/oneStep.py b/official/cv/patchcore/src/oneStep.py similarity index 100% rename from research/cv/PatchCore/src/oneStep.py rename to official/cv/patchcore/src/oneStep.py diff --git a/research/cv/PatchCore/src/operator.py b/official/cv/patchcore/src/operator.py similarity index 100% rename from research/cv/PatchCore/src/operator.py rename to official/cv/patchcore/src/operator.py diff --git a/research/cv/PatchCore/src/pthtockpt.py b/official/cv/patchcore/src/pthtockpt.py similarity index 100% rename from research/cv/PatchCore/src/pthtockpt.py rename to official/cv/patchcore/src/pthtockpt.py diff --git a/research/cv/PatchCore/src/sampling_methods/kcenter_greedy.py b/official/cv/patchcore/src/sampling_methods/kcenter_greedy.py similarity index 100% rename from research/cv/PatchCore/src/sampling_methods/kcenter_greedy.py rename to official/cv/patchcore/src/sampling_methods/kcenter_greedy.py diff --git a/research/cv/PatchCore/src/sampling_methods/sampling_def.py b/official/cv/patchcore/src/sampling_methods/sampling_def.py similarity index 100% rename from research/cv/PatchCore/src/sampling_methods/sampling_def.py rename to official/cv/patchcore/src/sampling_methods/sampling_def.py diff --git a/research/cv/PatchCore/train.py b/official/cv/patchcore/train.py similarity index 97% rename from research/cv/PatchCore/train.py rename to official/cv/patchcore/train.py index fe052cbc00e8b482dd7e43588dce3000eb56f50d..a607dcc4635e24470208719081a76972b2a02155 100644 --- a/research/cv/PatchCore/train.py +++ b/official/cv/patchcore/train.py @@ -52,6 +52,7 @@ if args.isModelArts: import moxing as mox if __name__ == '__main__': + current_path = os.path.abspath(os.path.dirname(__file__)) context.set_context(mode=context.GRAPH_MODE, device_target='Ascend', save_graphs=False) if args.isModelArts: device_id = int(os.getenv('DEVICE_ID')) @@ -69,7 +70,7 @@ if __name__ == '__main__': embedding_dir_path, _ = prep_dirs(prep_path, args.category) else: train_dataset, _, _, _ = createDataset(args.dataset_path, args.category) - embedding_dir_path, _ = prep_dirs('./', args.category) + embedding_dir_path, _ = prep_dirs(current_path, args.category) # network network = wide_resnet50_2() diff --git a/research/cv/PatchCore/scripts/run_eval.sh b/research/cv/PatchCore/scripts/run_eval.sh deleted file mode 100644 index 8ffcadf2c5b5ea38929f2b6fededaeaa9dc58af4..0000000000000000000000000000000000000000 --- a/research/cv/PatchCore/scripts/run_eval.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -echo "==============================================================================================================" -echo "Please run the script as: " -echo "bash run_eval.sh DATA_PATH DEVICE_ID PRETRAINED_PATH CATEGORY" -echo "For example: bash run_eval.sh /path/dataset 0 /path/pretrained_path category" -echo "It is better to use the absolute path." -echo "==============================================================================================================" -set -e -DATA_PATH=$1 - -export DATA_PATH=${DATA_PATH} - -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python - -cd ../ -env > env0.log -python3 eval.py --dataset_path $1 --device_id $2 --pre_ckpt_path $3 --category $4 > eval_$4.log 2>&1 - -if [ $? -eq 0 ];then - echo "training success" -else - echo "training failed" - exit 2 -fi -echo "finish" -cd ../ diff --git a/research/cv/PatchCore/scripts/run_train.sh b/research/cv/PatchCore/scripts/run_train.sh deleted file mode 100644 index 6cbc25babae597bf403058c156b8c7b0d6b78351..0000000000000000000000000000000000000000 --- a/research/cv/PatchCore/scripts/run_train.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -echo "==============================================================================================================" -echo "Please run the script as: " -echo "bash run_train.sh DATA_PATH DEVICE_ID PRETRAINED_PATH CATEGORY" -echo "For example: bash run_train.sh /path/dataset 0 /path/pretrained_path category" -echo "It is better to use the absolute path." -echo "==============================================================================================================" -set -e -DATA_PATH=$1 - -export DATA_PATH=${DATA_PATH} - -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python - -cd ../ -env > env0.log -python3 train.py --dataset_path $1 --isModelArts False --device_id $2 --pre_ckpt_path $3 --category $4 > train_$4.log 2>&1 - -if [ $? -eq 0 ];then - echo "training success" -else - echo "training failed" - exit 2 -fi -echo "finish" -cd ../