From c977d56157f76358053f6ad3f432ad4baf3ecd27 Mon Sep 17 00:00:00 2001
From: lihaoyang <li.haoyang@huawei.com>
Date: Tue, 27 Sep 2022 15:33:56 +0800
Subject: [PATCH] Remove TB-Net code. TB-Net is moved to MindSpore XAI.
---
official/recommend/tbnet/README.md | 361 +--------------
official/recommend/tbnet/README_CN.md | 362 +--------------
.../tbnet/ascend310_infer/CMakeLists.txt | 14 -
.../recommend/tbnet/ascend310_infer/build.sh | 29 --
.../tbnet/ascend310_infer/inc/utils.h | 32 --
.../tbnet/ascend310_infer/src/main.cc | 156 -------
.../tbnet/ascend310_infer/src/utils.cc | 129 ------
official/recommend/tbnet/default_config.yaml | 31 --
official/recommend/tbnet/eval.py | 72 ---
official/recommend/tbnet/export.py | 126 -----
official/recommend/tbnet/infer.py | 169 -------
.../recommend/tbnet/infer/convert/air2om.sh | 25 -
.../tbnet/infer/data/config/tbnet.pipeline | 70 ---
.../tbnet/infer/docker_start_infer.sh | 49 --
.../tbnet/infer/mxbase/CMakeLists.txt | 35 --
.../recommend/tbnet/infer/mxbase/build.sh | 48 --
.../tbnet/infer/mxbase/src/Tbnet.cpp | 273 -----------
.../recommend/tbnet/infer/mxbase/src/Tbnet.h | 61 ---
.../recommend/tbnet/infer/mxbase/src/main.cpp | 58 ---
official/recommend/tbnet/infer/sdk/main.py | 113 -----
.../tbnet/infer/sdk/prec/postprocess.py | 90 ----
official/recommend/tbnet/infer/sdk/run.sh | 27 --
.../tbnet/modelarts/train_modelarts.py | 158 -------
official/recommend/tbnet/postprocess.py | 88 ----
official/recommend/tbnet/preprocess.py | 142 ------
.../recommend/tbnet/preprocess_dataset.py | 82 ----
official/recommend/tbnet/requirements.txt | 1 -
.../recommend/tbnet/scripts/docker_start.sh | 38 --
official/recommend/tbnet/scripts/run_eval.sh | 46 --
.../recommend/tbnet/scripts/run_infer_310.sh | 116 -----
.../tbnet/scripts/run_standalone_train.sh | 45 --
official/recommend/tbnet/src/aggregator.py | 151 ------
official/recommend/tbnet/src/config.py | 43 --
official/recommend/tbnet/src/dataset.py | 148 ------
official/recommend/tbnet/src/embedding.py | 75 ---
official/recommend/tbnet/src/metrics.py | 80 ----
official/recommend/tbnet/src/path_gen.py | 435 ------------------
official/recommend/tbnet/src/steam.py | 61 ---
official/recommend/tbnet/src/tbnet.py | 383 ---------------
.../recommend/tbnet/src/utils/__init__.py | 0
.../tbnet/src/utils/device_adapter.py | 26 --
.../tbnet/src/utils/local_adapter.py | 37 --
.../tbnet/src/utils/moxing_adapter.py | 132 ------
official/recommend/tbnet/src/utils/param.py | 132 ------
official/recommend/tbnet/train.py | 118 -----
45 files changed, 10 insertions(+), 4857 deletions(-)
delete mode 100644 official/recommend/tbnet/ascend310_infer/CMakeLists.txt
delete mode 100644 official/recommend/tbnet/ascend310_infer/build.sh
delete mode 100644 official/recommend/tbnet/ascend310_infer/inc/utils.h
delete mode 100644 official/recommend/tbnet/ascend310_infer/src/main.cc
delete mode 100644 official/recommend/tbnet/ascend310_infer/src/utils.cc
delete mode 100644 official/recommend/tbnet/default_config.yaml
delete mode 100644 official/recommend/tbnet/eval.py
delete mode 100644 official/recommend/tbnet/export.py
delete mode 100644 official/recommend/tbnet/infer.py
delete mode 100644 official/recommend/tbnet/infer/convert/air2om.sh
delete mode 100644 official/recommend/tbnet/infer/data/config/tbnet.pipeline
delete mode 100644 official/recommend/tbnet/infer/docker_start_infer.sh
delete mode 100644 official/recommend/tbnet/infer/mxbase/CMakeLists.txt
delete mode 100644 official/recommend/tbnet/infer/mxbase/build.sh
delete mode 100644 official/recommend/tbnet/infer/mxbase/src/Tbnet.cpp
delete mode 100644 official/recommend/tbnet/infer/mxbase/src/Tbnet.h
delete mode 100644 official/recommend/tbnet/infer/mxbase/src/main.cpp
delete mode 100644 official/recommend/tbnet/infer/sdk/main.py
delete mode 100644 official/recommend/tbnet/infer/sdk/prec/postprocess.py
delete mode 100644 official/recommend/tbnet/infer/sdk/run.sh
delete mode 100644 official/recommend/tbnet/modelarts/train_modelarts.py
delete mode 100644 official/recommend/tbnet/postprocess.py
delete mode 100644 official/recommend/tbnet/preprocess.py
delete mode 100644 official/recommend/tbnet/preprocess_dataset.py
delete mode 100644 official/recommend/tbnet/requirements.txt
delete mode 100644 official/recommend/tbnet/scripts/docker_start.sh
delete mode 100644 official/recommend/tbnet/scripts/run_eval.sh
delete mode 100644 official/recommend/tbnet/scripts/run_infer_310.sh
delete mode 100644 official/recommend/tbnet/scripts/run_standalone_train.sh
delete mode 100644 official/recommend/tbnet/src/aggregator.py
delete mode 100644 official/recommend/tbnet/src/config.py
delete mode 100644 official/recommend/tbnet/src/dataset.py
delete mode 100644 official/recommend/tbnet/src/embedding.py
delete mode 100644 official/recommend/tbnet/src/metrics.py
delete mode 100644 official/recommend/tbnet/src/path_gen.py
delete mode 100644 official/recommend/tbnet/src/steam.py
delete mode 100644 official/recommend/tbnet/src/tbnet.py
delete mode 100644 official/recommend/tbnet/src/utils/__init__.py
delete mode 100644 official/recommend/tbnet/src/utils/device_adapter.py
delete mode 100644 official/recommend/tbnet/src/utils/local_adapter.py
delete mode 100644 official/recommend/tbnet/src/utils/moxing_adapter.py
delete mode 100644 official/recommend/tbnet/src/utils/param.py
delete mode 100644 official/recommend/tbnet/train.py
diff --git a/official/recommend/tbnet/README.md b/official/recommend/tbnet/README.md
index 84213e62d..9ccc7162c 100644
--- a/official/recommend/tbnet/README.md
+++ b/official/recommend/tbnet/README.md
@@ -1,360 +1,9 @@
-# Contents
+# TB-Net Description
-- [Contents](#contents)
- - [TBNet Description](#tbnet-description)
- - [Model Architecture](#model-architecture)
- - [Dataset](#dataset)
- - [Environment Requirements](#environment-requirements)
- - [Quick Start](#quick-start)
- - [Script Description](#script-description)
- - [Script and Sample Code](#script-and-sample-code)
- - [Script Parameters](#script-parameters)
- - [Inference Process](#inference-process)
- - [Export MindIR](#export-mindir)
- - [Infer on Ascend310](#infer-on-ascend310)
- - [Result](#result)
- - [Model Description](#model-description)
- - [Performance](#performance)
- - [Training Performance](#training-performance)
- - [Evaluation Performance](#evaluation-performance)
- - [Inference and Explanation Performance](#inference-explanation-performance)
- - [Description of Random Situation](#description-of-random-situation)
- - [ModelZoo Homepage](#modelzoo-homepage)
+TB-Net is a knowledge graph based explainable recommender system. The tutorial and code are released in MindSpore XAI repository.
-# [TBNet Description](#contents)
+Click [here](https://www.mindspore.cn/xai/docs/en/master/using_tbnet.html) to check the tutorial of using whitebox ecommendation model TB-Net.
-TB-Net is a knowledge graph based explainable recommender system.
+Click [here](https://gitee.com/mindspore/xai/tree/master/models/whitebox/tbnet) to check TB-Net code.
-Paper: Shendi Wang, Haoyang Li, Xiao-Hui Li, Caleb Chen Cao, Lei Chen. Tower Bridge Net (TB-Net): Bidirectional Knowledge Graph Aware Embedding Propagation for Explainable Recommender Systems
-
-# [Model Architecture](#contents)
-
-TB-Net constructs subgraphs in knowledge graph based on the interaction between users and items as well as the feature of items, and then calculates paths in the graphs using bidirectional conduction algorithm. Finally we can obtain explainable recommendation results.
-
-# [Dataset](#contents)
-
-[Interaction of users and games](https://www.kaggle.com/tamber/steam-video-games), and the [games' feature data](https://www.kaggle.com/nikdavis/steam-store-games?select=steam.csv) on the game platform Steam are public on Kaggle.
-
-Dataset directory: `./data/{DATASET}/`, e.g. `./data/steam/`.
-
-- train: train.csv, evaluation: test.csv
-
-Each line indicates a \<user\>, an \<item\>, the user-item \<rating\> (1 or 0), and PER_ITEM_NUM_PATHS paths between the item and the user's \<hist_item\> (\<hist_item\> is the item whose the user-item \<rating\> in historical data is 1).
-
-```text
-#format:user,item,rating,relation1,entity,relation2,hist_item,relation1,entity,relation2,hist_item,...,relation1,entity,relation2,hist_item # module [relation1,entity,relation2,hist_item] repeats PER_ITEM_NUM_PATHS times
-```
-
-- infer and explain: infer.csv
-
-Each line indicates the \<user\> and \<item\> to be inferred, \<rating\>, and PER_ITEM_NUM_PATHS paths between the item and the user's \<hist_item\> (\<hist_item\> is the item whose the user-item \<rating\> in historical data is 1).
-Note that the \<item\> needs to traverse candidate items (all items by default) in the dataset. \<rating\> can be randomly assigned (all values are assigned to 0 by default) and is not used in the inference and explanation phases.
-
-```text
-#format:user,item,rating,relation1,entity,relation2,hist_item,relation1,entity,relation2,hist_item,...,relation1,entity,relation2,hist_item # module [relation1,entity,relation2,hist_item] repeats PER_ITEM_NUM_PATHS times
-```
-
-We have to download the data package and put it underneath the current project path銆�
-
-```bash
-wget https://mindspore-website.obs.myhuaweicloud.com/notebook/datasets/xai/tbnet_data.tar.gz
-tar -xf tbnet_data.tar.gz
-```
-
-# [Environment Requirements](#contents)
-
-- Hardware锛圢VIDIA GPU or Ascend NPU锛�
- - Prepare hardware environment with NVIDIA GPU or Ascend NPU processor.
-- Framework
- - [MindSpore](https://www.mindspore.cn/install/en)
-- For more information, please check the resources below锛�
- - [MindSpore Tutorials](https://www.mindspore.cn/tutorials/en/master/index.html)
- - [MindSpore Python API](https://www.mindspore.cn/docs/en/master/index.html)
-
-# [Quick Start](#contents)
-
-After installing MindSpore via the official website, you can start training and evaluation as follows:
-
-- Data preprocessing
-
-Download the data package(e.g. 'steam' dataset) and put it underneath the current project path.
-
-```bash
-wget https://mindspore-website.obs.myhuaweicloud.com/notebook/datasets/xai/tbnet_data.tar.gz
-tar -xf tbnet_data.tar.gz
-```
-
-and then run code as follows.
-
-- Training
-
-```bash
-bash scripts/run_standalone_train.sh [DATA_NAME] [DEVICE_ID] [DEVICE_TARGET]
-```
-
-Example:
-
-```bash
-bash scripts/run_standalone_train.sh steam 0 Ascend
-```
-
-- Evaluation
-
-Evaluation model on test dataset.
-
-```bash
-bash scripts/run_eval.sh [CHECKPOINT_ID] [DATA_NAME] [DEVICE_ID] [DEVICE_TARGET]
-```
-
-Argument `[CHECKPOINT_ID]` is required.
-
-Example:
-
-```bash
-bash scripts/run_eval.sh 19 steam 0 Ascend
-```
-
-- Inference and Explanation
-
-Recommende items to user acrodding to `user`, the number of items is determined by `items`.
-
-```bash
-python infer.py \
- --dataset [DATASET] \
- --checkpoint_id [CHECKPOINT_ID] \
- --user [USER] \
- --items [ITEMS] \
- --explanations [EXPLANATIONS] \
- --csv [CSV] \
- --device_target [DEVICE_TARGET]
-```
-
-Arguments `--checkpoint_id` and `--user` are required.
-
-Example:
-
-```bash
-python infer.py \
- --dataset steam \
- --checkpoint_id 19 \
- --user 2 \
- --items 1 \
- --explanations 3 \
- --csv test.csv \
- --device_target Ascend
-```
-
-# [Script Description](#contents)
-
-## [Script and Sample Code](#contents)
-
-```text
-.
-鈹斺攢tbnet
- 鈹溾攢README.md
- 鈹溾攢鈹€ scripts
- 鈹溾攢run_infer_310.sh # Ascend310 inference script
- 鈹溾攢run_standalone_train.sh # NVIDIA GPU or Ascend NPU training script
- 鈹斺攢run_eval.sh # NVIDIA GPU or Ascend NPU evaluation script
- 鈹溾攢data
- 鈹溾攢steam
- 鈹溾攢config.json # data and training parameter configuration
- 鈹溾攢src_infer.csv # inference and explanation dataset
- 鈹溾攢src_test.csv # evaluation dataset
- 鈹溾攢src_train.csv # training dataset
- 鈹斺攢id_maps.json # explanation configuration
- 鈹溾攢src
- 鈹溾攢utils
- 鈹溾攢__init__.py # init file
- 鈹溾攢device_adapter.py # Get cloud ID
- 鈹溾攢local_adapter.py # Get local ID
- 鈹溾攢moxing_adapter.py # Parameter processing
- 鈹斺攢param.py # parse args
- 鈹溾攢aggregator.py # inference result aggregation
- 鈹溾攢config.py # parsing parameter configuration
- 鈹溾攢dataset.py # generate dataset
- 鈹溾攢embedding.py # 3-dim embedding matrix initialization
- 鈹溾攢metrics.py # model metrics
- 鈹溾攢steam.py # 'steam' dataset text explainer
- 鈹斺攢tbnet.py # TB-Net model
- 鈹溾攢export.py # export mindir script
- 鈹溾攢preprocess_dataset.py # dataset preprocess script
- 鈹溾攢preprocess.py # inference data preprocess script
- 鈹溾攢postprocess.py # inference result calculation script
- 鈹溾攢eval.py # evaluation
- 鈹溾攢infer.py # inference and explanation
- 鈹斺攢train.py # training
-```
-
-## [Script Parameters](#contents)
-
-The entire code structure is as following:
-
-```python
-data_path: "." # The location of input data
-load_path: "./checkpoint" # file path of stored checkpoint file in training
-checkpoint_id: 19 # checkpoint id
-same_relation: False # only generate paths that relation1 is same as relation2
-dataset: "steam" # dataset name
-train_csv: "train.csv" # the train csv datafile inside the dataset folder
-test_csv: "test.csv" # the test csv datafile inside the dataset folder
-infer_csv: "infer.csv" # the infer csv datafile inside the dataset folder
-device_id: 0 # Device id
-device_target: "GPU" # device id of GPU or Ascend
-run_mode: "graph" # run code by GRAPH mode or PYNATIVE mode
-epochs: 20 # number of training epochs
-```
-
-- preprocess_dataset.py parameters
-
-```text
---dataset 'steam' dataset is supported currently
---device_target run code on GPU or Ascend NPU
---same_relation only generate paths that relation1 is same as relation2
-```
-
-- train.py parameters
-
-```text
---dataset 'steam' dataset is supported currently
---train_csv the train csv datafile inside the dataset folder
---test_csv the test csv datafile inside the dataset folder
---device_id device id
---epochs number of training epochs
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-- eval.py parameters
-
-```text
---dataset 'steam' dataset is supported currently
---csv the csv datafile inside the dataset folder (e.g. test.csv)
---checkpoint_id use which checkpoint(.ckpt) file to eval
---device_id device id
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-- infer.py parameters
-
-```text
---dataset 'steam' dataset is supported currently
---csv the csv datafile inside the dataset folder (e.g. infer.csv)
---checkpoint_id use which checkpoint(.ckpt) file to infer
---user id of the user to be recommended to
---items no. of items to be recommended
---reasons no. of recommendation reasons to be shown
---device_id device id
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-## [Inference Process](#contents)
-
-### [Export MindIR](#contents)
-
-```shell
-python export.py --config_path [CONFIG_PATH] --checkpoint_path [CKPT_PATH] --device_target [DEVICE] --file_name [FILE_NAME] --file_format [FILE_FORMAT]
-```
-
-- `CKPT_PATH` parameter is required.
-- `CONFIG_PATH` is `config.json` file, data and training parameter configuration.
-- `DEVICE` should be in ['Ascend', 'GPU'].
-- `FILE_FORMAT` should be in ['MINDIR', 'AIR'].
-
-Example锛�
-
-```bash
-python export.py \
- --config_path ./data/steam/config.json \
- --checkpoint_path ./checkpoints/tbnet_epoch19.ckpt \
- --device_target Ascend \
- --file_name model \
- --file_format MINDIR
-```
-
-### [Infer on Ascend310](#contents)
-
-Before performing inference, the mindir file must be exported by `export.py` script. We only provide an example of inference using MINDIR model.
-
-```shell
-# Ascend310 inference
-cd scripts
-bash run_infer_310.sh [MINDIR_PATH] [DATA_PATH] [DEVICE_ID]
-```
-
-- `MINDIR_PATH` specifies path of used "MINDIR" model.
-- `DATA_PATH` specifies path of test.csv.
-- `DEVICE_ID` is optional, default value is 0.
-
-Example锛�
-
-```bash
-cd scripts
-bash run_infer_310.sh ../model.mindir ../data/steam/test.csv 0
-```
-
-### [Result](#contents)
-
-Inference result is saved in current path, you can find result like this in acc.log file.
-
-```bash
-auc: 0.8251359368836292
-```
-
-# [Model Description](#contents)
-
-## [Performance](#contents)
-
-### Training Performance
-
-| Parameters | GPU | Ascend NPU |
-| -------------------------- |--------------------------------------------------------------------------------------------| ---------------------------------------------|
-| Model Version | TB-Net | TB-Net |
-| Resource | NVIDIA RTX 3090 | Ascend 910 |
-| Uploaded Date | 2022-07-14 | 2022-06-30 |
-| MindSpore Version | 1.6.1 | 1.6.1 |
-| Dataset | steam | steam |
-| Training Parameter | epoch=20, batch_size=1024, lr=0.001 | epoch=20, batch_size=1024, lr=0.001 |
-| Optimizer | Adam | Adam |
-| Loss Function | Sigmoid Cross Entropy | Sigmoid Cross Entropy |
-| Outputs | AUC=0.8573锛孉ccuracy=0.7733 | AUC=0.8592锛屽噯纭巼=0.7741 |
-| Loss | 0.57 | 0.59 |
-| Speed | 1pc: 90ms/step | 鍗曞崱锛�80姣/姝� |
-| Total Time | 1pc: 297s | 鍗曞崱锛�336绉� |
-| Checkpoint for Fine Tuning | 686.3K (.ckpt file) | 671K (.ckpt 鏂囦欢) |
-| Scripts | [TB-Net scripts](https://gitee.com/mindspore/models/tree/master/official/recommend/tbnet) |
-
-### Evaluation Performance
-
-| Parameters | GPU | Ascend NPU |
-| ------------------------- |----------------------------| ----------------------------- |
-| Model Version | TB-Net | TB-Net |
-| Resource | NVIDIA RTX 3090 | Ascend 910 |
-| Uploaded Date | 2022-07-14 | 2022-06-30 |
-| MindSpore Version | 1.3.0 | 1.5.1 |
-| Dataset | steam | steam |
-| Batch Size | 1024 | 1024 |
-| Outputs | AUC=0.8487锛孉ccuracy=0.7699 | AUC=0.8486锛孉ccuracy=0.7704 |
-| Total Time | 1pc: 5.7s | 1pc: 1.1绉� |
-
-### Inference and Explanation Performance
-
-| Parameters | GPU |
-| --------------------------| ------------------------------------- |
-| Model Version | TB-Net |
-| Resource | Tesla V100-SXM2-32GB |
-| Uploaded Date | 2021-08-01 |
-| MindSpore Version | 1.3.0 |
-| Dataset | steam |
-| Outputs | Recommendation Result and Explanation |
-| Total Time | 1pc: 3.66s |
-
-# [Description of Random Situation](#contents)
-
-- Initialization of embedding matrix in `tbnet.py` and `embedding.py`.
-
-# [ModelZoo Homepage](#contents)
-
-Please check the official [homepage](https://gitee.com/mindspore/models).
\ No newline at end of file
+Click [here](https://ieeexplore.ieee.org/document/9835387) to check TB-Net paper (ICDE 2022): *Tower Bridge Net (TB-Net): Bidirectional Knowledge Graph Aware Embedding Propagation for Explainable Recommender Systems*.
\ No newline at end of file
diff --git a/official/recommend/tbnet/README_CN.md b/official/recommend/tbnet/README_CN.md
index dd6919f88..5601ffdef 100644
--- a/official/recommend/tbnet/README_CN.md
+++ b/official/recommend/tbnet/README_CN.md
@@ -1,361 +1,9 @@
-# 鐩綍
+# TB-Net姒傝堪
-<!-- TOC -->
+TB-Net鏄竴涓熀浜庣煡璇嗗浘璋辩殑鍙В閲婃帹鑽愮郴缁熴€備娇鐢ㄦ枃妗ｅ拰浠ｇ爜宸茬Щ鑷矼indSpore XAI浠撱€�
-- [鐩綍](#鐩綍)
- - [TBNet姒傝堪](#tbnet姒傝堪)
- - [妯″瀷鏋舵瀯](#妯″瀷鏋舵瀯)
- - [鏁版嵁闆哴(#鏁版嵁闆�)
- - [鐜瑕佹眰](#鐜瑕佹眰)
- - [蹇€熷叆闂╙(#蹇€熷叆闂�)
- - [鑴氭湰璇存槑](#鑴氭湰璇存槑)
- - [鑴氭湰鍜屾牱渚嬩唬鐮乚(#鑴氭湰鍜屾牱渚嬩唬鐮�)
- - [鑴氭湰鍙傛暟](#鑴氭湰鍙傛暟)
- - [鎺ㄧ悊杩囩▼](#鎺ㄧ悊杩囩▼)
- - [瀵煎嚭MindIR](#瀵煎嚭mindir)
- - [鍦ˋscend310鎵ц鎺ㄧ悊](#鍦╝scend310鎵ц鎺ㄧ悊)
- - [缁撴灉](#缁撴灉)
- - [妯″瀷鎻忚堪](#妯″瀷鎻忚堪)
- - [鎬ц兘](#鎬ц兘)
- - [璁粌鎬ц兘](#璁粌鎬ц兘)
- - [璇勪及鎬ц兘](#璇勪及鎬ц兘)
- - [鎺ㄧ悊鍜岃В閲婃€ц兘](#鎺ㄧ悊鍜岃В閲婃€ц兘)
- - [闅忔満鎯呭喌璇存槑](#闅忔満鎯呭喌璇存槑)
- - [ModelZoo涓婚〉](#modelzoo涓婚〉)
+鐐瑰嚮[姝ゅ](https://www.mindspore.cn/xai/docs/zh-CN/master/using_tbnet.html)鏌ョ湅濡備綍浣跨敤MindSpore XAI涓殑TB-Net鐧界洅鎺ㄨ崘妯″瀷銆�
-# [TBNet姒傝堪](#鐩綍)
+鐐瑰嚮[姝ゅ](https://gitee.com/mindspore/xai/tree/master/models/whitebox/tbnet)鏌ョ湅TB-Net婧愪唬鐮併€�
-TB-Net鏄竴涓熀浜庣煡璇嗗浘璋辩殑鍙В閲婃帹鑽愮郴缁熴€�
-
-璁烘枃锛歋hendi Wang, Haoyang Li, Xiao-Hui Li, Caleb Chen Cao, Lei Chen. Tower Bridge Net (TB-Net): Bidirectional Knowledge Graph Aware Embedding Propagation for Explainable Recommender Systems
-
-# [妯″瀷鏋舵瀯](#鐩綍)
-
-TB-Net灏嗙敤鎴峰拰鐗╁搧鐨勪氦浜掍俊鎭互鍙婄墿鍝佺殑灞炴€т俊鎭湪鐭ヨ瘑鍥捐氨涓瀯寤哄瓙鍥撅紝骞跺埄鐢ㄥ弻鍚戜紶瀵肩殑璁＄畻鏂规硶瀵瑰浘璋变腑鐨勮矾寰勮繘琛岃绠楋紝鏈€鍚庡緱鍒板彲瑙ｉ噴鐨勬帹鑽愮粨鏋溿€�
-
-# [鏁版嵁闆哴(#鐩綍)
-
-鏈ず渚嬫彁渚汯aggle涓婄殑Steam娓告垙骞冲彴鍏紑鏁版嵁闆嗭紝鍖呭惈[鐢ㄦ埛涓庢父鎴忕殑浜や簰璁板綍](https://www.kaggle.com/tamber/steam-video-games)鍜孾娓告垙鐨勫睘鎬т俊鎭痌(https://www.kaggle.com/nikdavis/steam-store-games?select=steam.csv)銆�
-
-鏁版嵁闆嗚矾寰勶細`./data/{DATASET}/`锛屽锛歚./data/steam/`銆�
-
-- 璁粌锛歵rain.csv锛岃瘎浼帮細test.csv
-
-姣忎竴琛岃褰曚唬琛ㄦ煇\<user\>瀵规煇\<item\>鐨刓<rating\>(1鎴�0)锛屼互鍙婅\<item\>涓嶾<hist_item\>(鍗宠\<user\>鍘嗗彶\<rating\>涓�1鐨刓<item\>)鐨凱ER_ITEM_NUM_PATHS鏉¤矾寰勩€�
-
-```text
-#format:user,item,rating,relation1,entity,relation2,hist_item,relation1,entity,relation2,hist_item,...,relation1,entity,relation2,hist_item # module [relation1,entity,relation2,hist_item] repeats PER_ITEM_NUM_PATHS times
-```
-
-- 鎺ㄧ悊鍜岃В閲婏細infer.csv
-
-姣忎竴琛岃褰曚唬琛�**寰呮帹鐞�**鐨刓<user\>鍜孿<item\>锛孿<rating\>锛屼互鍙婅\<item\>涓嶾<hist_item\>(鍗宠\<user\>鍘嗗彶\<rating\>涓�1鐨刓<item\>)鐨凱ER_ITEM_NUM_PATHS鏉¤矾寰勩€�
-鍏朵腑\<item\>闇€瑕侀亶鍘嗘暟鎹泦涓�**鎵€鏈�**寰呮帹鑽愮墿鍝侊紙榛樿鎵€鏈夌墿鍝侊級锛沑<rating\>鍙殢鏈鸿祴鍊硷紙榛樿鍏ㄩ儴璧嬪€间负0锛夛紝鍦ㄦ帹鐞嗗拰瑙ｉ噴闃舵涓嶄細浣跨敤銆�
-
-```text
-#format:user,item,rating,relation1,entity,relation2,hist_item,relation1,entity,relation2,hist_item,...,relation1,entity,relation2,hist_item # module [relation1,entity,relation2,hist_item] repeats PER_ITEM_NUM_PATHS times
-```
-
-# [鐜瑕佹眰](#鐩綍)
-
-- 纭欢锛圢VIDIA GPU or Ascend NPU锛�
- - 浣跨敤NVIDIA GPU澶勭悊鍣ㄦ垨鑰匒scend NPU澶勭悊鍣ㄥ噯澶囩‖浠剁幆澧冦€�
-- 妗嗘灦
- - [MindSpore](https://www.mindspore.cn/install)
-- 濡傞渶鏌ョ湅璇︽儏锛岃鍙傝濡備笅璧勬簮锛�
- - [MindSpore鏁欑▼](https://www.mindspore.cn/tutorials/zh-CN/master/index.html)
- - [MindSpore Python API](https://www.mindspore.cn/docs/zh-CN/master/index.html)
-
-# [蹇€熷叆闂╙(#鐩綍)
-
-閫氳繃瀹樻柟缃戠珯瀹夎MindSpore鍚庯紝鎮ㄥ彲浠ユ寜鐓у涓嬫楠よ繘琛岃缁冦€佽瘎浼般€佹帹鐞嗗拰瑙ｉ噴锛�
-
-- 鏁版嵁鍑嗗
-
-涓嬭浇鐢ㄤ緥鏁版嵁闆嗗寘锛堜互'steam'鏁版嵁闆嗕负渚嬶級锛岃В鍘嬪埌褰撳墠椤圭洰璺緞銆�
-
-```bash
-wget https://mindspore-website.obs.myhuaweicloud.com/notebook/datasets/xai/tbnet_data.tar.gz
-tar -xf tbnet_data.tar.gz
-```
-
-鐒跺悗鎸夌収浠ヤ笅姝ラ杩愯浠ｇ爜銆�
-
-- 璁粌
-
-```bash
-bash scripts/run_standalone_train.sh [DATA_NAME] [DEVICE_ID] [DEVICE_TARGET]
-```
-
-绀轰緥锛�
-
-```bash
-bash scripts/run_standalone_train.sh steam 0 Ascend
-```
-
-- 璇勪及
-
-璇勪及妯″瀷鍦ㄦ祴璇曢泦涓婄殑鎸囨爣銆�
-
-```bash
-bash scripts/run_eval.sh [CHECKPOINT_ID] [DATA_NAME] [DEVICE_ID] [DEVICE_TARGET]
-```
-
-鍙傛暟`[CHECKPOINT_ID]`鏄繀濉」銆�
-
-绀轰緥锛�
-
-```bash
-bash scripts/run_eval.sh 19 steam 0 Ascend
-```
-
-- 鎺ㄧ悊鍜岃В閲�
-
-鏍规嵁`user`鎺ㄨ崘涓€瀹氭暟閲忕殑鐗╁搧锛屾暟閲忕敱`items`鍐冲畾銆�
-
-```bash
-python infer.py \
- --dataset [DATASET] \
- --checkpoint_id [CHECKPOINT_ID] \
- --user [USER] \
- --items [ITEMS] \
- --explanations [EXPLANATIONS] \
- --csv [CSV] \
- --device_target [DEVICE_TARGET]
-```
-
-鍙傛暟`--checkpoint_id`鍜宍--user`鏄繀濉」銆�
-
-绀轰緥锛�
-
-```bash
-python infer.py \
- --dataset steam \
- --checkpoint_id 19 \
- --user 2 \
- --items 1 \
- --explanations 3 \
- --csv test.csv \
- --device_target Ascend
-```
-
-# [鑴氭湰璇存槑](#鐩綍)
-
-## [鑴氭湰鍜屾牱渚嬩唬鐮乚(#鐩綍)
-
-```text
-.
-鈹斺攢tbnet
- 鈹溾攢README.md
- 鈹溾攢鈹€ scripts
- 鈹溾攢run_infer_310.sh # 鐢ㄤ簬Ascend310鎺ㄧ悊鐨勮剼鏈�
- 鈹溾攢run_standalone_train.sh # 鐢ㄤ簬NVIDIA GPU鎴栬€匒scend NPU璁粌鐨勮剼鏈�
- 鈹斺攢run_eval.sh # 鐢ㄤ簬NVIDIA GPU鎴栬€匒scend NPU璇勪及鐨勮剼鏈�
- 鈹溾攢data
- 鈹溾攢steam
- 鈹溾攢config.json # 鏁版嵁鍜岃缁冨弬鏁伴厤缃�
- 鈹溾攢src_infer.csv # 鎺ㄧ悊鍜岃В閲婃暟鎹泦
- 鈹溾攢src_test.csv # 娴嬭瘯鏁版嵁闆�
- 鈹溾攢src_train.csv # 璁粌鏁版嵁闆�
- 鈹斺攢id_maps.json # 杈撳嚭瑙ｉ噴鐩稿叧閰嶇疆
- 鈹溾攢src
- 鈹溾攢utils
- 鈹溾攢__init__.py # 鍒濆鍖栨枃浠�
- 鈹溾攢device_adapter.py # 鑾峰緱浜戣澶噄d
- 鈹溾攢local_adapter.py # 鑾峰緱鏈湴id
- 鈹溾攢moxing_adapter.py # 鍙傛暟澶勭悊
- 鈹斺攢param.py # 瑙ｆ瀽鍙傛暟
- 鈹溾攢aggregator.py # 鎺ㄧ悊缁撴灉鑱氬悎
- 鈹溾攢config.py # 鍙傛暟閰嶇疆瑙ｆ瀽
- 鈹溾攢dataset.py # 鍒涘缓鏁版嵁闆�
- 鈹溾攢embedding.py # 涓夌淮embedding鐭╅樀鍒濆鍖�
- 鈹溾攢metrics.py # 妯″瀷搴﹂噺
- 鈹溾攢steam.py # 'steam'鏁版嵁闆嗘枃鏈В鏋�
- 鈹斺攢tbnet.py # TB-Net缃戠粶
- 鈹溾攢export.py # 瀵煎嚭MINDIR鑴氭湰
- 鈹溾攢preprocess_dataset.py # 鏁版嵁闆嗛澶勭悊鑴氭湰
- 鈹溾攢preprocess.py # 鎺ㄧ悊鏁版嵁棰勫鐞嗚剼鏈�
- 鈹溾攢postprocess.py # 鎺ㄧ悊缁撴灉璁＄畻鑴氭湰
- 鈹溾攢default_config.yaml # yaml閰嶇疆鏂囦欢
- 鈹溾攢eval.py # 璇勪及缃戠粶
- 鈹溾攢infer.py # 鎺ㄧ悊鍜岃В閲�
- 鈹斺攢train.py # 璁粌缃戠粶
-```
-
-## [鑴氭湰鍙傛暟](#鐩綍)
-
-train.py涓巔aram.py涓昏鍙傛暟濡備笅锛�
-
-```python
-data_path: "." # 鏁版嵁闆嗚矾寰�
-load_path: "./checkpoint" # 妫€鏌ョ偣淇濆瓨璺緞
-checkpoint_id: 19 # 妫€鏌ョ偣id
-same_relation: False # 棰勫鐞嗘暟鎹泦鏃讹紝鍙敓鎴恅relation1`涓巂relation2`鐩稿悓鐨勮矾寰�
-dataset: "steam" # 鏁版嵁闆嗗悕闄�
-train_csv: "train.csv" # 鏁版嵁闆嗕腑璁粌闆嗘枃浠跺悕
-test_csv: "test.csv" # 鏁版嵁闆嗕腑娴嬭瘯闆嗘枃浠跺悕
-infer_csv: "infer.csv" # 鏁版嵁闆嗕腑鎺ㄧ悊鏁版嵁鏂囦欢鍚�
-device_id: 0 # 璁惧id
-device_target: "GPU" # 杩愯骞冲彴
-run_mode: "graph" # 杩愯妯″紡
-epochs: 20 # 璁粌杞暟
-```
-
-- preprocess_dataset.py鍙傛暟
-
-```text
---dataset 'steam' dataset is supported currently
---device_target run code on GPU or Ascend NPU
---same_relation only generate paths that relation1 is same as relation2
-```
-
-- train.py鍙傛暟
-
-```text
---dataset 'steam' dataset is supported currently
---train_csv the train csv datafile inside the dataset folder
---test_csv the test csv datafile inside the dataset folder
---device_id device id
---epochs number of training epochs
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-- eval.py鍙傛暟
-
-```text
---dataset 'steam' dataset is supported currently
---csv the csv datafile inside the dataset folder (e.g. test.csv)
---checkpoint_id use which checkpoint(.ckpt) file to eval
---device_id device id
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-- infer.py鍙傛暟
-
-```text
---dataset 'steam' dataset is supported currently
---csv the csv datafile inside the dataset folder (e.g. infer.csv)
---checkpoint_id use which checkpoint(.ckpt) file to infer
---user id of the user to be recommended to
---items no. of items to be recommended
---reasons no. of recommendation reasons to be shown
---device_id device id
---device_target run code on GPU or Ascend NPU
---run_mode run code by GRAPH mode or PYNATIVE mode
-```
-
-## 鎺ㄧ悊杩囩▼
-
-### 瀵煎嚭MindIR
-
-```shell
-python export.py \
- --config_path [CONFIG_PATH] \
- --checkpoint_path [CKPT_PATH] \
- --device_target [DEVICE] \
- --file_name [FILE_NAME] \
- --file_format [FILE_FORMAT]
-```
-
-- `CKPT_PATH` 涓哄繀濉」銆�
-- `CONFIG_PATH` 鍗虫暟鎹泦鐨刞config.json`鏂囦欢, 鍖呭惈鏁版嵁鍜岃缁冨弬鏁伴厤缃€�
-- `DEVICE` 鍙€夐」涓� ['Ascend', 'GPU']銆�
-- `FILE_FORMAT` 鍙€夐」涓� ['MINDIR', 'AIR']銆�
-
-绀轰緥锛�
-
-```bash
-python export.py \
- --config_path ./data/steam/config.json \
- --checkpoint_path ./checkpoints/tbnet_epoch19.ckpt \
- --device_target Ascend \
- --file_name model \
- --file_format MINDIR
-```
-
-### 鍦ˋscend310鎵ц鎺ㄧ悊
-
-鍦ㄦ墽琛屾帹鐞嗗墠锛宮indir鏂囦欢蹇呴』閫氳繃`export.py`鑴氭湰瀵煎嚭銆備互涓嬪睍绀轰簡浣跨敤minir妯″瀷鎵ц鎺ㄧ悊鐨勭ず渚嬨€�
-
-```shell
-# Ascend310 inference
-cd scripts
-bash run_infer_310.sh [MINDIR_PATH] [DATA_PATH] [DEVICE_ID]
-```
-
-- `MINDIR_PATH` mindir鏂囦欢璺緞
-- `DATA_PATH` 鎺ㄧ悊鏁版嵁闆唗est.csv璺緞
-- `DEVICE_ID` 鍙€夛紝榛樿鍊间负0銆�
-
-绀轰緥锛�
-
-```bash
-cd scripts
-bash run_infer_310.sh ../model.mindir ../data/steam/test.csv 0
-```
-
-### 缁撴灉
-
-鎺ㄧ悊缁撴灉淇濆瓨鍦ㄨ剼鏈墽琛岀殑褰撳墠璺緞锛屼綘鍙互鍦╝cc.log涓湅鍒颁互涓嬬簿搴﹁绠楃粨鏋溿€�
-
-```bash
-auc: 0.8251359368836292
-```
-
-# [妯″瀷鎻忚堪](#鐩綍)
-
-## [鎬ц兘](#鐩綍)
-
-### [璁粌鎬ц兘](#鐩綍)
-
-| 鍙傛暟 | GPU | Ascend NPU |
-| ------------------- |-------------------------------------------------------------------------------------|-------------------------------------|
-| 妯″瀷鐗堟湰 | TB-Net | TB-Net |
-| 璧勬簮 | NVIDIA RTX 3090 | Ascend 910 |
-| 涓婁紶鏃ユ湡 | 2022-07-14 | 2022-06-30 |
-| MindSpore鐗堟湰 | 1.6.1 | 1.6.1 |
-| 鏁版嵁闆� | steam | steam |
-| 璁粌鍙傛暟 | epoch=20, batch_size=1024, lr=0.001 | epoch=20, batch_size=1024, lr=0.001 |
-| 浼樺寲鍣� | Adam | Adam |
-| 鎹熷け鍑芥暟 | Sigmoid浜ゅ弶鐔� | Sigmoid浜ゅ弶鐔� |
-| 杈撳嚭 | AUC=0.8573锛屽噯纭巼=0.7733 | AUC=0.8592锛屽噯纭巼=0.7741 |
-| 鎹熷け | 0.57 | 0.59 |
-| 閫熷害 | 鍗曞崱锛�90姣/姝� | 鍗曞崱锛�80姣/姝� |
-| 鎬绘椂闀� | 鍗曞崱锛�297绉� | 鍗曞崱锛�336绉� |
-| 寰皟妫€鏌ョ偣 | 686.3K (.ckpt 鏂囦欢) | 671K (.ckpt 鏂囦欢) |
-| 鑴氭湰 | [TB-Net鑴氭湰](https://gitee.com/mindspore/models/tree/master/official/recommend/tbnet) |
-
-### [璇勪及鎬ц兘](#鐩綍)
-
-| 鍙傛暟 | GPU | Ascend NPU |
-| -------------------------- |-----------------------| ----------------------------- |
-| 妯″瀷鐗堟湰 | TB-Net | TB-Net |
-| 璧勬簮 | NVIDIA RTX 3090 | Ascend 910 |
-| 涓婁紶鏃ユ湡 | 2022-07-14 | 2022-06-30 |
-| MindSpore鐗堟湰 | 1.6.1 | 1.6.1 |
-| 鏁版嵁闆� | steam | steam |
-| 鎵规澶у皬 | 1024 | 1024 |
-| 杈撳嚭 | AUC=0.8487锛屽噯纭巼=0.7699 | AUC=0.8486锛屽噯纭巼=0.7704 |
-| 鎬绘椂闀� | 鍗曞崱锛�5.7绉� | 鍗曞崱锛�1.1绉� |
-
-### [鎺ㄧ悊鍜岃В閲婃€ц兘](#鐩綍)
-
-| 鍙傛暟 | GPU |
-| -------------------------- | ----------------------------- |
-| 妯″瀷鐗堟湰 | TB-Net |
-| 璧勬簮 | Tesla V100-SXM2-32GB |
-| 涓婁紶鏃ユ湡 | 2021-08-01 |
-| MindSpore鐗堟湰 | 1.3.0 |
-| 鏁版嵁闆� | steam |
-| 杈撳嚭 | 鎺ㄨ崘缁撴灉鍜岃В閲婄粨鏋� |
-| 鎬绘椂闀� | 鍗曞崱锛�3.66绉� |
-
-# [闅忔満鎯呭喌璇存槑](#鐩綍)
-
-- `tbnet.py`鍜宍embedding.py`涓璄mbedding鐭╅樀鐨勯殢鏈哄垵濮嬪寲銆�
-
-# [ModelZoo涓婚〉](#鐩綍)
-
-璇锋祻瑙堝畼缃慬涓婚〉](https://gitee.com/mindspore/models)銆�
\ No newline at end of file
+鐐瑰嚮[姝ゅ](https://ieeexplore.ieee.org/document/9835387)鏌ョ湅TB-Net璁烘枃锛圛CDE 2022锛夛細*Tower Bridge Net (TB-Net): Bidirectional Knowledge Graph Aware Embedding Propagation for Explainable Recommender Systems*銆�
\ No newline at end of file
diff --git a/official/recommend/tbnet/ascend310_infer/CMakeLists.txt b/official/recommend/tbnet/ascend310_infer/CMakeLists.txt
deleted file mode 100644
index ee3c85447..000000000
--- a/official/recommend/tbnet/ascend310_infer/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-cmake_minimum_required(VERSION 3.14.1)
-project(Ascend310Infer)
-add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -std=c++17 -Werror -Wall -fPIE -Wl,--allow-shlib-undefined")
-set(PROJECT_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR}/)
-option(MINDSPORE_PATH "mindspore install path" "")
-include_directories(${MINDSPORE_PATH})
-include_directories(${MINDSPORE_PATH}/include)
-include_directories(${PROJECT_SRC_ROOT})
-find_library(MS_LIB libmindspore.so ${MINDSPORE_PATH}/lib)
-file(GLOB_RECURSE MD_LIB ${MINDSPORE_PATH}/_c_dataengine*)
-
-add_executable(main src/main.cc src/utils.cc)
-target_link_libraries(main ${MS_LIB} ${MD_LIB} gflags)
diff --git a/official/recommend/tbnet/ascend310_infer/build.sh b/official/recommend/tbnet/ascend310_infer/build.sh
deleted file mode 100644
index 713d7f657..000000000
--- a/official/recommend/tbnet/ascend310_infer/build.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-if [ -d out ]; then
- rm -rf out
-fi
-
-mkdir out
-cd out || exit
-
-if [ -f "Makefile" ]; then
- make clean
-fi
-
-cmake .. \
- -DMINDSPORE_PATH="`pip show mindspore-ascend | grep Location | awk '{print $2"/mindspore"}' | xargs realpath`"
-make
diff --git a/official/recommend/tbnet/ascend310_infer/inc/utils.h b/official/recommend/tbnet/ascend310_infer/inc/utils.h
deleted file mode 100644
index efebe03a8..000000000
--- a/official/recommend/tbnet/ascend310_infer/inc/utils.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Copyright 2021 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef MINDSPORE_INFERENCE_UTILS_H_
-#define MINDSPORE_INFERENCE_UTILS_H_
-
-#include <sys/stat.h>
-#include <dirent.h>
-#include <vector>
-#include <string>
-#include <memory>
-#include "include/api/types.h"
-
-std::vector<std::string> GetAllFiles(std::string_view dirName);
-DIR *OpenDir(std::string_view dirName);
-std::string RealPath(std::string_view path);
-mindspore::MSTensor ReadFileToTensor(const std::string &file);
-int WriteResult(const std::string& imageFile, const std::vector<mindspore::MSTensor> &outputs);
-#endif
diff --git a/official/recommend/tbnet/ascend310_infer/src/main.cc b/official/recommend/tbnet/ascend310_infer/src/main.cc
deleted file mode 100644
index 93d22c64e..000000000
--- a/official/recommend/tbnet/ascend310_infer/src/main.cc
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Copyright 2021 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <sys/time.h>
-#include <gflags/gflags.h>
-#include <dirent.h>
-#include <iostream>
-#include <string>
-#include <algorithm>
-#include <iosfwd>
-#include <vector>
-#include <fstream>
-#include <sstream>
-
-#include "include/api/model.h"
-#include "include/api/context.h"
-#include "include/api/types.h"
-#include "include/api/serialization.h"
-#include "include/dataset/execute.h"
-#include "include/dataset/vision.h"
-#include "inc/utils.h"
-
-using mindspore::Context;
-using mindspore::Serialization;
-using mindspore::Model;
-using mindspore::Status;
-using mindspore::MSTensor;
-using mindspore::dataset::Execute;
-using mindspore::ModelType;
-using mindspore::GraphCell;
-using mindspore::kSuccess;
-
-DEFINE_string(mindir_path, "", "mindir path");
-DEFINE_string(input0_path, ".", "input0 path");
-DEFINE_string(input1_path, ".", "input1 path");
-DEFINE_string(input2_path, ".", "input2 path");
-DEFINE_string(input3_path, ".", "input3 path");
-DEFINE_string(input4_path, ".", "input4 path");
-DEFINE_string(input5_path, ".", "input5 path");
-DEFINE_int32(device_id, 0, "device id");
-
-int main(int argc, char **argv) {
- gflags::ParseCommandLineFlags(&argc, &argv, true);
- if (RealPath(FLAGS_mindir_path).empty()) {
- std::cout << "Invalid mindir" << std::endl;
- return 1;
- }
-
- auto context = std::make_shared<Context>();
- auto ascend310 = std::make_shared<mindspore::Ascend310DeviceInfo>();
- ascend310->SetDeviceID(FLAGS_device_id);
- ascend310->SetPrecisionMode("allow_fp32_to_fp16");
- ascend310->SetOpSelectImplMode("high_precision");
- context->MutableDeviceInfo().push_back(ascend310);
- mindspore::Graph graph;
- Serialization::Load(FLAGS_mindir_path, ModelType::kMindIR, &graph);
-
- Model model;
- Status ret = model.Build(GraphCell(graph), context);
- if (ret != kSuccess) {
- std::cout << "ERROR: Build failed." << std::endl;
- return 1;
- }
-
- std::vector<MSTensor> model_inputs = model.GetInputs();
- if (model_inputs.empty()) {
- std::cout << "Invalid model, inputs is empty." << std::endl;
- return 1;
- }
-
- auto input0_files = GetAllFiles(FLAGS_input0_path);
- auto input1_files = GetAllFiles(FLAGS_input1_path);
- auto input2_files = GetAllFiles(FLAGS_input2_path);
- auto input3_files = GetAllFiles(FLAGS_input3_path);
- auto input4_files = GetAllFiles(FLAGS_input4_path);
- auto input5_files = GetAllFiles(FLAGS_input5_path);
-
- if (input0_files.empty() || input1_files.empty()) {
- std::cout << "ERROR: input data empty." << std::endl;
- return 1;
- }
-
- std::map<double, double> costTime_map;
- size_t size = input0_files.size();
-
- for (size_t i = 0; i < size; ++i) {
- struct timeval start = {0};
- struct timeval end = {0};
- double startTimeMs;
- double endTimeMs;
- std::vector<MSTensor> inputs;
- std::vector<MSTensor> outputs;
- std::cout << "Start predict input files:" << input0_files[i] << std::endl;
-
- auto input0 = ReadFileToTensor(input0_files[i]);
- auto input1 = ReadFileToTensor(input1_files[i]);
- auto input2 = ReadFileToTensor(input2_files[i]);
- auto input3 = ReadFileToTensor(input3_files[i]);
- auto input4 = ReadFileToTensor(input4_files[i]);
- auto input5 = ReadFileToTensor(input5_files[i]);
- inputs.emplace_back(model_inputs[0].Name(), model_inputs[0].DataType(), model_inputs[0].Shape(),
- input0.Data().get(), input0.DataSize());
- inputs.emplace_back(model_inputs[1].Name(), model_inputs[1].DataType(), model_inputs[1].Shape(),
- input1.Data().get(), input1.DataSize());
- inputs.emplace_back(model_inputs[2].Name(), model_inputs[2].DataType(), model_inputs[2].Shape(),
- input2.Data().get(), input2.DataSize());
- inputs.emplace_back(model_inputs[3].Name(), model_inputs[3].DataType(), model_inputs[3].Shape(),
- input3.Data().get(), input3.DataSize());
- inputs.emplace_back(model_inputs[4].Name(), model_inputs[4].DataType(), model_inputs[4].Shape(),
- input4.Data().get(), input4.DataSize());
- inputs.emplace_back(model_inputs[5].Name(), model_inputs[5].DataType(), model_inputs[5].Shape(),
- input5.Data().get(), input5.DataSize());
- gettimeofday(&start, nullptr);
- ret = model.Predict(inputs, &outputs);
- gettimeofday(&end, nullptr);
- if (ret != kSuccess) {
- std::cout << "Predict " << input0_files[i] << " failed." << std::endl;
- return 1;
- }
- startTimeMs = (1.0 * start.tv_sec * 1000000 + start.tv_usec) / 1000;
- endTimeMs = (1.0 * end.tv_sec * 1000000 + end.tv_usec) / 1000;
- costTime_map.insert(std::pair<double, double>(startTimeMs, endTimeMs));
- WriteResult(input0_files[i], outputs);
- }
- double average = 0.0;
- int inferCount = 0;
-
- for (auto iter = costTime_map.begin(); iter != costTime_map.end(); iter++) {
- double diff = 0.0;
- diff = iter->second - iter->first;
- average += diff;
- inferCount++;
- }
- average = average / inferCount;
- std::stringstream timeCost;
- timeCost << "NN inference cost average time: "<< average << " ms of infer_count " << inferCount << std::endl;
- std::cout << "NN inference cost average time: "<< average << "ms of infer_count " << inferCount << std::endl;
- std::string fileName = "./time_Result" + std::string("/test_perform_static.txt");
- std::ofstream fileStream(fileName.c_str(), std::ios::trunc);
- fileStream << timeCost.str();
- fileStream.close();
- costTime_map.clear();
- return 0;
-}
diff --git a/official/recommend/tbnet/ascend310_infer/src/utils.cc b/official/recommend/tbnet/ascend310_infer/src/utils.cc
deleted file mode 100644
index c947e4d5f..000000000
--- a/official/recommend/tbnet/ascend310_infer/src/utils.cc
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Copyright 2021 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <fstream>
-#include <algorithm>
-#include <iostream>
-#include "inc/utils.h"
-
-using mindspore::MSTensor;
-using mindspore::DataType;
-
-std::vector<std::string> GetAllFiles(std::string_view dirName) {
- struct dirent *filename;
- DIR *dir = OpenDir(dirName);
- if (dir == nullptr) {
- return {};
- }
- std::vector<std::string> res;
- while ((filename = readdir(dir)) != nullptr) {
- std::string dName = std::string(filename->d_name);
- if (dName == "." || dName == ".." || filename->d_type != DT_REG) {
- continue;
- }
- res.emplace_back(std::string(dirName) + "/" + filename->d_name);
- }
- std::sort(res.begin(), res.end());
- for (auto &f : res) {
- std::cout << "image file: " << f << std::endl;
- }
- return res;
-}
-
-int WriteResult(const std::string& imageFile, const std::vector<MSTensor> &outputs) {
- std::string homePath = "./result_Files";
- for (size_t i = 0; i < outputs.size(); ++i) {
- size_t outputSize;
- std::shared_ptr<const void> netOutput;
- netOutput = outputs[i].Data();
- outputSize = outputs[i].DataSize();
- int pos = imageFile.rfind('/');
- std::string fileName(imageFile, pos + 1);
- fileName.replace(fileName.find('.'), fileName.size() - fileName.find('.'), '_' + std::to_string(i) + ".bin");
- std::string outFileName = homePath + "/" + fileName;
- FILE * outputFile = fopen(outFileName.c_str(), "wb");
- fwrite(netOutput.get(), outputSize, sizeof(char), outputFile);
- fclose(outputFile);
- outputFile = nullptr;
- }
- return 0;
-}
-
-mindspore::MSTensor ReadFileToTensor(const std::string &file) {
- if (file.empty()) {
- std::cout << "Pointer file is nullptr" << std::endl;
- return mindspore::MSTensor();
- }
-
- std::ifstream ifs(file);
- if (!ifs.good()) {
- std::cout << "File: " << file << " is not exist" << std::endl;
- return mindspore::MSTensor();
- }
-
- if (!ifs.is_open()) {
- std::cout << "File: " << file << "open failed" << std::endl;
- return mindspore::MSTensor();
- }
-
- ifs.seekg(0, std::ios::end);
- size_t size = ifs.tellg();
- mindspore::MSTensor buffer(file, mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
-
- ifs.seekg(0, std::ios::beg);
- ifs.read(reinterpret_cast<char *>(buffer.MutableData()), size);
- ifs.close();
-
- return buffer;
-}
-
-
-DIR *OpenDir(std::string_view dirName) {
- if (dirName.empty()) {
- std::cout << " dirName is null ! " << std::endl;
- return nullptr;
- }
- std::string realPath = RealPath(dirName);
- struct stat s;
- lstat(realPath.c_str(), &s);
- if (!S_ISDIR(s.st_mode)) {
- std::cout << "dirName is not a valid directory !" << std::endl;
- return nullptr;
- }
- DIR *dir;
- dir = opendir(realPath.c_str());
- if (dir == nullptr) {
- std::cout << "Can not open dir " << dirName << std::endl;
- return nullptr;
- }
- std::cout << "Successfully opened the dir " << dirName << std::endl;
- return dir;
-}
-
-std::string RealPath(std::string_view path) {
- char realPathMem[PATH_MAX] = {0};
- char *realPathRet = nullptr;
- realPathRet = realpath(path.data(), realPathMem);
-
- if (realPathRet == nullptr) {
- std::cout << "File: " << path << " is not exist.";
- return "";
- }
-
- std::string realPath(realPathMem);
- std::cout << path << " realpath is: " << realPath << std::endl;
- return realPath;
-}
diff --git a/official/recommend/tbnet/default_config.yaml b/official/recommend/tbnet/default_config.yaml
deleted file mode 100644
index 76150eaa3..000000000
--- a/official/recommend/tbnet/default_config.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unlesee you know exactly what you are doing)
-enable_modelarts: False
-# url for modelarts
-data_url: ""
-train_url: ""
-checkpoint_url: ""
-
-# url for openi
-ckpt_url: ""
-result_url: ""
-
-# path for local
-data_path: "."
-output_path: "./output"
-load_path: "./checkpoint"
-
-# preprocess_data
-same_relation: False
-
-#train
-dataset: "steam"
-train_csv: "train.csv"
-test_csv: "test.csv"
-infer_csv: "infer.csv"
-device_id: 0
-epochs: 20
-device_target: "GPU"
-run_mode: "graph"
-
-#eval
-checkpoint_id: 19
diff --git a/official/recommend/tbnet/eval.py b/official/recommend/tbnet/eval.py
deleted file mode 100644
index 27fe5b820..000000000
--- a/official/recommend/tbnet/eval.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net evaluation."""
-
-import os
-import math
-
-from mindspore import context, Model, load_checkpoint, load_param_into_net
-import mindspore.common.dtype as mstype
-
-from src import tbnet, config, metrics, dataset
-
-from src.utils.param import param
-from src.utils.moxing_adapter import moxing_wrapper
-from preprocess_dataset import preprocess_data
-
-
-@moxing_wrapper(preprocess_data)
-def eval_tbnet():
- """Evaluation process."""
- config_path = os.path.join(param.data_path, 'data', param.dataset, 'config.json')
- test_csv_path = os.path.join(param.data_path, 'data', param.dataset, param.test_csv)
- ckpt_path = param.load_path
-
- context.set_context(device_id=param.device_id)
- if param.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=param.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=param.device_target)
-
- print(f"creating dataset from {test_csv_path}...")
- net_config = config.TBNetConfig(config_path)
- if param.device_target == 'Ascend':
- net_config.per_item_paths = math.ceil(net_config.per_item_paths / 16) * 16
- net_config.embedding_dim = math.ceil(net_config.embedding_dim / 16) * 16
- eval_ds = dataset.create(test_csv_path, net_config.per_item_paths, train=True).batch(net_config.batch_size)
-
- print(f"creating TBNet from checkpoint {param.checkpoint_id} for evaluation...")
- network = tbnet.TBNet(net_config)
- if param.device_target == 'Ascend':
- network.to_float(mstype.float16)
- param_dict = load_checkpoint(os.path.join(ckpt_path, f'tbnet_epoch{param.checkpoint_id}.ckpt'))
- load_param_into_net(network, param_dict)
-
- loss_net = tbnet.NetWithLossClass(network, net_config)
- train_net = tbnet.TrainStepWrap(loss_net, net_config.lr)
- train_net.set_train()
- eval_net = tbnet.PredictWithSigmoid(network)
- model = Model(network=train_net, eval_network=eval_net, metrics={'auc': metrics.AUC(), 'acc': metrics.ACC()})
-
- print("evaluating...")
- e_out = model.eval(eval_ds, dataset_sink_mode=False)
- print(f'Test AUC:{e_out["auc"]} ACC:{e_out["acc"]}')
- if param.enable_modelarts:
- with open(os.path.join(param.output_path, 'result.txt'), 'w') as f:
- f.write(f'Test AUC:{e_out["auc"]} ACC:{e_out["acc"]}')
-
-
-if __name__ == '__main__':
- eval_tbnet()
diff --git a/official/recommend/tbnet/export.py b/official/recommend/tbnet/export.py
deleted file mode 100644
index 4110028d4..000000000
--- a/official/recommend/tbnet/export.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""export."""
-
-import os
-import argparse
-import math
-import numpy as np
-
-from mindspore import context, load_checkpoint, load_param_into_net, Tensor, export
-
-from src import tbnet, config
-
-
-def get_args():
- """Parse commandline arguments."""
- parser = argparse.ArgumentParser(description='Export.')
-
- parser.add_argument(
- '--config_path',
- type=str,
- required=True,
- default='',
- help="json file for dataset"
- )
-
- parser.add_argument(
- '--checkpoint_path',
- type=str,
- required=True,
- help="use which checkpoint(.ckpt) file to export"
- )
-
- parser.add_argument(
- '--device_id',
- type=int,
- required=False,
- default=0,
- help="device id"
- )
-
- parser.add_argument(
- '--device_target',
- type=str,
- required=False,
- default='Ascend',
- choices=['Ascend', 'GPU'],
- help="run code on platform"
- )
-
- parser.add_argument(
- '--run_mode',
- type=str,
- required=False,
- default='graph',
- choices=['graph', 'pynative'],
- help="run code by GRAPH mode or PYNATIVE mode"
- )
-
- parser.add_argument(
- '--file_name',
- type=str,
- default='tbnet',
- help="model name."
- )
-
- parser.add_argument(
- '--file_format',
- type=str,
- default='MINDIR',
- choices=['MINDIR', 'AIR'],
- help="model format."
- )
- return parser.parse_args()
-
-
-def export_tbnet():
- """Data preprocess for inference."""
- args = get_args()
-
- config_path = args.config_path
- ckpt_path = args.checkpoint_path
- if not os.path.exists(config_path):
- raise ValueError("please check the config path.")
- if not os.path.exists(ckpt_path):
- raise ValueError("please check the checkpoint path.")
-
- context.set_context(device_id=args.device_id)
- if args.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=args.device_target)
-
- net_config = config.TBNetConfig(config_path)
- if args.device_target == 'Ascend':
- net_config.per_item_paths = math.ceil(net_config.per_item_paths / 16) * 16
- net_config.embedding_dim = math.ceil(net_config.embedding_dim / 16) * 16
- network = tbnet.TBNet(net_config)
- param_dict = load_checkpoint(ckpt_path)
- load_param_into_net(network, param_dict)
- eval_net = tbnet.PredictWithSigmoid(network)
-
- item = Tensor(np.ones((1,)).astype(np.int))
- rl1 = Tensor(np.ones((1, net_config.per_item_paths)).astype(np.int))
- ety = Tensor(np.ones((1, net_config.per_item_paths)).astype(np.int))
- rl2 = Tensor(np.ones((1, net_config.per_item_paths)).astype(np.int))
- his = Tensor(np.ones((1, net_config.per_item_paths)).astype(np.int))
- rate = Tensor(np.ones((1,)).astype(np.float32))
- inputs = [item, rl1, ety, rl2, his, rate]
- export(eval_net, *inputs, file_name=args.file_name, file_format=args.file_format)
-
-
-if __name__ == '__main__':
- export_tbnet()
diff --git a/official/recommend/tbnet/infer.py b/official/recommend/tbnet/infer.py
deleted file mode 100644
index 1e48b5b6c..000000000
--- a/official/recommend/tbnet/infer.py
+++ /dev/null
@@ -1,169 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TBNet inference."""
-
-import os
-import argparse
-import math
-
-from mindspore import load_checkpoint, load_param_into_net, context
-import mindspore.common.dtype as mstype
-from src.config import TBNetConfig
-from src.tbnet import TBNet
-from src.aggregator import InferenceAggregator
-from src import dataset
-from src import steam
-
-
-def get_args():
- """Parse commandline arguments."""
- parser = argparse.ArgumentParser(description='Infer TBNet.')
-
- parser.add_argument(
- '--dataset',
- type=str,
- required=False,
- default='steam',
- help="'steam' dataset is supported currently"
- )
-
- parser.add_argument(
- '--csv',
- type=str,
- required=False,
- default='infer.csv',
- help="the csv datafile inside the dataset folder (e.g. infer.csv)"
- )
-
- parser.add_argument(
- '--checkpoint_id',
- type=int,
- required=True,
- help="use which checkpoint(.ckpt) file to infer"
- )
-
- parser.add_argument(
- '--user',
- type=int,
- required=True,
- help="id of the user to be recommended to"
- )
-
- parser.add_argument(
- '--items',
- type=int,
- required=False,
- default=1,
- help="no. of items to be recommended"
- )
-
- parser.add_argument(
- '--explanations',
- type=int,
- required=False,
- default=3,
- help="no. of recommendation explanations to be shown"
- )
-
- parser.add_argument(
- '--device_id',
- type=int,
- required=False,
- default=0,
- help="device id"
- )
-
- parser.add_argument(
- '--device_target',
- type=str,
- required=False,
- default='GPU',
- choices=['GPU', 'Ascend'],
- help="run code on GPU or Ascend NPU"
- )
-
- parser.add_argument(
- '--run_mode',
- type=str,
- required=False,
- default='graph',
- choices=['graph', 'pynative'],
- help="run code by GRAPH mode or PYNATIVE mode"
- )
-
- return parser.parse_args()
-
-
-def infer_tbnet():
- """Inference process."""
- args = get_args()
- context.set_context(device_id=args.device_id)
- if args.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=args.device_target)
-
- home = os.path.dirname(os.path.realpath(__file__))
- config_path = os.path.join(home, 'data', args.dataset, 'config.json')
- translate_path = os.path.join(home, 'data', args.dataset, 'translate.json')
- data_path = os.path.join(home, 'data', args.dataset, args.csv)
- ckpt_path = os.path.join(home, 'checkpoints')
-
- print(f"creating TBNet from checkpoint {args.checkpoint_id}...")
- config = TBNetConfig(config_path)
- if args.device_target == 'Ascend':
- config.per_item_paths = math.ceil(config.per_item_paths / 16) * 16
- config.embedding_dim = math.ceil(config.embedding_dim / 16) * 16
- network = TBNet(config)
- if args.device_target == 'Ascend':
- network.to_float(mstype.float16)
- param_dict = load_checkpoint(os.path.join(ckpt_path, f'tbnet_epoch{args.checkpoint_id}.ckpt'))
- load_param_into_net(network, param_dict)
-
- print(f"creating dataset from {data_path}...")
- infer_ds = dataset.create(data_path, config.per_item_paths, train=False, users=args.user)
- infer_ds = infer_ds.batch(config.batch_size)
-
- print("inferring...")
- # infer and aggregate results
- aggregator = InferenceAggregator(top_k=args.items)
- for user, item, relation1, entity, relation2, hist_item, rating in infer_ds:
- del rating
- result = network(item, relation1, entity, relation2, hist_item)
- item_score = result[0]
- path_importance = result[1]
- aggregator.aggregate(user, item, relation1, entity, relation2, hist_item, item_score, path_importance)
-
- # show recommendations with explanations
- explainer = steam.TextExplainer(translate_path)
- recomms = aggregator.recommend()
- for user, recomm in recomms.items():
- for item_rec in recomm.item_records:
-
- item_name = explainer.translate_item(item_rec.item)
- print(f"Recommend <{item_name}> to user:{user}, because:")
-
- # show explanations
- explanation = 0
- for path in item_rec.paths:
- print(" - " + explainer.explain(path))
- explanation += 1
- if explanation >= args.explanations:
- break
- print("")
-
-
-if __name__ == '__main__':
- infer_tbnet()
diff --git a/official/recommend/tbnet/infer/convert/air2om.sh b/official/recommend/tbnet/infer/convert/air2om.sh
deleted file mode 100644
index 097818ded..000000000
--- a/official/recommend/tbnet/infer/convert/air2om.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-model_path=$1
-output_model_name=$2
-
-atc --model=$model_path \
- --framework=1 \
- --output=$output_model_name \
- --input_format=NCHW \
- --soc_version=Ascend310 \
- --output_type=FP32
\ No newline at end of file
diff --git a/official/recommend/tbnet/infer/data/config/tbnet.pipeline b/official/recommend/tbnet/infer/data/config/tbnet.pipeline
deleted file mode 100644
index d7056e33b..000000000
--- a/official/recommend/tbnet/infer/data/config/tbnet.pipeline
+++ /dev/null
@@ -1,70 +0,0 @@
-{
- "tbnet": {
- "stream_config": {
- "deviceId": "0"
- },
- "appsrc0": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:0"
- },
- "appsrc1": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:1"
- },
- "appsrc2": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:2"
- },
- "appsrc3": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:3"
- },
- "appsrc4": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:4"
- },
- "appsrc5": {
- "props": {
- "blocksize": "409600"
- },
- "factory": "appsrc",
- "next": "mxpi_tensorinfer0:5"
- },
- "mxpi_tensorinfer0": {
- "props": {
- "dataSource": "appsrc0,appsrc1,appsrc2,appsrc3,appsrc4,appsrc5",
- "modelPath": "../data/model/tbnet.om"
- },
- "factory": "mxpi_tensorinfer",
- "next": "mxpi_dataserialize0"
- },
- "mxpi_dataserialize0": {
- "props": {
- "outputDataKeys": "mxpi_tensorinfer0"
- },
- "factory": "mxpi_dataserialize",
- "next": "appsink0"
- },
- "appsink0": {
- "props": {
- "blocksize": "4096000"
- },
- "factory": "appsink"
- }
- }
-}
diff --git a/official/recommend/tbnet/infer/docker_start_infer.sh b/official/recommend/tbnet/infer/docker_start_infer.sh
deleted file mode 100644
index 2678ff3f9..000000000
--- a/official/recommend/tbnet/infer/docker_start_infer.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-docker_image=$1
-model_dir=$2
-
-
-function show_help() {
- echo "Usage: docker_start.sh docker_image model_dir data_dir"
-}
-
-function param_check() {
- if [ -z "${docker_image}" ]; then
- echo "please input docker_image"
- show_help
- exit 1
- fi
-
- if [ -z "${model_dir}" ]; then
- echo "please input model_dir"
- show_help
- exit 1
- fi
-}
-
-param_check
-
-docker run -it -u root \
- --device=/dev/davinci0 \
- --device=/dev/davinci_manager \
- --device=/dev/devmm_svm \
- --device=/dev/hisi_hdc \
- -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
- -v ${model_dir}:${model_dir} \
- ${docker_image} \
- /bin/bash
diff --git a/official/recommend/tbnet/infer/mxbase/CMakeLists.txt b/official/recommend/tbnet/infer/mxbase/CMakeLists.txt
deleted file mode 100644
index e97200d02..000000000
--- a/official/recommend/tbnet/infer/mxbase/CMakeLists.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-cmake_minimum_required(VERSION 3.5.2)
-project(Tbnet)
-add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
-
-
-set(TARGET_MAIN Tbnet)
-
-set(ACL_LIB_PATH $ENV{ASCEND_HOME}/ascend-toolkit/latest/acllib)
-
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
-include_directories($ENV{MX_SDK_HOME}/include)
-include_directories($ENV{MX_SDK_HOME}/opensource/include)
-include_directories($ENV{MX_SDK_HOME}/opensource/include/opencv4)
-include_directories($ENV{MX_SDK_HOME}/opensource/include/gstreamer-1.0)
-include_directories($ENV{MX_SDK_HOME}/opensource/include/glib-2.0)
-include_directories($ENV{MX_SDK_HOME}/opensource/lib/glib-2.0/include)
-
-link_directories($ENV{MX_SDK_HOME}/lib)
-link_directories($ENV{MX_SDK_HOME}/opensource/lib/)
-
-
-add_compile_options(-std=c++11 -fPIC -fstack-protector-all -pie -Wno-deprecated-declarations)
-add_compile_options("-DPLUGIN_NAME=${PLUGIN_NAME}")
-add_compile_options("-Dgoogle=mindxsdk_private")
-
-add_definitions(-DENABLE_DVPP_INTERFACE)
-
-include_directories(${ACL_LIB_PATH}/include)
-link_directories(${ACL_LIB_PATH}/lib64/)
-
-
-
-add_executable(${TARGET_MAIN} src/main.cpp src/Tbnet.cpp)
-target_link_libraries(${TARGET_MAIN} ${TARGET_LIBRARY} glog cpprest mxbase libascendcl.so)
diff --git a/official/recommend/tbnet/infer/mxbase/build.sh b/official/recommend/tbnet/infer/mxbase/build.sh
deleted file mode 100644
index 71bc7df1a..000000000
--- a/official/recommend/tbnet/infer/mxbase/build.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export ASCEND_VERSION=ascend-toolkit/latest
-export ARCH_PATTERN=.
-export LD_LIBRARY_PATH=${MX_SDK_HOME}/lib/modelpostprocessors:${LD_LIBRARY_PATH}
-
-mkdir -p build
-cd build || exit
-
-function make_plugin() {
- if ! cmake ..;
- then
- echo "cmake failed."
- return 1
- fi
-
- if ! (make);
- then
- echo "make failed."
- return 1
- fi
-
- return 0
-}
-
-if make_plugin;
-then
- echo "INFO: Build successfully."
-else
- echo "ERROR: Build failed."
-fi
-
-cd - || exit
-
diff --git a/official/recommend/tbnet/infer/mxbase/src/Tbnet.cpp b/official/recommend/tbnet/infer/mxbase/src/Tbnet.cpp
deleted file mode 100644
index 368cc46dd..000000000
--- a/official/recommend/tbnet/infer/mxbase/src/Tbnet.cpp
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Copyright 2022 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Tbnet.h"
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <cmath>
-#include <vector>
-#include <algorithm>
-#include <queue>
-#include <utility>
-#include <fstream>
-#include <map>
-#include <iostream>
-#include "acl/acl.h"
-#include "MxBase/DeviceManager/DeviceManager.h"
-#include "MxBase/Log/Log.h"
-namespace {
- const std::vector<std::vector<uint32_t>> SHAPE = {{1}, {1, 39}, {1, 39},
- {1, 39}, {1, 39}, {1}};
- const int FLOAT_SIZE = 4;
- const int INT_SIZE = 8;
- const int DATA_SIZE_1 = 1;
- const int DATA_SIZE_39 = 39;
-}
-
-void WriteResult(const std::string &file_name, const std::vector<MxBase::TensorBase> &outputs) {
- std::string homePath = "./result";
- for (size_t i = 0; i < outputs.size(); ++i) {
- float *boxes = reinterpret_cast<float *>(outputs[i].GetBuffer());
- std::string outFileName = homePath + "/tbnet_item_bs1_" + file_name + "_" +
- std::to_string(i) + ".txt";
- std::ofstream outfile(outFileName, std::ios::app);
- size_t outputSize;
- outputSize = outputs[i].GetSize();
- for (size_t j = 0; j < outputSize; ++j) {
- if (j != 0) {
- outfile << ",";
- }
- outfile << boxes[j];
- }
- outfile.close();
- }
-}
-
-APP_ERROR Tbnet::Init(const InitParam &initParam) {
- deviceId_ = initParam.deviceId;
- APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices();
- if (ret != APP_ERR_OK) {
- LogError << "Init devices failed, ret=" << ret << ".";
- return ret;
- }
- ret = MxBase::TensorContext::GetInstance()->SetContext(initParam.deviceId);
- if (ret != APP_ERR_OK) {
- LogError << "Set context failed, ret=" << ret << ".";
- return ret;
- }
- model_Tbnet = std::make_shared<MxBase::ModelInferenceProcessor>();
- ret = model_Tbnet->Init(initParam.modelPath, modelDesc_);
- if (ret != APP_ERR_OK) {
- LogError << "ModelInferenceProcessor init failed, ret=" << ret << ".";
- return ret;
- }
-
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::DeInit() {
- model_Tbnet->DeInit();
- MxBase::DeviceManager::GetInstance()->DestroyDevices();
- return APP_ERR_OK;
-}
-
-
-APP_ERROR Tbnet::ReadBin_float(const std::string &path, std::vector<std::vector<float>> &dataset,
- const int datasize) {
- std::ifstream inFile(path, std::ios::binary);
-
- float *data = new float[datasize];
- inFile.read(reinterpret_cast<char *>(data), datasize * sizeof(data[0]));
- std::vector<float> temp(data, data + datasize);
- dataset.push_back(temp);
-
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::ReadBin_int(const std::string &path, std::vector<std::vector<int64_t>> &dataset,
- const int datasize) {
- std::ifstream inFile(path, std::ios::binary);
-
- int64_t *data = new int64_t[datasize];
- inFile.read(reinterpret_cast<char *>(data), datasize * sizeof(data[0]));
- std::vector<int64_t> temp(data, data + datasize);
- dataset.push_back(temp);
-
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::VectorToTensorBase_float(const std::vector<std::vector<float>> &input,
- MxBase::TensorBase &tensorBase,
- const std::vector<uint32_t> &shape) {
- uint32_t dataSize = 1;
- for (int i = 0; i < shape.size(); i++) {
- dataSize = dataSize * shape[i];
- } // input shape
- float *metaFeatureData = new float[dataSize];
-
- uint32_t idx = 0;
- for (size_t bs = 0; bs < input.size(); bs++) {
- for (size_t c = 0; c < input[bs].size(); c++) {
- metaFeatureData[idx++] = input[bs][c];
- }
- }
- MxBase::MemoryData memoryDataDst(dataSize * FLOAT_SIZE, MxBase::MemoryData::MEMORY_DEVICE, deviceId_);
- MxBase::MemoryData memoryDataSrc(reinterpret_cast<void *>(metaFeatureData), dataSize * FLOAT_SIZE,
- MxBase::MemoryData::MEMORY_HOST_MALLOC);
-
- APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDataDst, memoryDataSrc);
- if (ret != APP_ERR_OK) {
- LogError << GetError(ret) << "Memory malloc failed.";
- return ret;
- }
-
- tensorBase = MxBase::TensorBase(memoryDataDst, false, shape, MxBase::TENSOR_DTYPE_FLOAT32);
-
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::VectorToTensorBase_int(const std::vector<std::vector<int64_t>> &input,
- MxBase::TensorBase &tensorBase,
- const std::vector<uint32_t> &shape) {
- int dataSize = 1;
- for (int i = 0; i < shape.size(); i++) {
- dataSize = dataSize * shape[i];
- } // input shape
-
- int64_t *metaFeatureData = new int64_t[dataSize];
-
- uint32_t idx = 0;
- for (size_t bs = 0; bs < input.size(); bs++) {
- for (size_t c = 0; c < input[bs].size(); c++) {
- metaFeatureData[idx++] = input[bs][c];
- }
- }
- MxBase::MemoryData memoryDataDst(dataSize * INT_SIZE, MxBase::MemoryData::MEMORY_DEVICE, deviceId_);
- MxBase::MemoryData memoryDataSrc(reinterpret_cast<void *>(metaFeatureData), dataSize * INT_SIZE,
- MxBase::MemoryData::MEMORY_HOST_MALLOC);
-
- APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDataDst, memoryDataSrc);
- if (ret != APP_ERR_OK) {
- LogError << GetError(ret) << "Memory malloc failed.";
- return ret;
- }
-
- tensorBase = MxBase::TensorBase(memoryDataDst, false, shape, MxBase::TENSOR_DTYPE_INT64);
-
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::Inference(const std::vector<MxBase::TensorBase> &inputs,
- std::vector<MxBase::TensorBase> &outputs) {
- auto dtypes = model_Tbnet->GetOutputDataType();
- for (size_t i = 0; i < modelDesc_.outputTensors.size(); ++i) {
- std::vector<uint32_t> shape = {};
- for (size_t j = 0; j < modelDesc_.outputTensors[i].tensorDims.size(); ++j) {
- shape.push_back((uint32_t)modelDesc_.outputTensors[i].tensorDims[j]);
- }
- MxBase::TensorBase tensor(shape, dtypes[i], MxBase::MemoryData::MemoryType::MEMORY_DEVICE, deviceId_);
- APP_ERROR ret = MxBase::TensorBase::TensorBaseMalloc(tensor);
- if (ret != APP_ERR_OK) {
- LogError << "TensorBaseMalloc failed, ret=" << ret << ".";
- return ret;
- }
- outputs.push_back(tensor);
- }
- MxBase::DynamicInfo dynamicInfo = {};
- dynamicInfo.dynamicType = MxBase::DynamicType::STATIC_BATCH;
- auto startTime = std::chrono::high_resolution_clock::now();
- APP_ERROR ret = model_Tbnet->ModelInference(inputs, outputs, dynamicInfo);
- auto endTime = std::chrono::high_resolution_clock::now();
- double costMs = std::chrono::duration<double, std::milli>(endTime - startTime).count();
- inferCostTimeMilliSec += costMs;
- if (ret != APP_ERR_OK) {
- LogError << "ModelInference Tbnet failed, ret=" << ret << ".";
- return ret;
- }
- return APP_ERR_OK;
-}
-
-APP_ERROR Tbnet::Process(const int &index, const std::string &datapath,
- const InitParam &initParam, std::vector<int> &outputs) {
- std::vector<MxBase::TensorBase> inputs = {};
- std::vector<MxBase::TensorBase> outputs_tb = {};
-
- std::vector<std::vector<int64_t>> item;
- APP_ERROR ret = ReadBin_int(datapath + "00_item/tbnet_item_bs1_" +
- std::to_string(index) + ".bin", item, DATA_SIZE_1);
- std::vector<std::vector<int64_t>> rl1;
- ReadBin_int(datapath + "01_rl1/tbnet_rl1_bs1_" +
- std::to_string(index) + ".bin", rl1, DATA_SIZE_39);
- std::vector<std::vector<int64_t>> ety;
- ReadBin_int(datapath + "02_ety/tbnet_ety_bs1_" +
- std::to_string(index) + ".bin", ety, DATA_SIZE_39);
- std::vector<std::vector<int64_t>> rl2;
- ReadBin_int(datapath + "03_rl2/tbnet_rl2_bs1_" +
- std::to_string(index) + ".bin", rl2, DATA_SIZE_39);
- std::vector<std::vector<int64_t>> his;
- ReadBin_int(datapath + "04_his/tbnet_his_bs1_" +
- std::to_string(index) + ".bin", his, DATA_SIZE_39);
- std::vector<std::vector<float>> rate;
- ReadBin_float(datapath + "05_rate/tbnet_rate_bs1_" +
- std::to_string(index) + ".bin", rate, DATA_SIZE_1);
-
- if (ret != APP_ERR_OK) {
- LogError << "ToTensorBase failed, ret=" << ret << ".";
- return ret;
- }
-
- MxBase::TensorBase tensorBase0;
- APP_ERROR ret1 = VectorToTensorBase_int(item, tensorBase0, SHAPE[0]);
- inputs.push_back(tensorBase0);
- MxBase::TensorBase tensorBase1;
- VectorToTensorBase_int(rl1, tensorBase1, SHAPE[1]);
- inputs.push_back(tensorBase1);
- MxBase::TensorBase tensorBase2;
- VectorToTensorBase_int(ety, tensorBase2, SHAPE[2]);
- inputs.push_back(tensorBase2);
- MxBase::TensorBase tensorBase3;
- VectorToTensorBase_int(rl2, tensorBase3, SHAPE[3]);
- inputs.push_back(tensorBase3);
- MxBase::TensorBase tensorBase4;
- VectorToTensorBase_int(his, tensorBase4, SHAPE[4]);
- inputs.push_back(tensorBase4);
- MxBase::TensorBase tensorBase5;
- VectorToTensorBase_float(rate, tensorBase5, SHAPE[5]);
- inputs.push_back(tensorBase5);
-
- if (ret1 != APP_ERR_OK) {
- LogError << "ToTensorBase failed, ret=" << ret1 << ".";
- return ret1;
- }
-
- auto startTime = std::chrono::high_resolution_clock::now();
- APP_ERROR ret3 = Inference(inputs, outputs_tb);
-
- auto endTime = std::chrono::high_resolution_clock::now();
- double costMs = std::chrono::duration<double, std::milli>(endTime - startTime).count();
- inferCostTimeMilliSec += costMs;
- if (ret3 != APP_ERR_OK) {
- LogError << "Inference failed, ret=" << ret3 << ".";
- return ret3;
- }
- for (size_t i = 0; i < outputs_tb.size(); ++i) {
- if (!outputs_tb[i].IsHost()) {
- outputs_tb[i].ToHost();
- }
- }
- WriteResult(std::to_string(index), outputs_tb);
-}
diff --git a/official/recommend/tbnet/infer/mxbase/src/Tbnet.h b/official/recommend/tbnet/infer/mxbase/src/Tbnet.h
deleted file mode 100644
index d1487a893..000000000
--- a/official/recommend/tbnet/infer/mxbase/src/Tbnet.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2022 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef MXBASE_Tbnet_H
-#define MXBASE_Tbnet_H
-#include <memory>
-#include <string>
-#include <vector>
-#include "acl/acl.h"
-#include "MxBase/DvppWrapper/DvppWrapper.h"
-#include "MxBase/ModelInfer/ModelInferenceProcessor.h"
-#include "MxBase/Tensor/TensorContext/TensorContext.h"
-#include "MxBase/CV/Core/DataType.h"
-
-struct InitParam {
- uint32_t deviceId;
- bool checkTensor;
- std::string modelPath;
-};
-
-class Tbnet {
- public:
- APP_ERROR Init(const InitParam &initParam);
- APP_ERROR DeInit();
- APP_ERROR VectorToTensorBase_int(const std::vector<std::vector<int64_t>> &input, MxBase::TensorBase &tensorBase,
- const std::vector<uint32_t> &shape);
- APP_ERROR VectorToTensorBase_float(const std::vector<std::vector<float>> &input, MxBase::TensorBase &tensorBase,
- const std::vector<uint32_t> &shape);
- APP_ERROR Inference(const std::vector<MxBase::TensorBase> &inputs, std::vector<MxBase::TensorBase> &outputs);
- APP_ERROR Process(const int &index, const std::string &datapath,
- const InitParam &initParam, std::vector<int> &outputss);
- APP_ERROR ReadBin_int(const std::string &path, std::vector<std::vector<int64_t>> &dataset,
- const int shape);
- APP_ERROR ReadBin_float(const std::string &path, std::vector<std::vector<float>> &dataset,
- const int shape);
- // get infer time
- double GetInferCostMilliSec() const {return inferCostTimeMilliSec;}
-
-
- private:
- std::shared_ptr<MxBase::ModelInferenceProcessor> model_Tbnet;
- MxBase::ModelDesc modelDesc_;
- uint32_t deviceId_ = 0;
- // infer time
- double inferCostTimeMilliSec = 0.0;
-};
-
-#endif
diff --git a/official/recommend/tbnet/infer/mxbase/src/main.cpp b/official/recommend/tbnet/infer/mxbase/src/main.cpp
deleted file mode 100644
index d3ce994a9..000000000
--- a/official/recommend/tbnet/infer/mxbase/src/main.cpp
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2022 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <dirent.h>
-#include <fstream>
-#include "MxBase/Log/Log.h"
-#include "Tbnet.h"
-
-namespace {
- const uint32_t DATA_SIZE = 18415;
-} // namespace
-
-int main(int argc, char* argv[]) {
- InitParam initParam = {};
- initParam.deviceId = 0;
- initParam.checkTensor = true;
- initParam.modelPath = "../data/model/tbnet.om";
- std::string dataPath = "../../preprocess_Result/";
-
- auto model_Tbnet = std::make_shared<Tbnet>();
- APP_ERROR ret = model_Tbnet->Init(initParam);
- if (ret != APP_ERR_OK) {
- LogError << "Tagging init failed, ret=" << ret << ".";
- model_Tbnet->DeInit();
- return ret;
- }
-
- std::vector<int> outputs;
- for (int i=0; i < DATA_SIZE; i++) {
- LogInfo << "processing " << i;
- ret = model_Tbnet->Process(i, dataPath, initParam, outputs);
- if (ret !=APP_ERR_OK) {
- LogError << "Tbnet process failed, ret=" << ret << ".";
- model_Tbnet->DeInit();
- return ret;
- }
- }
-
- model_Tbnet->DeInit();
-
- double total_time = model_Tbnet->GetInferCostMilliSec() / 1000;
- LogInfo<< "inferance total cost time: "<< total_time<< ", FPS: "<< DATA_SIZE/total_time;
-
- return APP_ERR_OK;
-}
diff --git a/official/recommend/tbnet/infer/sdk/main.py b/official/recommend/tbnet/infer/sdk/main.py
deleted file mode 100644
index 1498f3aa9..000000000
--- a/official/recommend/tbnet/infer/sdk/main.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-""" main.py """
-import argparse
-import os
-from StreamManagerApi import StreamManagerApi, StringVector
-from StreamManagerApi import MxDataInput, InProtobufVector, MxProtobufIn
-import MxpiDataType_pb2 as MxpiDataType
-import numpy as np
-
-
-def parse_args(parsers):
- """
- Parse commandline arguments.
- """
- parsers.add_argument('--data_path', type=str,
- default="../../preprocess_Result",
- help='text path')
- return parsers
-
-def create_protobuf(path, id1, shape):
- # Construct the input of the stream
- data_input = MxDataInput()
- with open(path, 'rb') as f:
- data = f.read()
- data_input.data = data
- tensorPackageList1 = MxpiDataType.MxpiTensorPackageList()
- tensorPackage1 = tensorPackageList1.tensorPackageVec.add()
- tensorVec1 = tensorPackage1.tensorVec.add()
- tensorVec1.deviceId = 0
- tensorVec1.memType = 0
- for t in shape:
- tensorVec1.tensorShape.append(t)
- tensorVec1.dataStr = data_input.data
- tensorVec1.tensorDataSize = len(data)
-
- protobuf1 = MxProtobufIn()
- protobuf1.key = b'appsrc%d' % id1
- protobuf1.type = b'MxTools.MxpiTensorPackageList'
- protobuf1.protobuf = tensorPackageList1.SerializeToString()
-
- return protobuf1
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description='Om tbnet Inference')
- parser = parse_args(parser)
- args, _ = parser.parse_known_args()
- # init stream manager
- stream_manager = StreamManagerApi()
- ret = stream_manager.InitManager()
- if ret != 0:
- print("Failed to init Stream manager, ret=%s" % str(ret))
- exit()
-
- # create streams by pipeline config file
- with open("../data/config/tbnet.pipeline", 'rb') as fl:
- pipeline = fl.read()
- ret = stream_manager.CreateMultipleStreams(pipeline)
- if ret != 0:
- print("Failed to create Stream, ret=%s" % str(ret))
- exit()
-
- # Construct the input of the stream
-
- res_dir_name = 'result'
- if not os.path.exists(res_dir_name):
- os.makedirs(res_dir_name)
-
- results = []
- input_names = ['00_item', '01_rl1', '02_ety', '03_rl2', '04_his', '05_rate']
- shape_list = [[1], [1, 39], [1, 39], [1, 39], [1, 39], [1]]
-
- for idx in range(18415):
- print('infer %d' % idx)
- for index, name in enumerate(input_names):
- protobufVec = InProtobufVector()
- path_tmp = os.path.join(args.data_path, name,
- 'tbnet_' + name.split('_')[1] + '_bs1_' + str(idx) + '.bin')
- protobufVec.push_back(create_protobuf(path_tmp, index, shape_list[index]))
- unique_id = stream_manager.SendProtobuf(b'tbnet', b'appsrc%d' % index, protobufVec)
-
- keyVec = StringVector()
- keyVec.push_back(b'mxpi_tensorinfer0')
- infer_result = stream_manager.GetProtobuf(b'tbnet', 0, keyVec)
- if infer_result.size() == 0:
- print("inferResult is null")
- exit()
- if infer_result[0].errorCode != 0:
- print("GetProtobuf error. errorCode=%d" % (
- infer_result[0].errorCode))
- exit()
- # get infer result
- result = MxpiDataType.MxpiTensorPackageList()
- result.ParseFromString(infer_result[0].messageBuf)
- for i in range(4):
- res = np.frombuffer(result.tensorPackageVec[0].tensorVec[i].dataStr, dtype=np.float32)
- np.savetxt("./result/tbnet_item_bs1_%d_%d.txt" % (idx, i), res, fmt='%.06f')
-
- # destroy streams
- stream_manager.DestroyAllStreams()
diff --git a/official/recommend/tbnet/infer/sdk/prec/postprocess.py b/official/recommend/tbnet/infer/sdk/prec/postprocess.py
deleted file mode 100644
index 413757245..000000000
--- a/official/recommend/tbnet/infer/sdk/prec/postprocess.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""preprocess data"""
-import argparse
-import os
-import numpy as np
-
-parser = argparse.ArgumentParser(description='Postprocess of Hypertext Inference')
-parser.add_argument('--result_Path', type=str, default='./result_Files',
- help='result path')
-parser.add_argument('--label_Path', default='./result_Files', type=str,
- help='label file path')
-parser.add_argument('--batch_size', default=1, type=int, help='batch_size')
-args = parser.parse_args()
-
-def calculate_auc(labels_list, preds_list):
- """
- The AUC calculation function
- Input:
- labels_list: list of true label
- preds_list: list of predicted label
- Outputs
- Float, means of AUC
- """
- auc = []
- n_bins = labels_list.shape[0] // 2
- if labels_list.ndim == 1:
- labels_list = labels_list.reshape(-1, 1)
- preds_list = preds_list.reshape(-1, 1)
- for i in range(labels_list.shape[1]):
- labels = labels_list[:, i]
- preds = preds_list[:, i]
- postive_len = labels.sum()
- negative_len = labels.shape[0] - postive_len
- total_case = postive_len * negative_len
- positive_histogram = np.zeros((n_bins))
- negative_histogram = np.zeros((n_bins))
- bin_width = 1.0 / n_bins
-
- for j, _ in enumerate(labels):
- nth_bin = int(preds[j] // bin_width)
- if labels[j]:
- positive_histogram[nth_bin] = positive_histogram[nth_bin] + 1
- else:
- negative_histogram[nth_bin] = negative_histogram[nth_bin] + 1
-
- accumulated_negative = 0
- satisfied_pair = 0
- for k in range(n_bins):
- satisfied_pair += (
- positive_histogram[k] * accumulated_negative +
- positive_histogram[k] * negative_histogram[k] * 0.5)
- accumulated_negative += negative_histogram[k]
- auc.append(satisfied_pair / total_case)
-
- return np.mean(auc)
-
-dirs = os.listdir(args.label_Path)
-cur, total = 0, 0
-print('---------- start cal acc ----------')
-gt_list = []
-pred_list = []
-for file in dirs:
- label = np.fromfile(os.path.join(args.label_Path, file), dtype=np.float32)
- gt_list.append(label)
-
- file_name = file.split('.')[0]
- idx = file_name.split('_')[-1]
- predict_file_name = "tbnet_item_bs1_" + str(idx) + "_1.txt"
- predict_file = os.path.join(args.result_Path, predict_file_name)
-
- predict = np.loadtxt(predict_file, dtype=np.float32).reshape(1)
- pred_list.append(predict)
-
-res_pred = np.concatenate(pred_list, axis=0)
-res_true = np.concatenate(gt_list, axis=0)
-rst_auc = calculate_auc(res_true, res_pred)
-print('auc:', rst_auc)
diff --git a/official/recommend/tbnet/infer/sdk/run.sh b/official/recommend/tbnet/infer/sdk/run.sh
deleted file mode 100644
index 00dcdb1da..000000000
--- a/official/recommend/tbnet/infer/sdk/run.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-# Simple log helper functions
-info() { echo -e "\033[1;34m[INFO ][MxStream] $1\033[1;37m" ; }
-warn() { echo >&2 -e "\033[1;31m[WARN ][MxStream] $1\033[1;37m" ; }
-
-#to set PYTHONPATH, import the StreamManagerApi.py
-export PYTHONPATH=$PYTHONPATH:${MX_SDK_HOME}/python
-
-python3 main.py
-exit 0
\ No newline at end of file
diff --git a/official/recommend/tbnet/modelarts/train_modelarts.py b/official/recommend/tbnet/modelarts/train_modelarts.py
deleted file mode 100644
index bbb8029e3..000000000
--- a/official/recommend/tbnet/modelarts/train_modelarts.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net training."""
-
-import os
-import argparse
-
-import numpy as np
-from mindspore import context, Model, Tensor, export
-from mindspore.train.serialization import save_checkpoint
-from mindspore.train.callback import Callback, TimeMonitor
-
-from src import tbnet, config, metrics, dataset
-
-
-class MyLossMonitor(Callback):
- """My loss monitor definition."""
-
- def epoch_end(self, run_context):
- """Print loss at each epoch end."""
- cb_params = run_context.original_args()
- loss = cb_params.net_outputs
-
- if isinstance(loss, (tuple, list)):
- if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray):
- loss = loss[0]
-
- if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray):
- loss = np.mean(loss.asnumpy())
- print('loss:' + str(loss))
-
-
-def get_args():
- """Parse commandline arguments."""
- parser = argparse.ArgumentParser(description='Train TBNet.')
-
- parser.add_argument(
- '--dataset',
- type=str,
- required=False,
- default='steam',
- help="'steam' dataset is supported currently"
- )
-
- parser.add_argument(
- '--data_url',
- type=str,
- required=False,
- default='train.csv',
- help="the train csv datafile inside the dataset folder"
- )
-
- parser.add_argument(
- '--train_url',
- type=str,
- required=False,
- default='',
- help="data to ckpt"
- )
-
- parser.add_argument(
- '--device_id',
- type=int,
- required=False,
- default=0,
- help="device id"
- )
-
- parser.add_argument(
- '--epochs',
- type=int,
- required=False,
- default=20,
- help="number of training epochs"
- )
-
- parser.add_argument(
- '--device_target',
- type=str,
- required=False,
- default='GPU',
- help="run code on GPU"
- )
-
- parser.add_argument(
- '--run_mode',
- type=str,
- required=False,
- default='graph',
- choices=['graph', 'pynative'],
- help="run code by GRAPH mode or PYNATIVE mode"
- )
-
- return parser.parse_args()
-
-
-def train_tbnet():
- """Training process."""
- args = get_args()
-
- home = os.path.dirname(os.path.realpath(__file__))
- config_path = os.path.join(home, 'data', args.dataset, 'config.json')
- train_csv_path = args.data_url + '/train.csv'
- ckpt_path = args.train_url
-
- context.set_context(device_id=args.device_id)
- if args.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=args.device_target)
-
- if not os.path.exists(ckpt_path):
- os.makedirs(ckpt_path)
-
- print(f"creating dataset from {train_csv_path}...")
- net_config = config.TBNetConfig(config_path)
- train_ds = dataset.create(train_csv_path, net_config.per_item_num_paths, train=True).batch(net_config.batch_size)
- print('datasize:', train_ds.get_dataset_size())
- print("creating TBNet for training...")
- network = tbnet.TBNet(net_config)
- loss_net = tbnet.NetWithLossClass(network, net_config)
- train_net = tbnet.TrainStepWrap(loss_net, net_config.lr)
- train_net.set_train()
- eval_net = tbnet.PredictWithSigmoid(network)
- time_callback = TimeMonitor(data_size=train_ds.get_dataset_size())
- loss_callback = MyLossMonitor()
- model = Model(network=train_net, eval_network=eval_net, metrics={'auc': metrics.AUC(), 'acc': metrics.ACC()})
- print("training...")
- model.train(epoch=args.epochs, train_dataset=train_ds,
- callbacks=[time_callback, loss_callback], dataset_sink_mode=False)
-
- save_checkpoint(network, os.path.join(ckpt_path, f'tbnet.ckpt'))
-
- eval_net = tbnet.PredictWithSigmoid(network)
-
- item = Tensor(np.ones((1,)).astype(np.int))
- rl1 = Tensor(np.ones((1, 39)).astype(np.int))
- ety = Tensor(np.ones((1, 39)).astype(np.int))
- rl2 = Tensor(np.ones((1, 39)).astype(np.int))
- his = Tensor(np.ones((1, 39)).astype(np.int))
- rate = Tensor(np.ones((1,)).astype(np.float32))
- inputs = [item, rl1, ety, rl2, his, rate]
- export(eval_net, *inputs, file_name=ckpt_path + '/tbnet.air', file_format='AIR')
-
-if __name__ == '__main__':
- train_tbnet()
diff --git a/official/recommend/tbnet/postprocess.py b/official/recommend/tbnet/postprocess.py
deleted file mode 100644
index 996743e8e..000000000
--- a/official/recommend/tbnet/postprocess.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""preprocess data"""
-import argparse
-import os
-import numpy as np
-
-parser = argparse.ArgumentParser(description='Postprocess of Hypertext Inference')
-parser.add_argument('--result_Path', type=str, default='./result_Files',
- help='result path')
-parser.add_argument('--label_Path', default='./result_Files', type=str,
- help='label file path')
-parser.add_argument('--batch_size', default=1, type=int, help='batch_size')
-args = parser.parse_args()
-
-def calculate_auc(labels_list, preds_list):
- """
- The AUC calculation function
- Input:
- labels_list: list of true label
- preds_list: list of predicted label
- Outputs
- Float, means of AUC
- """
- auc = []
- n_bins = labels_list.shape[0] // 2
- if labels_list.ndim == 1:
- labels_list = labels_list.reshape(-1, 1)
- preds_list = preds_list.reshape(-1, 1)
- for i in range(labels_list.shape[1]):
- labels = labels_list[:, i]
- preds = preds_list[:, i]
- postive_len = labels.sum()
- negative_len = labels.shape[0] - postive_len
- total_case = postive_len * negative_len
- positive_histogram = np.zeros((n_bins))
- negative_histogram = np.zeros((n_bins))
- bin_width = 1.0 / n_bins
-
- for j, _ in enumerate(labels):
- nth_bin = int(preds[j] // bin_width)
- if labels[j]:
- positive_histogram[nth_bin] = positive_histogram[nth_bin] + 1
- else:
- negative_histogram[nth_bin] = negative_histogram[nth_bin] + 1
-
- accumulated_negative = 0
- satisfied_pair = 0
- for k in range(n_bins):
- satisfied_pair += (
- positive_histogram[k] * accumulated_negative +
- positive_histogram[k] * negative_histogram[k] * 0.5)
- accumulated_negative += negative_histogram[k]
- auc.append(satisfied_pair / total_case)
-
- return np.mean(auc)
-
-dirs = os.listdir(args.label_Path)
-cur, total = 0, 0
-print('---------- start cal acc ----------')
-gt_list = []
-pred_list = []
-for file in dirs:
- label = np.fromfile(os.path.join(args.label_Path, file), dtype=np.float32)
- gt_list.append(label)
-
- file_name = file.split('.')[0]
- idx = file_name.split('_')[-1]
- predict_file_name = "tbnet_item_bs1_" + str(idx) + "_1.bin"
- predict_file = os.path.join(args.result_Path, predict_file_name)
- predict = np.fromfile(predict_file, dtype=np.float32)
- pred_list.append(predict)
-res_pred = np.concatenate(pred_list, axis=0)
-res_true = np.concatenate(gt_list, axis=0)
-rst_auc = calculate_auc(res_true, res_pred)
-print('auc:', rst_auc)
diff --git a/official/recommend/tbnet/preprocess.py b/official/recommend/tbnet/preprocess.py
deleted file mode 100644
index f22d5c6d4..000000000
--- a/official/recommend/tbnet/preprocess.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net evaluation."""
-
-import os
-import argparse
-import shutil
-import math
-import numpy as np
-
-from mindspore import context
-
-from src import config, dataset
-
-
-def get_args():
- """Parse commandline arguments."""
- parser = argparse.ArgumentParser(description='Preprocess TBNet training data.')
-
- parser.add_argument(
- '--dataset',
- type=str,
- required=False,
- default='steam',
- help="'steam' dataset is supported currently"
- )
-
- parser.add_argument(
- '--csv',
- type=str,
- required=False,
- default='test.csv',
- help="the csv datafile inside the dataset folder (e.g. test.csv)"
- )
-
- parser.add_argument(
- '--device_id',
- type=int,
- required=False,
- default=0,
- help="device id"
- )
-
- parser.add_argument(
- '--device_target',
- type=str,
- required=False,
- default='Ascend',
- choices=['Ascend', 'GPU'],
- help="run code on GPU or Ascend NPU"
- )
-
- parser.add_argument(
- '--run_mode',
- type=str,
- required=False,
- default='graph',
- choices=['graph', 'pynative'],
- help="run code by GRAPH mode or PYNATIVE mode"
- )
-
- return parser.parse_args()
-
-
-def preprocess_tbnet():
- """Data preprocess for inference."""
- args = get_args()
-
- home = os.path.dirname(os.path.realpath(__file__))
- config_path = os.path.join(home, 'data', args.dataset, 'config.json')
- test_csv_path = os.path.join(home, 'data', args.dataset, args.csv)
-
- context.set_context(device_id=args.device_id)
- if args.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=args.device_target)
-
- print(f"creating dataset from {test_csv_path}...")
- net_config = config.TBNetConfig(config_path)
- if args.device_target == 'Ascend':
- net_config.per_item_paths = math.ceil(net_config.per_item_paths / 16) * 16
- eval_ds = dataset.create(test_csv_path, net_config.per_item_paths, train=True).batch(1)
- item_path = os.path.join('./preprocess_Result/', '00_item')
- rl1_path = os.path.join('./preprocess_Result/', '01_rl1')
- ety_path = os.path.join('./preprocess_Result/', '02_ety')
- rl2_path = os.path.join('./preprocess_Result/', '03_rl2')
- his_path = os.path.join('./preprocess_Result/', '04_his')
- rate_path = os.path.join('./preprocess_Result/', '05_rate')
- rst_path = [item_path, rl1_path, ety_path, rl2_path, his_path, rate_path]
- if os.path.isdir('./preprocess_Result/'):
- shutil.rmtree('./preprocess_Result/')
- for paths in rst_path:
- os.makedirs(paths)
-
- idx = 0
- for d in eval_ds.create_dict_iterator():
- item_rst = d['item'].asnumpy().astype(np.int)
- rl1_rst = np.expand_dims(d['relation1'].asnumpy().astype(np.int), axis=0)
- ety_rst = np.expand_dims(d['entity'].asnumpy().astype(np.int), axis=0)
- rl2_rst = np.expand_dims(d['relation2'].asnumpy().astype(np.int), axis=0)
- his_rst = np.expand_dims(d['hist_item'].asnumpy().astype(np.int), axis=0)
- rate_rst = d['rating'].asnumpy().astype(np.float32)
-
- item_name = 'tbnet_item_bs1_' + str(idx) + '.bin'
- rl1_name = 'tbnet_rl1_bs1_' + str(idx) + '.bin'
- ety_name = 'tbnet_ety_bs1_' + str(idx) + '.bin'
- rl2_name = 'tbnet_rl2_bs1_' + str(idx) + '.bin'
- his_name = 'tbnet_his_bs1_' + str(idx) + '.bin'
- rate_name = 'tbnet_rate_bs1_' + str(idx) + '.bin'
-
- item_real_path = os.path.join(item_path, item_name)
- rl1_real_path = os.path.join(rl1_path, rl1_name)
- ety_real_path = os.path.join(ety_path, ety_name)
- rl2_real_path = os.path.join(rl2_path, rl2_name)
- his_real_path = os.path.join(his_path, his_name)
- rate_real_path = os.path.join(rate_path, rate_name)
-
- item_rst.tofile(item_real_path)
- rl1_rst.tofile(rl1_real_path)
- ety_rst.tofile(ety_real_path)
- rl2_rst.tofile(rl2_real_path)
- his_rst.tofile(his_real_path)
- rate_rst.tofile(rate_real_path)
-
- idx += 1
-
-
-if __name__ == '__main__':
- preprocess_tbnet()
diff --git a/official/recommend/tbnet/preprocess_dataset.py b/official/recommend/tbnet/preprocess_dataset.py
deleted file mode 100644
index fb3806b89..000000000
--- a/official/recommend/tbnet/preprocess_dataset.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Data Preprocessing app."""
-
-# This script should be run directly with 'python <script> <args>'.
-
-import os
-import io
-import json
-import math
-
-from src.path_gen import PathGen
-from src.config import TBNetConfig
-from src.utils.param import param
-
-
-def preprocess_csv(path_gen, data_home, src_name, out_name):
- """Pre-process a csv file."""
- src_path = os.path.join(data_home, src_name)
- out_path = os.path.join(data_home, out_name)
- print(f'converting {src_path} to {out_path} ...')
- rows = path_gen.generate(src_path, out_path)
- print(f'{rows} rows of path data generated.')
-
-
-def preprocess_data():
- """Pre-process the dataset."""
- data_home = os.path.join(param.data_path, 'data', param.dataset)
- config_path = os.path.join(data_home, 'config.json')
- id_maps_path = os.path.join(data_home, 'id_maps.json')
-
- cfg = TBNetConfig(config_path)
- if param.device_target == 'Ascend':
- cfg.per_item_paths = math.ceil(cfg.per_item_paths / 16) * 16
- path_gen = PathGen(per_item_paths=cfg.per_item_paths, same_relation=param.same_relation)
-
- preprocess_csv(path_gen, data_home, 'src_train.csv', 'train.csv')
-
- # save id maps for the later use by Recommender in infer.py
- with io.open(id_maps_path, mode="w", encoding="utf-8") as f:
- json.dump(path_gen.id_maps(), f, indent=4)
-
- # count distinct objects from the training set
- cfg.num_items = path_gen.num_items
- cfg.num_references = path_gen.num_references
- cfg.num_relations = path_gen.num_relations
- cfg.save(config_path)
-
- print(f'{config_path} updated.')
- print(f'num_items: {cfg.num_items}')
- print(f'num_references: {cfg.num_references}')
- print(f'num_relations: {cfg.num_relations}')
-
- # treat new items and references in test and infer set as unseen entities
- # dummy internal id 0 will be assigned to them
- path_gen.grow_id_maps = False
-
- preprocess_csv(path_gen, data_home, 'src_test.csv', 'test.csv')
-
- # for inference, only take interacted('c') and other('x') items as candidate items,
- # the purchased('p') items won't be recommended.
- # assume there is only one user in src_infer.csv
- path_gen.subject_ratings = "cx"
- preprocess_csv(path_gen, data_home, 'src_infer.csv', 'infer.csv')
-
- print(f'Dataset {data_home} processed.')
-
-
-if __name__ == '__main__':
- preprocess_data()
diff --git a/official/recommend/tbnet/requirements.txt b/official/recommend/tbnet/requirements.txt
deleted file mode 100644
index 044bdd758..000000000
--- a/official/recommend/tbnet/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-sklearn
\ No newline at end of file
diff --git a/official/recommend/tbnet/scripts/docker_start.sh b/official/recommend/tbnet/scripts/docker_start.sh
deleted file mode 100644
index 65023c664..000000000
--- a/official/recommend/tbnet/scripts/docker_start.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2022. Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-
-docker_image=$1
-data_dir=$2
-model_dir=$3
-
-docker run -it -u root--ipc=host \
- --device=/dev/davinci0 \
- --device=/dev/davinci1 \
- --device=/dev/davinci2 \
- --device=/dev/davinci3 \
- --device=/dev/davinci4 \
- --device=/dev/davinci5 \
- --device=/dev/davinci6 \
- --device=/dev/davinci7 \
- --device=/dev/davinci_manager \
- --device=/dev/devmm_svm \
- --device=/dev/hisi_hdc \
- --privileged \
- -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
- -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons \
- -v ${data_dir}:${data_dir} \
- -v ${model_dir}:${model_dir} \
- -v /root/ascend/log:/root/ascend/log ${docker_image} /bin/bash
\ No newline at end of file
diff --git a/official/recommend/tbnet/scripts/run_eval.sh b/official/recommend/tbnet/scripts/run_eval.sh
deleted file mode 100644
index 45d4060f7..000000000
--- a/official/recommend/tbnet/scripts/run_eval.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-if [[ $# -lt 3 || $# -gt 4 ]]; then
- echo "Usage: bash run_train.sh [CHECKPOINT_ID] [DATA_NAME] [CUDA_VISIBLE_DEVICES]/[DEVICE_ID] [DEVICE_TARGET]
- CHECKPOINT_ID means model checkpoint id.
- DATA_NAME means dataset name, it's value is 'steam'.
- CUDA_VISIBLE_DEVICES means cuda visible device id.
- DEVICE_ID means device id, it can be set by environment variable DEVICE_ID.
- DEVICE_TARGET is optional, it's value is ['GPU', 'Ascend'], default 'GPU'."
-exit 1
-fi
-
-CHECKPOINT_ID=$1
-DATA_NAME=$2
-
-DEVICE_TARGET='GPU'
-if [ $# == 4 ]; then
- DEVICE_TARGET=$4
-fi
-
-if [ "$DEVICE_TARGET" = "GPU" ];
-then
- export CUDA_VISIBLE_DEVICES=$3
- python eval.py --checkpoint_id $CHECKPOINT_ID --dataset $DATA_NAME --device_target $DEVICE_TARGET \
- --device_id 0 &> scripts/eval_standalone_gpu_log &
-fi
-
-if [ "$DEVICE_TARGET" = "Ascend" ];
-then
- export DEVICE_ID=$3
- python eval.py --checkpoint_id $CHECKPOINT_ID --dataset $DATA_NAME --device_target $DEVICE_TARGET \
- --device_id $DEVICE_ID &> scripts/eval_standalone_gpu_log &
-fi
\ No newline at end of file
diff --git a/official/recommend/tbnet/scripts/run_infer_310.sh b/official/recommend/tbnet/scripts/run_infer_310.sh
deleted file mode 100644
index bd1cbb695..000000000
--- a/official/recommend/tbnet/scripts/run_infer_310.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/bin/bash
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-
-if [[ $# -lt 2 || $# -gt 3 ]]; then
- echo "Usage: bash run_infer_310.sh [MINDIR_PATH] [[DATA_PATH] [DEVICE_ID] (optional)
- DEVICE_ID is optional, it can be set by environment variable device_id, otherwise the value is zero"
-exit 1
-fi
-
-get_real_path(){
- if [ "${1:0:1}" == "/" ]; then
- echo "$1"
- else
- echo "$(realpath -m $PWD/$1)"
- fi
-}
-
-model=$(get_real_path $1)
-data_path=$(get_real_path $2)
-
-device_id=0
-
-if [ $# == 3 ]; then
- device_id=$3
-fi
-
-echo $model
-echo $data_path
-echo $device_id
-
-export ASCEND_HOME=/usr/local/Ascend/
-if [ -d ${ASCEND_HOME}/ascend-toolkit ]; then
- export PATH=$ASCEND_HOME/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:$ASCEND_HOME/ascend-toolkit/latest/atc/bin:$PATH
- export LD_LIBRARY_PATH=/usr/local/lib:$ASCEND_HOME/ascend-toolkit/latest/atc/lib64:$ASCEND_HOME/ascend-toolkit/latest/fwkacllib/lib64:$ASCEND_HOME/driver/lib64:$ASCEND_HOME/add-ons:$LD_LIBRARY_PATH
- export TBE_IMPL_PATH=$ASCEND_HOME/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe
- export PYTHONPATH=${TBE_IMPL_PATH}:$ASCEND_HOME/ascend-toolkit/latest/fwkacllib/python/site-packages:$PYTHONPATH
- export ASCEND_OPP_PATH=$ASCEND_HOME/ascend-toolkit/latest/opp
-else
- export ASCEND_HOME=/usr/local/Ascend/latest/
- export PATH=$ASCEND_HOME/atc/ccec_compiler/bin:$ASCEND_HOME/atc/bin:$PATH
- export LD_LIBRARY_PATH=/usr/local/lib:$ASCEND_HOME/atc/lib64:$ASCEND_HOME/acllib/lib64:$ASCEND_HOME/driver/lib64:$ASCEND_HOME/add-ons:$LD_LIBRARY_PATH
- export PYTHONPATH=$ASCEND_HOME/atc/python/site-packages:$PYTHONPATH
- export ASCEND_OPP_PATH=$ASCEND_HOME/opp
-fi
-
-function preprocess_data()
-{
- if [ -d preprocess_Result ]; then
- rm -rf ./preprocess_Result
- fi
- mkdir preprocess_Result
- python ../preprocess.py --csv=$data_path
-}
-
-function compile_app()
-{
- cd ../ascend310_infer || exit
- if [ -f "Makefile" ]; then
- make clean
- fi
- sh build.sh &> build.log
-
- if [ $? -ne 0 ]; then
- echo "compile app code failed"
- exit 1
- fi
- cd - || exit
-}
-
-function infer()
-{
- if [ -d result_Files ]; then
- rm -rf ./result_Files
- fi
- if [ -d time_Result ]; then
- rm -rf ./time_Result
- fi
- mkdir result_Files
- mkdir time_Result
- ../ascend310_infer/out/main --mindir_path=$model --input0_path=./preprocess_Result/00_item/ --input1_path=./preprocess_Result/01_rl1/ \
- --input2_path=./preprocess_Result/02_ety/ --input3_path=./preprocess_Result/03_rl2/ --input4_path=./preprocess_Result/04_his/ \
- --input5_path=./preprocess_Result/05_rate/ --device_id=$device_id &> infer.log
-
- if [ $? -ne 0 ]; then
- echo "execute inference failed"
- exit 1
- fi
-}
-
-function cal_acc()
-{
- python ../postprocess.py --result_Path=./result_Files \
- --label_Path=./preprocess_Result/05_rate &> acc.log
- if [ $? -ne 0 ]; then
- echo "calculate accuracy failed"
- exit 1
- fi
-}
-
-preprocess_data
-compile_app
-infer
-cal_acc
\ No newline at end of file
diff --git a/official/recommend/tbnet/scripts/run_standalone_train.sh b/official/recommend/tbnet/scripts/run_standalone_train.sh
deleted file mode 100644
index fbd633324..000000000
--- a/official/recommend/tbnet/scripts/run_standalone_train.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-if [[ $# -lt 2 || $# -gt 3 ]]; then
- echo "Usage: bash run_train.sh [DATA_NAME] [CUDA_VISIBLE_DEVICES]/[DEVICE_ID] [DEVICE_TARGET]
- DATA_NAME means dataset name, it's value is 'steam'.
- CUDA_VISIBLE_DEVICES means cuda visible device id.
- DEVICE_ID means device id, it can be set by environment variable DEVICE_ID.
- DEVICE_TARGET is optional, it's value is ['GPU', 'Ascend'], default 'GPU'."
-exit 1
-fi
-
-DATA_NAME=$1
-DEVICE_TARGET='GPU'
-
-if [ $# == 3 ]; then
- DEVICE_TARGET=$3
-fi
-
-
-if [ "$DEVICE_TARGET" = "GPU" ];
-then
- export CUDA_VISIBLE_DEVICES=$2
- python preprocess_dataset.py --dataset $DATA_NAME --device_target $DEVICE_TARGET &> scripts/train_standalone_log &&
- python train.py --dataset $DATA_NAME --device_target $DEVICE_TARGET --device_id 0 &>> scripts/train_standalone_log &
-fi
-
-if [ "$DEVICE_TARGET" = "Ascend" ];
-then
- export DEVICE_ID=$2
- python preprocess_dataset.py --dataset $DATA_NAME --device_target $DEVICE_TARGET &> scripts/train_standalone_log &&
- python train.py --dataset $DATA_NAME --device_target $DEVICE_TARGET --device_id $DEVICE_ID &>> scripts/train_standalone_log &
-fi
\ No newline at end of file
diff --git a/official/recommend/tbnet/src/aggregator.py b/official/recommend/tbnet/src/aggregator.py
deleted file mode 100644
index 394c5ab5d..000000000
--- a/official/recommend/tbnet/src/aggregator.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Inference result aggregator."""
-
-import copy
-
-
-class Recommendation:
- """Recommendation."""
-
- class Path:
- """Item path."""
- def __init__(self, relation1, entity, relation2, hist_item, importance):
- self.relation1 = relation1
- self.entity = entity
- self.relation2 = relation2
- self.hist_item = hist_item
- self.importance = importance
-
- class ItemRecord:
- """Recommended item info."""
- def __init__(self, item, score):
- self.item = item
- self.score = score
- # paths must be sorted with importance in descending order
- self.paths = []
-
- def __init__(self, user):
- self.user = user
- # item_records must be sorted with score in descending order
- self.item_records = []
-
-
-class InferenceAggregator:
- """
- Inference result aggregator.
-
- Args:
- top_k (int): The number of items to be recommended for each distinct user.
- """
- def __init__(self, top_k=1):
- if top_k < 1:
- raise ValueError('top_k is less than 1.')
- self._top_k = top_k
- self._user_recomms = dict()
- self._paths_sorted = False
-
- def aggregate(self, user, item, relation1, entity, relation2, hist_item, item_score, path_importance):
- """
- Aggregate inference results.
-
- Args:
- user (Tensor): User IDs, int Tensor in shape of [N, ].
- item (Tensor): Candidate item IDs, int Tensor in shape of [N, ].
- relation1 (Tensor): IDs of item-entity relations, int Tensor in shape of [N, <no. of per-item path>].
- entity (Tensor): Entity IDs, int Tensor in shape of [N, <no. of per-item path>].
- relation2 (Tensor): IDs of entity-hist_item relations, int Tensor in shape of [N, <no. of per-item path>].
- hist_item (Tensor): Historical item IDs, int Tensor in shape of [N, <no. of per-item path>].
- item_score (Tensor): TBNet output, recommendation scores of candidate items, float Tensor in shape of [N, ].
- path_importance (Tensor): TBNet output, the importance of each item to hist_item path for the
- recommendations, float Tensor in shape of [N, <no. of per-item path>].
- """
- user = user.asnumpy()
- item = item.asnumpy()
- relation1 = relation1.asnumpy()
- entity = entity.asnumpy()
- relation2 = relation2.asnumpy()
- hist_item = hist_item.asnumpy()
- item_score = item_score.asnumpy()
- path_importance = path_importance.asnumpy()
-
- batch_size = user.shape[0]
-
- added_users = set()
- for i in range(batch_size):
- if self._add(user[i], item[i], relation1[i], entity[i], relation2[i],
- hist_item[i], item_score[i], path_importance[i]):
- added_users.add(user[i])
- self._paths_sorted = False
-
- for added_user in added_users:
- recomm = self._user_recomms[added_user]
- if len(recomm.item_records) > self._top_k:
- recomm.item_records = recomm.item_records[0:self._top_k]
-
- def recommend(self):
- """
- Generate recommendations for all distinct users.
-
- Returns:
- dict[int, Recommendation], a dictionary with user id as keys and Recommendation objects as values.
- """
- if not self._paths_sorted:
- self._sort_paths()
- return copy.deepcopy(self._user_recomms)
-
- def _add(self, user, item, relation1, entity, relation2, hist_item, item_score, path_importance):
- """Add a single infer record."""
-
- recomm = self._user_recomms.get(user, None)
- if recomm is None:
- recomm = Recommendation(user)
- self._user_recomms[user] = recomm
-
- # insert at the appropriate position
- for i, old_item_rec in enumerate(recomm.item_records):
- if i >= self._top_k:
- return False
- if item_score > old_item_rec.score:
- rec = self._infer_2_item_rec(item, relation1, entity, relation2,
- hist_item, item_score, path_importance)
- recomm.item_records.insert(i, rec)
- return True
-
- # append if has rooms
- if len(recomm.item_records) < self._top_k:
- rec = self._infer_2_item_rec(item, relation1, entity, relation2,
- hist_item, item_score, path_importance)
- recomm.item_records.append(rec)
- return True
-
- return False
-
- @staticmethod
- def _infer_2_item_rec(item, relation1, entity, relation2, hist_item, item_score, path_importance):
- """Converts a single infer result to a item record."""
- item_rec = Recommendation.ItemRecord(item, item_score)
- num_paths = path_importance.shape[0]
- for i in range(num_paths):
- path = Recommendation.Path(relation1[i], entity[i], relation2[i], hist_item[i], path_importance[i])
- item_rec.paths.append(path)
- return item_rec
-
- def _sort_paths(self):
- """Sort all item paths."""
- for recomm in self._user_recomms.values():
- for item_rec in recomm.item_records:
- item_rec.paths.sort(key=lambda x: x.importance, reverse=True)
- self._paths_sorted = True
diff --git a/official/recommend/tbnet/src/config.py b/official/recommend/tbnet/src/config.py
deleted file mode 100644
index 0ad2a8cc5..000000000
--- a/official/recommend/tbnet/src/config.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TBNet configurations."""
-
-import json
-
-
-class TBNetConfig:
- """
- TBNet config file parser and holder.
-
- Args:
- config_path (str): json config file path.
- """
- def __init__(self, config_path):
- with open(config_path) as f:
- json_dict = json.load(f)
- self.num_items = int(json_dict['num_items'])
- self.num_relations = int(json_dict['num_relations'])
- self.num_references = int(json_dict['num_references'])
- self.per_item_paths = int(json_dict['per_item_paths'])
- self.embedding_dim = int(json_dict['embedding_dim'])
- self.batch_size = int(json_dict['batch_size'])
- self.lr = float(json_dict['lr'])
- self.kge_weight = float(json_dict['kge_weight'])
- self.node_weight = float(json_dict['node_weight'])
- self.l2_weight = float(json_dict['l2_weight'])
-
- def save(self, config_path):
- with open(config_path, 'w') as f:
- json.dump(self.__dict__, f, indent=4)
diff --git a/official/recommend/tbnet/src/dataset.py b/official/recommend/tbnet/src/dataset.py
deleted file mode 100644
index 34f4ac6d5..000000000
--- a/official/recommend/tbnet/src/dataset.py
+++ /dev/null
@@ -1,148 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Dataset loader."""
-
-import os
-from functools import partial
-
-import numpy as np
-import mindspore.dataset as ds
-import mindspore.mindrecord as record
-
-
-def create(data_path, per_item_num_paths, train, users=None, **kwargs):
- """
- Create a dataset for TBNet.
-
- Args:
- data_path (str): The csv datafile path.
- per_item_num_paths (int): The number of paths per item.
- train (bool): True to create for training with columns:
- 'item', 'relation1', 'entity', 'relation2', 'hist_item', 'rating'
- otherwise:
- 'user', 'item', 'relation1', 'entity', 'relation2', 'hist_item', 'rating'
- users (Union[list[int], int], optional): Users data to be loaded, if None is provided, all data will be loaded.
- **kwargs (any): Other arguments for GeneratorDataset(), except 'source' and 'column_names'.
-
- Returns:
- GeneratorDataset, the generator dataset that reads from the csv datafile.
- """
- if isinstance(users, int):
- users = (users,)
-
- if train:
- kwargs['columns_list'] = ['item', 'relation1', 'entity', 'relation2', 'hist_item', 'rating']
- else:
- kwargs['columns_list'] = ['user', 'item', 'relation1', 'entity', 'relation2', 'hist_item', 'rating']
- mindrecord_file_path = csv_dataset(partial(csv_generator, data_path, per_item_num_paths, users, train), data_path,
- train)
- return ds.MindDataset(mindrecord_file_path, **kwargs)
-
-
-def csv_dataset(generator, csv_path, train):
- """Dataset for csv datafile."""
- file_name = os.path.basename(csv_path)
- mindrecord_file_path = os.path.join(os.path.dirname(csv_path), file_name[0:file_name.rfind('.')] + '.mindrecord')
-
- if os.path.exists(mindrecord_file_path):
- os.remove(mindrecord_file_path)
-
- if os.path.exists(mindrecord_file_path + ".db"):
- os.remove(mindrecord_file_path + ".db")
-
- data_schema = {
- "item": {"type": "int32", "shape": []},
- "relation1": {"type": "int32", "shape": [-1]},
- "entity": {"type": "int32", "shape": [-1]},
- "relation2": {"type": "int32", "shape": [-1]},
- "hist_item": {"type": "int32", "shape": [-1]},
- "rating": {"type": "float32", "shape": []},
- }
- if not train:
- data_schema["user"] = {"type": "int32", "shape": []}
-
- writer = record.FileWriter(file_name=mindrecord_file_path, shard_num=1)
- writer.add_schema(data_schema, "Preprocessed dataset.")
-
- data = []
- for i, row in enumerate(generator()):
- if train:
- sample = {
- "item": row[0],
- "relation1": row[1],
- "entity": row[2],
- "relation2": row[3],
- "hist_item": row[4],
- "rating": row[5],
- }
- else:
- sample = {
- "user": row[0],
- "item": row[1],
- "relation1": row[2],
- "entity": row[3],
- "relation2": row[4],
- "hist_item": row[5],
- "rating": row[6],
- }
- data.append(sample)
-
- if i % 10 == 0:
- writer.write_raw_data(data)
- data = []
- if data:
- writer.write_raw_data(data)
- writer.commit()
- return mindrecord_file_path
-
-
-def csv_generator(csv_path, per_item_num_paths, users, train):
- """Generator for csv datafile."""
- expected_columns = 3 + (per_item_num_paths * 4)
- file = open(csv_path)
- for line in file:
- line = line.strip()
- if not line or line[0] == '#':
- continue
- id_list = line.split(',')
- if len(id_list) < expected_columns:
- raise ValueError(f'Expecting {expected_columns} values but got {len(id_list)} only!')
- id_list = list(map(int, id_list))
- user = id_list[0]
- if users and user not in users:
- continue
- item = id_list[1]
- rating = id_list[2]
-
- relation1 = np.empty(shape=(per_item_num_paths,), dtype=np.int)
- entity = np.empty_like(relation1)
- relation2 = np.empty_like(relation1)
- hist_item = np.empty_like(relation1)
-
- for p in range(per_item_num_paths):
- offset = 3 + (p * 4)
- relation1[p] = id_list[offset]
- entity[p] = id_list[offset + 1]
- relation2[p] = id_list[offset + 2]
- hist_item[p] = id_list[offset + 3]
-
- if train:
- # item, relation1, entity, relation2, hist_item, rating
- yield np.array(item, dtype=np.int), relation1, entity, relation2, hist_item, \
- np.array(rating, dtype=np.float32)
- else:
- # user, item, relation1, entity, relation2, hist_item, rating
- yield np.array(user, dtype=np.int), np.array(item, dtype=np.int), \
- relation1, entity, relation2, hist_item, np.array(rating, dtype=np.float32)
diff --git a/official/recommend/tbnet/src/embedding.py b/official/recommend/tbnet/src/embedding.py
deleted file mode 100644
index 87972953a..000000000
--- a/official/recommend/tbnet/src/embedding.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Three-dimension embedding vector initialization."""
-
-import mindspore.common.dtype as mstype
-from mindspore.common.tensor import Tensor
-from mindspore.ops import operations as P
-from mindspore.common.parameter import Parameter
-from mindspore.common.initializer import initializer
-from mindspore._checkparam import Rel
-from mindspore._checkparam import Validator as validator
-from mindspore.nn import Cell
-
-
-class EmbeddingMatrix(Cell):
- """
- Support three-dimension embedding vector initialization.
- """
-
- def __init__(self, vocab_size, embedding_size, embedding_table='normal',
- dtype=mstype.float32, padding_idx=None):
- super(EmbeddingMatrix, self).__init__()
- self.vocab_size = validator.check_value_type('vocab_size', vocab_size, [int], self.cls_name)
- self.embedding_size = validator.check_value_type('embedding_size', embedding_size,
- [int, tuple, list], self.cls_name)
- validator.check_subclass("dtype", dtype, mstype.number_type, self.cls_name)
- self.dtype = dtype
- if isinstance(self.embedding_size, int):
- self.init_tensor = initializer(embedding_table, [vocab_size, embedding_size])
- self.embedding_out = (self.embedding_size,)
- else:
- if len(self.embedding_size) != 2:
- raise ValueError("embedding_size should be a int or a tuple of two ints")
- self.init_tensor = initializer(embedding_table, [vocab_size, self.embedding_size[0],
- self.embedding_size[1]])
- self.embedding_out = (self.embedding_size[0], self.embedding_size[1],)
- self.padding_idx = padding_idx
- if padding_idx is not None:
- self.padding_idx = validator.check_int_range(padding_idx, 0, vocab_size, Rel.INC_BOTH,
- "padding_idx", self.cls_name)
- if isinstance(self.init_tensor, Tensor) and self.init_tensor.init is not None:
- self.init_tensor = self.init_tensor.init_data()
- self.init_tensor = self.init_tensor.asnumpy()
- self.init_tensor[self.padding_idx] = 0
- self.init_tensor = Tensor(self.init_tensor)
- self.embedding_table = Parameter(self.init_tensor, name='embedding_table')
- self.expand = P.ExpandDims()
- self.reshape_flat = P.Reshape()
- self.shp_flat = (-1,)
- self.gather = P.Gather()
- self.reshape = P.Reshape()
- self.get_shp = P.Shape()
-
- def construct(self, ids):
- """
- Return the initialized three-dimension embedding vector
- """
- extended_ids = self.expand(ids, -1)
- out_shape = self.get_shp(ids) + self.embedding_out
- flat_ids = self.reshape_flat(extended_ids, self.shp_flat)
- output_for_reshape = self.gather(self.embedding_table, flat_ids, 0)
- output = self.reshape(output_for_reshape, out_shape)
- return output
diff --git a/official/recommend/tbnet/src/metrics.py b/official/recommend/tbnet/src/metrics.py
deleted file mode 100644
index e1986c703..000000000
--- a/official/recommend/tbnet/src/metrics.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net metrics."""
-
-import numpy as np
-from sklearn.metrics import roc_auc_score
-from mindspore.nn.metrics import Metric
-
-
-class AUC(Metric):
- """TB-Net metrics method. Compute model metrics AUC."""
-
- def __init__(self):
- super(AUC, self).__init__()
- self.clear()
-
- def clear(self):
- """Clear the internal evaluation result."""
- self.true_labels = []
- self.pred_probs = []
-
- def update(self, *inputs):
- """Update list of predictions and labels."""
- all_predict = inputs[1].asnumpy().flatten().tolist()
- all_label = inputs[2].asnumpy().flatten().tolist()
- self.pred_probs.extend(all_predict)
- self.true_labels.extend(all_label)
-
- def eval(self):
- """Return AUC score"""
- if len(self.true_labels) != len(self.pred_probs):
- raise RuntimeError(
- 'true_labels.size is not equal to pred_probs.size()')
-
- auc = roc_auc_score(self.true_labels, self.pred_probs)
-
- return auc
-
-
-class ACC(Metric):
- """TB-Net metrics method. Compute model metrics ACC."""
-
- def __init__(self):
- super(ACC, self).__init__()
- self.clear()
-
- def clear(self):
- """Clear the internal evaluation result."""
- self.true_labels = []
- self.pred_probs = []
-
- def update(self, *inputs):
- """Update list of predictions and labels."""
- all_predict = inputs[1].asnumpy().flatten().tolist()
- all_label = inputs[2].asnumpy().flatten().tolist()
- self.pred_probs.extend(all_predict)
- self.true_labels.extend(all_label)
-
- def eval(self):
- """Return accuracy score"""
- if len(self.true_labels) != len(self.pred_probs):
- raise RuntimeError(
- 'true_labels.size is not equal to pred_probs.size()')
-
- predictions = [1 if i >= 0.5 else 0 for i in self.pred_probs]
- acc = np.mean(np.equal(predictions, self.true_labels))
-
- return acc
diff --git a/official/recommend/tbnet/src/path_gen.py b/official/recommend/tbnet/src/path_gen.py
deleted file mode 100644
index c53378cca..000000000
--- a/official/recommend/tbnet/src/path_gen.py
+++ /dev/null
@@ -1,435 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Relation path data generator."""
-import io
-import random
-import csv
-import warnings
-
-
-class _UserRec:
- """User record, helper class for path generation."""
-
- def __init__(self, src_id, intern_id):
- self.src_id = src_id
- self.intern_id = intern_id
- self.positive_items = dict()
- self.interact_items = dict()
- self.other_items = dict()
- self.has_unseen_ref = False
-
- def add_item(self, item_rec, rating):
- """Add an item."""
- if rating == 'p':
- item_dict = self.positive_items
- elif rating == 'c':
- item_dict = self.interact_items
- else:
- item_dict = self.other_items
- item_dict[item_rec.intern_id] = item_rec
-
-
-class _ItemRec:
- """Item record, helper class for path generation."""
-
- def __init__(self, src_id, intern_id, ref_src_ids, ref_ids):
- self.src_id = src_id
- self.intern_id = intern_id
- self.ref_src_ids = ref_src_ids
- self.ref_ids = ref_ids
-
-
-class PathGen:
- """
- Generate relation path csv from the source csv table.
-
- Args:
- per_item_paths (int): Number of relation paths per subject item, must be positive.
- same_relation (bool): True to only generate paths that relation1 is same as relation2, usually faster.
- id_maps (dict[str, Union[dict[str, int], int]], Optional): Object id maps, the internal id baseline, new user,
- item and entity IDs will be based on that. If Which is None or empty, grow_id_maps will be True by
- default.
- """
-
- def __init__(self, per_item_paths, same_relation=False, id_maps=None):
-
- self._per_item_paths = per_item_paths
- self._same_relation = same_relation
-
- self._user_id_counter = 1
- self._entity_id_counter = 1
- self._num_relations = 0
- self._rows_generated = 0
- self._user_rec = None
-
- if id_maps:
- self._item_id_map = id_maps.get('item', dict())
- self._ref_id_map = id_maps.get('reference', dict())
- self._rl_id_map = id_maps.get('relation', None)
- self._user_id_counter = id_maps.get('_user_id_counter', self._user_id_counter)
- max_item_id = max(self._item_id_map.values()) if self._item_id_map else 0
- max_ref_id = max(self._ref_id_map.values()) if self._ref_id_map else 0
- self._entity_id_counter = max(max_item_id, max_ref_id) + 1
- else:
- self._item_id_map = dict()
- self._ref_id_map = dict()
- self._rl_id_map = None
-
- self.grow_id_maps = not (bool(self._item_id_map) and bool(self._ref_id_map))
- self.subject_ratings = ""
-
- self._unseen_items = 0
- self._unseen_refs = 0
-
- @property
- def num_users(self):
- """int, the number of distinct users."""
- return self._user_id_counter - 1
-
- @property
- def num_references(self):
- """int, the number of distinct references."""
- return len(self._ref_id_map)
-
- @property
- def num_items(self):
- """int, the number of distinct items."""
- return len(self._item_id_map)
-
- @property
- def num_relations(self):
- """int, the number of distinct relations."""
- return self._num_relations
-
- @property
- def rows_generated(self):
- """int, total number of rows generated to the output CSVs."""
- return self._rows_generated
-
- @property
- def per_item_paths(self):
- """int, the number of path per subject item."""
- return self._per_item_paths
-
- @property
- def same_relation(self):
- """bool, only generate paths with the same relation on both sides."""
- return self._same_relation
-
- @property
- def unseen_items(self):
- """int, total number of unseen items has encountered."""
- return self._unseen_items
-
- @property
- def unseen_refs(self):
- """int, total number of unseen references has encountered."""
- return self._unseen_refs
-
- def id_maps(self):
- """dict, object ID maps."""
- maps = {
- "item": dict(self._item_id_map),
- "reference": dict(self._ref_id_map),
- "_user_id_counter": self._user_id_counter
- }
- if self._rl_id_map is not None:
- maps["relation"] = dict(self._rl_id_map)
- return maps
-
- def generate(self, in_csv, out_csv, in_sep=',', in_mv_sep=';', in_encoding='utf-8'):
- """
- Generate paths csv from the source CSV files.
-
- args:
- in_csv (Union[str, TextIOBase]): The input source csv path or stream.
- out_csv (Union[str, TextIOBase]): The output source csv path or stream.
- in_sep (str): Separator of the input csv.
- in_mv_sep (str): Multi-value separator of the input csv in a single column.
- in_encoding (str): Encoding of the input source csv, ignored if in_csv is a text stream already.
-
- Returns:
- int, the number of rows that generated to the output csv in this call.
- """
- if not isinstance(in_csv, (str, io.TextIOBase)):
- raise TypeError(f"Unexpected in_csv type:{type(in_csv)}")
- if not isinstance(out_csv, (str, io.TextIOBase)):
- raise TypeError(f"Unexpected out_csv type:{type(out_csv)}")
-
- opened_files = []
- try:
- if isinstance(in_csv, str):
- in_csv = io.open(in_csv, mode="r", encoding=in_encoding)
- opened_files.append(in_csv)
- in_csv = csv.reader(in_csv, delimiter=in_sep)
- col_indices = self._pre_generate(in_csv, None)
-
- if isinstance(out_csv, str):
- out_csv = io.open(out_csv, mode="w", encoding="ascii")
- opened_files.append(out_csv)
- rows_generated = self._do_generate(in_csv, out_csv, in_mv_sep, col_indices)
-
- except (IOError, ValueError, RuntimeError, PermissionError, KeyError) as e:
- raise e
- finally:
- for f in opened_files:
- f.close()
- return rows_generated
-
- def _pre_generate(self, in_csv, in_col_map):
- """Prepare for the path generation."""
- if in_col_map is not None:
- expected_cols = self._default_abstract_header(len(in_col_map) - 3)
- map_values = list(in_col_map.values())
- for col in expected_cols:
- if col not in map_values:
- raise ValueError("col_map has no '{col}' value.")
-
- header = self._read_header(in_csv)
- if len(header) < 4:
- raise IOError(f"No. of in_csv columns:{len(header)} is less than 4.")
- num_relations = len(header) - 3
- if self._num_relations > 0:
- if num_relations != self._num_relations:
- raise IOError(f"Inconsistent no. of in_csv relations.")
- else:
- self._num_relations = num_relations
-
- col_indices = self._get_col_indices(header, in_col_map)
- rl_id_map = self._to_relation_id_map(header, col_indices)
-
- if not self._rl_id_map:
- self._rl_id_map = rl_id_map
- elif rl_id_map != self._rl_id_map:
- raise IOError(f"Inconsistent in_csv relations.")
-
- return col_indices
-
- def _do_generate(self, in_csv, out_csv, in_mv_sep, col_indices):
- """Do generate the paths."""
- old_rows_generated = self._rows_generated
- old_unseen_items = self._unseen_items
- old_unseen_refs = self._unseen_refs
-
- col_count = len(col_indices)
- self._user_rec = None
- for line in in_csv:
- values = list(map(lambda x: x.strip(), line))
- if len(values) != col_count:
- raise IOError(f"No. of in_csv columns:{len(values)} is not {col_count}.")
- self._process_line(values, in_mv_sep, col_indices, out_csv)
-
- if self._user_rec is not None:
- self._process_user_rec(self._user_rec, out_csv)
- self._user_rec = None
-
- delta_unseen_items = self._unseen_items - old_unseen_items
- delta_unseen_refs = self._unseen_refs - old_unseen_refs
- if delta_unseen_items > 0:
- warnings.warn(f"{delta_unseen_items} unseen items' internal IDs were set to 0, "
- f"set grow_id_maps to True for adding new internal IDs.", RuntimeWarning)
- if delta_unseen_refs > 0:
- warnings.warn(f"{delta_unseen_refs} unseen references' internal IDs were set to 0, "
- f"set grow_id_maps to True for adding new internal IDs.", RuntimeWarning)
-
- return self._rows_generated - old_rows_generated
-
- def _process_line(self, values, in_mv_sep, col_indices, out_csv):
- """Process a line from the input CSV."""
- user_src = values[col_indices[0]]
- item_src = values[col_indices[1]]
- rating = values[col_indices[2]].lower()
- if rating not in ('p', 'c', 'x'):
- raise IOError(f"Unrecognized rating:'{rating}', must be one of 'p', 'c' or 'x'.")
- ref_srcs = [values[col_indices[i]] for i in range(3, len(col_indices))]
-
- if in_mv_sep:
- ref_srcs = list(map(lambda x: list(map(lambda y: y.strip(), x.split(in_mv_sep))), ref_srcs))
- else:
- ref_srcs = list(map(lambda x: [x], ref_srcs))
-
- if self._user_rec is not None and user_src != self._user_rec.src_id:
- # user changed
- self._process_user_rec(self._user_rec, out_csv)
- self._user_rec = None
-
- if self._user_rec is None:
- self._user_rec = _UserRec(user_src, self._user_id_counter)
- self._user_id_counter += 1
-
- item_rec, has_unseen_ref = self._to_item_rec(item_src, ref_srcs)
- self._user_rec.add_item(item_rec, rating)
- self._user_rec.has_unseen_ref |= has_unseen_ref
-
- def _process_user_rec(self, user_rec, out_csv):
- """Generate paths for an user."""
- positive_count = 0
-
- subject_items = []
-
- if self.subject_ratings == "":
- subject_items.extend(user_rec.positive_items.values())
- subject_items.extend(user_rec.other_items.values())
- positive_count = len(user_rec.positive_items)
- else:
- if 'p' in self.subject_ratings:
- subject_items.extend(user_rec.positive_items.values())
- positive_count = len(user_rec.positive_items)
- if 'c' in self.subject_ratings:
- subject_items.extend(user_rec.interact_items.values())
- if 'x' in self.subject_ratings:
- subject_items.extend(user_rec.other_items.values())
-
- hist_items = []
- hist_items.extend(user_rec.positive_items.values())
- hist_items.extend(user_rec.interact_items.values())
-
- for i, subject in enumerate(subject_items):
-
- paths = []
- for hist in hist_items:
- if hist.src_id == subject.src_id:
- continue
- self._find_paths(not user_rec.has_unseen_ref, subject, hist, paths)
-
- if not paths:
- continue
-
- paths = random.sample(paths, min(len(paths), self._per_item_paths))
-
- row = [0] * (3 + self._per_item_paths * 4)
- row[0] = user_rec.src_id
- row[1] = subject.intern_id # subject item
- row[2] = 1 if i < positive_count else 0 # label
- for p, path in enumerate(paths):
- offset = 3 + p * 4
- for j in range(4):
- row[offset + j] = path[j]
- out_csv.write(','.join(map(str, row)))
- out_csv.write('\n')
- self._rows_generated += 1
-
- def _find_paths(self, by_intern_id, subject_item, hist_item, paths):
- """Find paths between the subject and historical item."""
- if by_intern_id:
- for i, ref_list in enumerate(subject_item.ref_ids):
- for ref in ref_list:
- self._find_paths_by_intern_id(i, ref, hist_item, paths)
- else:
- for i, (ref_src_list, ref_list) in enumerate(zip(subject_item.ref_src_ids,
- subject_item.ref_ids)):
- for src_ref, ref in zip(ref_src_list, ref_list):
- self._find_paths_by_src(i, src_ref, ref, hist_item, paths)
-
- def _find_paths_by_intern_id(self, subject_ridx, ref_id, hist_item, paths):
- """Find paths by internal reference ID, a bit faster."""
- if self._same_relation:
- if ref_id in hist_item.ref_ids[subject_ridx]:
- relation_id = self._ridx_to_relation_id(subject_ridx)
- paths.append((relation_id,
- ref_id,
- relation_id,
- hist_item.intern_id))
- else:
- for hist_ridx, hist_ref_list in enumerate(hist_item.ref_ids):
- if ref_id in hist_ref_list:
- paths.append((self._ridx_to_relation_id(subject_ridx),
- ref_id,
- self._ridx_to_relation_id(hist_ridx),
- hist_item.intern_id))
-
- def _find_paths_by_src(self, subject_ridx, ref_src_id, ref_id, hist_item, paths):
- """Find paths by source reference ID."""
- if self._same_relation:
- if ref_src_id in hist_item.ref_src_ids[subject_ridx]:
- relation_id = self._ridx_to_relation_id(subject_ridx)
- paths.append((relation_id,
- ref_id,
- relation_id,
- hist_item.intern_id))
- else:
- for hist_ridx, hist_ref_src_list in enumerate(hist_item.ref_src_ids):
- if ref_src_id in hist_ref_src_list:
- paths.append((self._ridx_to_relation_id(subject_ridx),
- ref_id,
- self._ridx_to_relation_id(hist_ridx),
- hist_item.intern_id))
-
- def _ridx_to_relation_id(self, idx):
- """Relation index to id."""
- return idx
-
- def _to_relation_id_map(self, header, col_indices):
- """Convert input csv header to a relation id map."""
- id_map = {}
- id_counter = 0
- for i in range(3, len(col_indices)):
- id_map[header[col_indices[i]]] = id_counter
- id_counter += 1
- if len(id_map) < len(header) - 3:
- raise IOError("Duplicated column!")
- return id_map
-
- def _to_item_rec(self, item_src, ref_srcs):
- """Convert the item src id and the source reference to an item record."""
- item_id = self._item_id_map.get(item_src, -1)
- if item_id == -1:
- if not self.grow_id_maps:
- item_id = 0
- self._unseen_items += 1
- else:
- item_id = self._entity_id_counter
- self._item_id_map[item_src] = item_id
- self._entity_id_counter += 1
-
- has_unseen_ref = False
- ref_ids = [[] for _ in range(len(ref_srcs))]
- for i, ref_src_list in enumerate(ref_srcs):
- for ref_src in ref_src_list:
- ref_id = self._ref_id_map.get(ref_src, -1)
- if ref_id == -1:
- if not self.grow_id_maps:
- ref_id = 0
- self._unseen_refs += 1
- has_unseen_ref = True
- else:
- ref_id = self._entity_id_counter
- self._ref_id_map[ref_src] = ref_id
- self._entity_id_counter += 1
- ref_ids[i].append(ref_id)
-
- return _ItemRec(item_src, item_id, ref_srcs, ref_ids), has_unseen_ref
-
- def _get_col_indices(self, header, col_map):
- """Find the column indices base on the mapping."""
- if col_map:
- mapped = [col_map[col] for col in header]
- default_header = self._default_abstract_header(len(header) - 3)
- return [mapped.index(col) for col in default_header]
- return range(len(header))
-
- @staticmethod
- def _read_header(in_csv):
- """Read the CSV header."""
- line = next(in_csv)
- splited = list(map(lambda x: x.strip(), line))
- return splited
-
- @staticmethod
- def _default_abstract_header(num_relation):
- """Get the default abstract header."""
- abstract_header = ["user", "item", "rating"]
- abstract_header.extend([f"r{i + 1}" for i in num_relation])
- return abstract_header
diff --git a/official/recommend/tbnet/src/steam.py b/official/recommend/tbnet/src/steam.py
deleted file mode 100644
index ba58f8a8e..000000000
--- a/official/recommend/tbnet/src/steam.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""STEAM dataset explainer."""
-
-import json
-
-from src.aggregator import Recommendation
-
-
-class TextExplainer:
- """Text explainer for STEAM game recommendations."""
- SAME_RELATION_TPL = 'User played the game <%s> before, which has the same %s '\
- '<%s> as the recommend game.'
- DIFF_RELATION_TPL = 'User played the game <%s> before, which has the %s <%s> '\
- 'while <%s> is the %s of the recommended game.'
-
- def __init__(self, translate_path: str):
- """Construct from the translate json file."""
- with open(translate_path) as file:
- self._translator = json.load(file)
-
- def explain(self, path: Recommendation.Path) -> str:
- """Explain the path."""
- rel1_str = self.translate_relation(path.relation1)
- entity_str = self.translate_entity(path.entity)
- hist_item_str = self.translate_item(path.hist_item)
- if path.relation1 == path.relation2:
- return self.SAME_RELATION_TPL % (hist_item_str, rel1_str, entity_str)
- rel2_str = self.translate_relation(path.relation2)
- return self.DIFF_RELATION_TPL % (hist_item_str, rel2_str, entity_str, entity_str, rel1_str)
-
- def translate_item(self, item: int) -> str:
- """Translate an item."""
- return self._translate('item', item)
-
- def translate_entity(self, entity: int) -> str:
- """Translate an entity."""
- return self._translate('entity', entity)
-
- def translate_relation(self, relation: int) -> str:
- """Translate a relation."""
- return self._translate('relation', relation)
-
- def _translate(self, obj_type, obj_id):
- """Translate an object."""
- try:
- return self._translator[obj_type][str(obj_id)]
- except KeyError:
- return f'[{obj_type}:{obj_id}]'
diff --git a/official/recommend/tbnet/src/tbnet.py b/official/recommend/tbnet/src/tbnet.py
deleted file mode 100644
index bc67715b8..000000000
--- a/official/recommend/tbnet/src/tbnet.py
+++ /dev/null
@@ -1,383 +0,0 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net Model."""
-
-from mindspore import nn, Tensor
-from mindspore import ParameterTuple
-from mindspore.ops import operations as P
-from mindspore.ops import functional as F
-from mindspore.ops import composite as C
-from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_gradients_mean
-from mindspore.context import ParallelMode
-from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
-import mindspore.common.dtype as mstype
-
-from src.embedding import EmbeddingMatrix
-
-grad_scale = C.MultitypeFuncGraph("grad_scale")
-
-
-@grad_scale.register("Tensor", "Tensor")
-def gradient_scale(scale, grad):
- return grad * F.cast(scale, F.dtype(grad))
-
-
-class TBNet(nn.Cell):
- """
- TB-Net model architecture.
-
- Args:
- num_entity (int): number of entities, depends on dataset
- num_relation (int): number of relations, depends on dataset
- dim (int): dimension of entity and relation embedding vectors
- kge_weight (float): weight of the KG Embedding loss term
- node_weight (float): weight of the node loss term (default=0.002)
- l2_weight (float): weight of the L2 regularization term (default=1e-7)
- lr (float): learning rate of model training (default=1e-4)
- batch_size (int): batch size (default=1024)
- """
-
- def __init__(self, config):
- super(TBNet, self).__init__()
-
- self._parse_config(config)
- self.matmul = C.matmul
- self.sigmoid = P.Sigmoid()
- self.embedding_initializer = "normal"
-
- self.entity_emb_matrix = EmbeddingMatrix(int(self.num_entity),
- self.dim,
- embedding_table=self.embedding_initializer)
- self.relation_emb_matrix = EmbeddingMatrix(int(self.num_relation),
- embedding_size=(self.dim, self.dim),
- embedding_table=self.embedding_initializer)
-
- self.expand_dims = P.ExpandDims()
- self.squeeze = P.Squeeze(3)
- self.abs = P.Abs()
- self.reduce_sum = P.ReduceSum()
- self.reduce_mean = P.ReduceMean()
-
- self.transpose = P.Transpose()
- self.reshape = P.Reshape()
- self.softmax = nn.Softmax()
-
- def _parse_config(self, config):
- """Argument parsing."""
-
- self.num_entity = config.num_items + config.num_references + 1
- self.num_relation = config.num_relations
- self.dim = config.embedding_dim
- self.kge_weight = config.kge_weight
- self.node_weight = config.node_weight
- self.l2_weight = config.l2_weight
- self.lr = config.lr
- self.batch_size = config.batch_size
-
- def construct(self, items, relation1, mid_entity, relation2, hist_item):
- """
- TB-Net main computation process.
-
- Args:
- items (Tensor): rated item IDs, int Tensor in shape of [batch size, ].
- relation1 (Tensor): relation1 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- mid_entity (Tensor): middle entity IDs, int Tensor in shape of [batch size, per_item_num_paths]
- relation2 (Tensor): relation2 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- hist_item (Tensor): historical item IDs, int Tensor in shape of [batch size, per_item_num_paths]
-
- Returns:
- scores (Tensor): model prediction score, float Tensor in shape of [batch size, ]
- probs_exp (Tensor): path probability/importance, float Tensor in shape of [batch size, per_item_num_paths]
- item_embeddings (Tensor): rated item embeddings, float Tensor in shape of [batch size, dim]
- relation1_emb (Tensor): relation1 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- mid_entity_emb (Tensor): middle entity embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
- relation2_emb (Tensor): relation2 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- hist_item_emb (Tensor): historical item embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
- """
- item_embeddings = self.entity_emb_matrix(items)
-
- relation1_emb = self.relation_emb_matrix(relation1)
- mid_entity_emb = self.entity_emb_matrix(mid_entity)
- relation2_emb = self.relation_emb_matrix(relation2)
- hist_item_emb = self.entity_emb_matrix(hist_item)
-
- response, probs_exp = self._key_pathing(item_embeddings,
- relation1_emb,
- mid_entity_emb,
- relation2_emb,
- hist_item_emb)
-
- scores = P.Squeeze()(self._predict(item_embeddings, response))
-
- return scores, probs_exp, item_embeddings, relation1_emb, mid_entity_emb, relation2_emb, hist_item_emb
-
- def _key_pathing(self, item_embeddings, relation1_emb, mid_entity_emb, relation2_emb, hist_item_emb):
- """
- Compute the response and path probability using item and entity embedding.
- Path structure: (rated item, relation1, entity, relation2, historical item).
-
- Args:
- item_embeddings (Tensor): rated item embeddings, float Tensor in shape of [batch size, dim]
- relation1_emb (Tensor): relation1 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- mid_entity_emb (Tensor): middle entity embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
- relation2_emb (Tensor): relation2 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- hist_item_emb (Tensor): historical item embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
-
- Returns:
- response (Tensor): user's response towards middle entity, float Tensor in shape of [batch size, dim]
- probs_exp (Tensor): path probability/importance, float Tensor in shape of [batch size, per_item_num_paths]
- """
-
- hist_item_e_4d = self.expand_dims(hist_item_emb, 3)
- mul_r2_hist = self.squeeze(self.matmul(relation2_emb, hist_item_e_4d))
- # path_right shape: [batch size, per_item_num_paths, dim]
- path_right = self.abs(mul_r2_hist + self.reduce_sum(relation2_emb, 2))
-
- item_emb_3d = self.expand_dims(item_embeddings, 2)
- mul_r1_item = self.squeeze(self.matmul(relation1_emb, self.expand_dims(item_emb_3d, 1)))
- path_left = self.abs(mul_r1_item + self.reduce_sum(relation1_emb, 2))
- # path_left shape: [batch size, dim, per_item_num_paths]
- path_left = self.transpose(path_left, (0, 2, 1))
-
- probs = self.reduce_sum(self.matmul(path_right, path_left), 2)
- # probs_exp shape: [batch size, per_item_num_paths]
- probs_exp = self.softmax(probs)
-
- probs_3d = self.expand_dims(probs_exp, 2)
- # response shape: [batch size, dim]
- response = self.reduce_sum(mid_entity_emb * probs_3d, 1)
-
- return response, probs_exp
-
- def _predict(self, item_embeddings, response):
- scores = self.reduce_sum(item_embeddings * response, 1)
-
- return scores
-
-
-class NetWithLossClass(nn.Cell):
- """NetWithLossClass definition."""
-
- def __init__(self, network, config):
- super(NetWithLossClass, self).__init__(auto_prefix=False)
- self.network = network
- self.loss = P.SigmoidCrossEntropyWithLogits()
- self.matmul = C.matmul
- self.expand_dims = P.ExpandDims()
- self.squeeze = P.Squeeze(3)
- self.abs = P.Abs()
- self.maximum = P.Maximum()
- self.reduce_sum = P.ReduceSum()
- self.reduce_mean = P.ReduceMean()
- self.cast = P.Cast()
- self.reshape = P.Reshape()
- self.sigmoid = P.Sigmoid()
-
- self.kge_weight = config.kge_weight
- self.node_weight = config.node_weight
- self.l2_weight = config.l2_weight
- self.batch_size = config.batch_size
- self.dim = config.embedding_dim
-
- self.embedding_initializer = "normal"
-
- def construct(self, items, relation1, mid_entity, relation2, hist_item, labels):
- """
- Args:
- items (Tensor): rated item IDs, int Tensor in shape of [batch size, ].
- relation1 (Tensor): relation1 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- mid_entity (Tensor): middle entity IDs, int Tensor in shape of [batch size, per_item_num_paths]
- relation2 (Tensor): relation2 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- hist_item (Tensor): historical item IDs, int Tensor in shape of [batch size, per_item_num_paths]
- labels (Tensor): label of rated item record, int Tensor in shape of [batch size, ]
-
- Returns:
- loss (float): loss value
- """
- scores, _, item_embeddings, relation1_emb, mid_entity_emb, relation2_emb, hist_item_emb = \
- self.network(items, relation1, mid_entity, relation2, hist_item)
- loss = self._loss_fun(item_embeddings, relation1_emb, mid_entity_emb,
- relation2_emb, hist_item_emb, scores, labels)
-
- return loss
-
- def _loss_fun(self, item_embeddings, relation1_emb, mid_entity_emb, relation2_emb, hist_item_emb, scores, labels):
- """
- Loss function definition.
-
- Args:
- item_embeddings (Tensor): rated item embeddings, float Tensor in shape of [batch size, dim]
- relation1_emb (Tensor): relation1 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- mid_entity_emb (Tensor): middle entity embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
- relation2_emb (Tensor): relation2 embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim, dim]
- hist_item_emb (Tensor): historical item embeddings,
- float Tensor in shape of [batch size, per_item_num_paths, dim]
- scores (Tensor): model prediction score, float Tensor in shape of [batch size, ]
- labels (Tensor): label of rated item record, int Tensor in shape of [batch size, ]
- Returns:
- loss: includes four part:
- pred_loss: cross entropy of the model prediction score and labels
- transr_loss: TransR KG Embedding loss
- node_loss: node matching loss
- l2_loss: L2 regularization loss
- """
- pred_loss = self.reduce_mean(self.loss(scores, labels))
-
- item_emb_3d = self.expand_dims(item_embeddings, 2)
- item_emb_4d = self.expand_dims(item_emb_3d, 1)
-
- mul_r1_item = self.squeeze(self.matmul(relation1_emb, item_emb_4d))
-
- hist_item_e_4d = self.expand_dims(hist_item_emb, 3)
- mul_r2_hist = self.squeeze(self.matmul(relation2_emb, hist_item_e_4d))
-
- relation1_3d = self.reduce_sum(relation1_emb, 2)
- relation2_3d = self.reduce_sum(relation2_emb, 2)
-
- path_left = self.reduce_sum(self.abs(mul_r1_item + relation1_3d), 2)
- path_right = self.reduce_sum(self.abs(mul_r2_hist + relation2_3d), 2)
-
- transr_loss = self.reduce_sum(self.maximum(self.abs(path_left - path_right), 0))
- transr_loss = self.reduce_mean(self.sigmoid(transr_loss))
-
- mid_entity_emb_4d = self.expand_dims(mid_entity_emb, 3)
- mul_r2_mid = self.squeeze(self.matmul(relation2_emb, mid_entity_emb_4d))
- path_r2_mid = self.abs(mul_r2_mid + relation2_3d)
-
- node_loss = self.reduce_sum(self.maximum(mul_r2_hist - path_r2_mid, 0))
- node_loss = self.reduce_mean(self.sigmoid(node_loss))
-
- l2_loss = self.reduce_mean(self.reduce_sum(relation1_emb * relation1_emb))
- l2_loss += self.reduce_mean(self.reduce_sum(mid_entity_emb * mid_entity_emb))
- l2_loss += self.reduce_mean(self.reduce_sum(relation2_emb * relation2_emb))
- l2_loss += self.reduce_mean(self.reduce_sum(hist_item_emb * hist_item_emb))
-
- transr_loss = self.kge_weight * transr_loss
- node_loss = self.node_weight * node_loss
-
- l2_loss = self.l2_weight * l2_loss
-
- loss = pred_loss + transr_loss + node_loss + l2_loss
-
- return loss
-
-
-class TrainStepWrap(nn.Cell):
- """TrainStepWrap definition."""
-
- def __init__(self, network, lr, sens=1, loss_scale=False):
- super(TrainStepWrap, self).__init__(auto_prefix=False)
- self.network = network
- self.network.set_train()
- self.network.add_flags(defer_inline=True)
- self.weights = ParameterTuple(network.trainable_params())
- self.lr = lr
- self.optimizer = nn.Adam(self.weights,
- learning_rate=self.lr,
- beta1=0.9,
- beta2=0.999,
- eps=1e-8,
- loss_scale=sens)
-
- self.hyper_map = C.HyperMap()
- self.reciprocal_sense = Tensor(1 / sens, mstype.float32)
- self.grad = C.GradOperation(get_by_list=True, sens_param=True)
- self.sens = sens
-
- self.reducer_flag = False
- self.grad_reducer = None
- self.loss_scale = loss_scale
- parallel_mode = _get_parallel_mode()
- if parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL):
- self.reducer_flag = True
- if self.reducer_flag:
- mean = _get_gradients_mean()
- degree = _get_device_num()
- self.grad_reducer = DistributedGradReducer(self.optimizer.parameters, mean, degree)
-
- def scale_grad(self, gradients):
- gradients = self.hyper_map(F.partial(grad_scale, self.reciprocal_sense), gradients)
- return gradients
-
- def construct(self, items, relation1, mid_entity, relation2, hist_item, labels):
- """
- Args:
- items (Tensor): rated item IDs, int Tensor in shape of [batch size, ].
- relation1 (Tensor): relation1 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- mid_entity (Tensor): middle entity IDs, int Tensor in shape of [batch size, per_item_num_paths]
- relation2 (Tensor): relation2 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- hist_item (Tensor): historical item IDs, int Tensor in shape of [batch size, per_item_num_paths]
- labels (Tensor): label of rated item record, int Tensor in shape of [batch size, ]
-
- Returns:
- loss and gradient
- """
- weights = self.weights
- loss = self.network(items, relation1, mid_entity, relation2, hist_item, labels)
- sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
- grads = self.grad(self.network, weights)(items, relation1, mid_entity, relation2, hist_item, labels, sens)
-
- if self.loss_scale:
- grads = self.scale_grad(grads)
-
- if self.reducer_flag:
- # apply grad reducer on grads
- grads = self.grad_reducer(grads)
-
- return F.depend(loss, self.optimizer(grads))
-
-
-class PredictWithSigmoid(nn.Cell):
- """Predict method."""
-
- def __init__(self, network):
- super(PredictWithSigmoid, self).__init__(auto_prefix=False)
- self.network = network
- self.sigmoid = P.Sigmoid()
-
- def construct(self, items, relation1, mid_entity, relation2, hist_item, labels):
- """
- Predict with sigmoid definition.
-
- Args:
- items (Tensor): rated item IDs, int Tensor in shape of [batch size, ].
- relation1 (Tensor): relation1 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- mid_entity (Tensor): middle entity IDs, int Tensor in shape of [batch size, per_item_num_paths]
- relation2 (Tensor): relation2 IDs, int Tensor in shape of [batch size, per_item_num_paths]
- hist_item (Tensor): historical item IDs, int Tensor in shape of [batch size, per_item_num_paths]
- labels (Tensor): label of rated item record, int Tensor in shape of [batch size, ]
-
- Returns:
- scores (Tensor): model prediction score, float Tensor in shape of [batch size, ]
- pred_probs (Tensor): prediction probability, float Tensor in shape of [batch size, ]
- labels (Tensor): label of rated item record, int Tensor in shape of [batch size, ]
- probs_exp (Tensor): path probability/importance, float Tensor in shape of [batch size, per_item_num_paths]
- """
-
- scores, probs_exp, _, _, _, _, _ = self.network(items, relation1, mid_entity, relation2, hist_item)
- pred_probs = self.sigmoid(scores)
-
- return scores, pred_probs, labels, probs_exp
diff --git a/official/recommend/tbnet/src/utils/__init__.py b/official/recommend/tbnet/src/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/official/recommend/tbnet/src/utils/device_adapter.py b/official/recommend/tbnet/src/utils/device_adapter.py
deleted file mode 100644
index 22ef7ea73..000000000
--- a/official/recommend/tbnet/src/utils/device_adapter.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License Version 2.0(the "License");
-# you may not use this file except in compliance with the License.
-# you may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0#
-#
-# Unless required by applicable law or agreed to in writing software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANT IES OR CONITTONS OF ANY KIND锛� either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ====================================================================================
-
-"""Device adapter for ModelArts"""
-
-from .param import config
-if config.enable_modelarts:
- from .moxing_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
-else:
- from .local_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
-
-__all__ = [
- 'get_device_id', 'get_device_num', 'get_job_id', 'get_rank_id'
-]
diff --git a/official/recommend/tbnet/src/utils/local_adapter.py b/official/recommend/tbnet/src/utils/local_adapter.py
deleted file mode 100644
index edba7343d..000000000
--- a/official/recommend/tbnet/src/utils/local_adapter.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License Version 2.0(the "License");
-# you may not use this file except in compliance with the License.
-# you may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0#
-#
-# Unless required by applicable law or agreed to in writing software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANT IES OR CONITTONS OF ANY KIND锛� either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ====================================================================================
-
-"""Local adapter"""
-
-import os
-
-
-def get_device_id():
- device_id = os.getenv('DEVICE_ID', '0')
- return int(device_id)
-
-
-def get_device_num():
- device_num = os.getenv('RANK_SIZE', '1')
- return int(device_num)
-
-
-def get_rank_id():
- global_rank_id = os.getenv('RANK_ID', '0')
- return int(global_rank_id)
-
-
-def get_job_id():
- return 'Local Job'
diff --git a/official/recommend/tbnet/src/utils/moxing_adapter.py b/official/recommend/tbnet/src/utils/moxing_adapter.py
deleted file mode 100644
index f3261f7e2..000000000
--- a/official/recommend/tbnet/src/utils/moxing_adapter.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License Version 2.0(the "License");
-# you may not use this file except in compliance with the License.
-# you may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0#
-#
-# Unless required by applicable law or agreed to in writing software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANT IES OR CONITTONS OF ANY KIND锛� either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ====================================================================================
-
-"""Moxing adapter for ModelArts"""
-
-import os
-import functools
-from mindspore import context
-from .param import param
-
-
-_global_syn_count = 0
-
-
-def get_device_id():
- device_id = os.getenv('DEVICE_ID', '0')
- return int(device_id)
-
-
-def get_device_num():
- device_num = os.getenv('RANK_SIZE', '1')
- return int(device_num)
-
-
-def get_rank_id():
- global_rank_id = os.getenv('RANK_ID', '0')
- return int(global_rank_id)
-
-
-def get_job_id():
- job_id = os.getenv('JOB_ID')
- job_id = job_id if job_id != "" else "default"
- return job_id
-
-
-def sync_data(from_path, to_path):
- """
- Download data from remote obs to local directory if the first url is remote url and the second one is local
- Uploca data from local directory to remote obs in contrast
- """
- import moxing as mox
- import time
- global _global_syn_count
- sync_lock = '/tmp/copy_sync.lock' + str(_global_syn_count)
- _global_syn_count += 1
-
- # Each server contains 8 devices as most
- if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
- print('from path: ', from_path)
- print('to path: ', to_path)
- mox.file.copy_parallel(from_path, to_path)
- print('===finished data synchronization===')
- try:
- os.mknod(sync_lock)
- except IOError:
- pass
- print('===save flag===')
-
- while True:
- if os.path.exists(sync_lock):
- break
- time.sleep(1)
- print('Finish sync data from {} to {}'.format(from_path, to_path))
-
-
-def moxing_wrapper(pre_process=None, post_process=None):
- """
- Moxing wrapper to download dataset and upload outputs
- """
- def wrapper(run_func):
- @functools.wraps(run_func)
- def wrapped_func(*args, **kwargs):
- # Download data from data_url
- if param.enable_modelarts:
- if param.data_url:
- sync_data(param.data_url, param.data_path)
- print('Dataset downloaded: ', os.listdir(param.data_path))
- if param.checkpoint_url or param.ckpt_url:
- if not os.path.exists(param.load_path):
- os.makedirs(param.load_path)
- print('=' * 20 + 'makedirs')
- if os.path.isdir(param.load_path):
- print('=' * 20 + 'makedirs success')
- else:
- print('=' * 20 + 'makedirs fail')
- if param.checkpoint_url:
- sync_data(param.checkpoint_url, param.load_path)
- else:
- sync_data(os.path.dirname(param.ckpt_url), param.load_path)
- print('Preload downloaded: ', os.listdir(param.load_path))
- if param.train_url:
- sync_data(param.train_url, param.output_path)
- print('Workspace downloaded: ', os.listdir(param.output_path))
-
- context.set_context(save_graphs_path=os.path.join(param.output_path, str(get_rank_id())))
- param.device_num = get_device_num()
- param.device_id = get_device_id()
- if not os.path.exists(param.output_path):
- os.makedirs(param.output_path)
-
- if pre_process:
- pre_process()
-
- run_func(*args, **kwargs)
-
- # Upload data to train_url
- if param.enable_modelarts:
- if post_process:
- post_process()
-
- if param.train_url:
- print('Start to copy output directory')
- sync_data(param.output_path, param.train_url)
-
- if param.result_url:
- print('Start to copy output directory')
- sync_data(param.output_path, param.result_url)
-
- return wrapped_func
- return wrapper
diff --git a/official/recommend/tbnet/src/utils/param.py b/official/recommend/tbnet/src/utils/param.py
deleted file mode 100644
index 4a87e96a2..000000000
--- a/official/recommend/tbnet/src/utils/param.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License Version 2.0(the "License");
-# you may not use this file except in compliance with the License.
-# you may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0#
-#
-# Unless required by applicable law or agreed to in writing software
-# distributed under the License is distributed on an "AS IS" BASIS
-# WITHOUT WARRANT IES OR CONITTONS OF ANY KIND锛� either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ====================================================================================
-
-"""Parse arguments"""
-import os
-import ast
-import argparse
-from pprint import pprint, pformat
-import yaml
-
-_config_path = '../../default_config.yaml'
-
-
-class Param:
- """
- Configuration namespace. Convert dictionary to members
- """
-
- def __init__(self, cfg_dict):
- for k, v in cfg_dict.items():
- if isinstance(v, (list, tuple)):
- setattr(self, k, [Param(x) if isinstance(x, dict) else x for x in v])
- else:
- setattr(self, k, Param(v) if isinstance(v, dict) else v)
-
- def __str__(self):
- return pformat(self.__dict__)
-
- def __repr__(self):
- return self.__str__()
-
-
-def parse_cli_to_yaml(parser, cfg, helper=None, choices=None, cfg_path='default_config.yaml'):
- """
- Parse command line arguments to the configuration according to the default yaml
-
- Args:
- parser: Parent parser
- cfg: Base configuration
- helper: Helper description
- cfg_path: Path to the default yaml config
- """
- parser = argparse.ArgumentParser(description='[REPLACE THIS at param.py]',
- parents=[parser])
- helper = {} if helper is None else helper
- choices = {} if choices is None else choices
- for item in cfg:
- if not isinstance(cfg[item], list) and not isinstance(cfg[item], dict):
- help_description = helper[item] if item in helper else 'Please reference to {}'.format(cfg_path)
- choice = choices[item] if item in choices else None
- if isinstance(cfg[item], bool):
- parser.add_argument('--' + item, type=ast.literal_eval, default=cfg[item], choices=choice,
- help=help_description)
- else:
- parser.add_argument('--' + item, type=type(cfg[item]), default=cfg[item], choices=choice,
- help=help_description)
- args = parser.parse_args()
- return args
-
-
-def parse_yaml(yaml_path):
- """
- Parse the yaml config file
-
- Args:
- yaml_path: Path to the yaml config
- """
- with open(yaml_path, 'r') as fin:
- try:
- cfgs = yaml.load_all(fin.read(), Loader=yaml.FullLoader)
- cfgs = [x for x in cfgs]
- if len(cfgs) == 1:
- cfg_helper = {}
- cfg = cfgs[0]
- cfg_choices = {}
- elif len(cfgs) == 2:
- cfg, cfg_helper = cfgs
- cfg_choices = {}
- elif len(cfgs) == 3:
- cfg, cfg_helper, cfg_choices = cfgs
- else:
- raise ValueError('At most 3 docs (config description for help, choices) are supported in config yaml')
- print(cfg_helper)
- except:
- raise ValueError('Failed to parse yaml')
- return cfg, cfg_helper, cfg_choices
-
-
-def merge(args, cfg):
- """
- Merge the base config from yaml file and command line arguments
-
- Args:
- args: command line arguments
- cfg: Base configuration
- """
- args_var = vars(args)
- for item in args_var:
- cfg[item] = args_var[item]
- return cfg
-
-
-def get_param():
- """
- Get Config according to the yaml file and cli arguments
- """
- parser = argparse.ArgumentParser(description='default name', add_help=False)
- current_dir = os.path.dirname(os.path.abspath(__file__))
- parser.add_argument('--config_path', type=str, default=os.path.join(current_dir, _config_path),
- help='Config file path')
- path_args, _ = parser.parse_known_args()
- default, helper, choices = parse_yaml(path_args.config_path)
- args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path)
- final_config = merge(args, default)
- pprint(final_config)
- print("Please check the above information for the configurations", flush=True)
- return Param(final_config)
-
-
-param = get_param()
diff --git a/official/recommend/tbnet/train.py b/official/recommend/tbnet/train.py
deleted file mode 100644
index 5ff1f9744..000000000
--- a/official/recommend/tbnet/train.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""TB-Net training."""
-
-import os
-import math
-
-import numpy as np
-from mindspore import context, Model, Tensor
-from mindspore.train.serialization import save_checkpoint
-from mindspore.train.callback import Callback, TimeMonitor
-import mindspore.common.dtype as mstype
-
-from src import tbnet, config, metrics, dataset
-
-from src.utils.param import param
-from src.utils.moxing_adapter import moxing_wrapper
-from preprocess_dataset import preprocess_data
-
-
-class MyLossMonitor(Callback):
- """My loss monitor definition."""
-
- def on_train_epoch_end(self, run_context):
- """Print loss at each epoch end."""
- cb_params = run_context.original_args()
- loss = cb_params.net_outputs
-
- if isinstance(loss, (tuple, list)):
- if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray):
- loss = loss[0]
-
- if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray):
- loss = np.mean(loss.asnumpy())
- print('loss:' + str(loss))
-
- def on_eval_epoch_end(self, run_context):
- """Print loss at each epoch end."""
- cb_params = run_context.original_args()
- loss = cb_params.net_outputs
-
- if isinstance(loss, (tuple, list)):
- if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray):
- loss = loss[0]
-
- if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray):
- loss = np.mean(loss.asnumpy())
- print('loss:' + str(loss))
-
-
-@moxing_wrapper(preprocess_data)
-def train_tbnet():
- """Training process."""
- config_path = os.path.join(param.data_path, 'data', param.dataset, 'config.json')
- train_csv_path = os.path.join(param.data_path, 'data', param.dataset, param.train_csv)
- test_csv_path = os.path.join(param.data_path, 'data', param.dataset, param.test_csv)
- ckpt_path = param.load_path
-
- context.set_context(device_id=param.device_id)
- if param.run_mode == 'graph':
- context.set_context(mode=context.GRAPH_MODE, device_target=param.device_target)
- else:
- context.set_context(mode=context.PYNATIVE_MODE, device_target=param.device_target)
-
- if not os.path.exists(ckpt_path):
- os.makedirs(ckpt_path)
-
- print(f"creating dataset from {train_csv_path}...")
- net_config = config.TBNetConfig(config_path)
- if param.device_target == 'Ascend':
- net_config.per_item_paths = math.ceil(net_config.per_item_paths / 16) * 16
- net_config.embedding_dim = math.ceil(net_config.embedding_dim / 16) * 16
- train_ds = dataset.create(train_csv_path, net_config.per_item_paths, train=True).batch(net_config.batch_size)
- test_ds = dataset.create(test_csv_path, net_config.per_item_paths, train=True).batch(net_config.batch_size)
-
- print("creating TBNet for training...")
- network = tbnet.TBNet(net_config)
- loss_net = tbnet.NetWithLossClass(network, net_config)
- if param.device_target == 'Ascend':
- loss_net.to_float(mstype.float16)
- train_net = tbnet.TrainStepWrap(loss_net, net_config.lr, loss_scale=True)
- else:
- train_net = tbnet.TrainStepWrap(loss_net, net_config.lr)
-
- train_net.set_train()
- eval_net = tbnet.PredictWithSigmoid(network)
- time_callback = TimeMonitor(data_size=train_ds.get_dataset_size())
- loss_callback = MyLossMonitor()
- model = Model(network=train_net, eval_network=eval_net, metrics={'auc': metrics.AUC(), 'acc': metrics.ACC()})
- print("training...")
- for i in range(param.epochs):
- print(f'===================== Epoch {i} =====================')
- model.train(epoch=1, train_dataset=train_ds, callbacks=[time_callback, loss_callback], dataset_sink_mode=False)
- train_out = model.eval(train_ds, dataset_sink_mode=False)
- test_out = model.eval(test_ds, dataset_sink_mode=False)
- print(f'Train AUC:{train_out["auc"]} ACC:{train_out["acc"]} Test AUC:{test_out["auc"]} ACC:{test_out["acc"]}')
-
- if i >= param.epochs - 5:
- if param.enable_modelarts:
- save_checkpoint(network, os.path.join(param.output_path, f'tbnet_epoch{i}.ckpt'))
- else:
- save_checkpoint(network, os.path.join(ckpt_path, f'tbnet_epoch{i}.ckpt'))
-
-
-if __name__ == '__main__':
- train_tbnet()
--
GitLab