diff --git a/official/cv/yolov3_darknet53/scripts/run_standalone_train.sh b/official/cv/yolov3_darknet53/scripts/run_standalone_train.sh index f4faf6b22025688ab181df72a92fe9ba7ff577fb..a0bbe09ebdbf287d3fdf49fa7bf116609c9ee75f 100644 --- a/official/cv/yolov3_darknet53/scripts/run_standalone_train.sh +++ b/official/cv/yolov3_darknet53/scripts/run_standalone_train.sh @@ -64,16 +64,20 @@ cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -cpus=`cat /proc/cpuinfo| grep "processor"| wc -l` -if [ $cpus -ge $CPU_BIND_NUM ] +cmdopt=`lscpu | grep NUMA | tail -1 | awk '{print $4}'` +if test -z $cmdopt then - start=`expr $cpus - $CPU_BIND_NUM` - end=`expr $cpus - 1` -else - start=0 - end=`expr $cpus - 1` + cpus=`cat /proc/cpuinfo| grep "processor"| wc -l` + if [ $cpus -ge $CPU_BIND_NUM ] + then + start=`expr $cpus - $CPU_BIND_NUM` + end=`expr $cpus - 1` + else + start=0 + end=`expr $cpus - 1` + fi + cmdopt=$start"-"$end fi -cmdopt=$start"-"$end taskset -c $cmdopt python train.py \ --data_dir=$DATASET_PATH \ diff --git a/official/cv/yolov3_darknet53/src/yolo_dataset.py b/official/cv/yolov3_darknet53/src/yolo_dataset.py index 6cb7fdfafc4a012d7a38f95c07a1012c8462b26d..36faf1d5c61240f029aa58c821861d67d18a5de6 100644 --- a/official/cv/yolov3_darknet53/src/yolo_dataset.py +++ b/official/cv/yolov3_darknet53/src/yolo_dataset.py @@ -18,6 +18,7 @@ import os import multiprocessing import cv2 from PIL import Image +import numpy as np from pycocotools.coco import COCO import mindspore.dataset as de import mindspore.dataset.vision.c_transforms as CV @@ -97,9 +98,10 @@ class COCOYoloDataset: coco = self.coco img_id = self.img_ids[index] img_path = coco.loadImgs(img_id)[0]["file_name"] - img = Image.open(os.path.join(self.root, img_path)).convert("RGB") if not self.is_training: + img = Image.open(os.path.join(self.root, img_path)).convert("RGB") return img, img_id + img = np.fromfile(os.path.join(self.root, img_path), dtype="int8") ann_ids = coco.getAnnIds(imgIds=img_id) target = coco.loadAnns(ann_ids) @@ -168,10 +170,12 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num, if device_num != 8: ds = de.GeneratorDataset(yolo_dataset, column_names=dataset_column_names, sampler=distributed_sampler) + ds = ds.map(operations=CV.Decode(), input_columns=["image"]) ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=dataset_column_names, num_parallel_workers=min(32, num_parallel_workers), drop_remainder=True) else: ds = de.GeneratorDataset(yolo_dataset, column_names=dataset_column_names, sampler=distributed_sampler) + ds = ds.map(operations=CV.Decode(), input_columns=["image"]) ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=dataset_column_names, num_parallel_workers=min(8, num_parallel_workers), drop_remainder=True) else: