diff --git a/oneflow/core/job/oneflow.cpp b/oneflow/core/job/oneflow.cpp
index 464200335b409e64aef53e2dc56a835c936414ef..f108aa53bf350e50bc4f80267169d4687e6b7773 100644
--- a/oneflow/core/job/oneflow.cpp
+++ b/oneflow/core/job/oneflow.cpp
@@ -67,7 +67,9 @@ void FixCpuDeviceNum() {
     Global<CtrlClient>::Get()->PullKVT("cpu_device_num", &cpu_device_num);
   }
   OF_BARRIER();
-  Global<CtrlClient>::Get()->ClearKV("cpu_device_num");
+  if (Global<MachineCtx>::Get()->IsThisMachineMaster()) {
+    Global<CtrlClient>::Get()->ClearKV("cpu_device_num");
+  }
   CHECK_GT(cpu_device_num, 0);
   Global<JobDesc>::Get()->SetCpuDeviceNum(cpu_device_num);
 }