diff --git a/oneflow/core/job_rewriter/fuse_add_to_output_pass.cpp b/oneflow/core/job_rewriter/fuse_add_to_output_pass.cpp
index 92fa477becfe668f563a56a0db6cdd5255b7f79c..49484d4f1a470fca87de29b3a2de5fe9186c935a 100644
--- a/oneflow/core/job_rewriter/fuse_add_to_output_pass.cpp
+++ b/oneflow/core/job_rewriter/fuse_add_to_output_pass.cpp
@@ -81,6 +81,7 @@ Maybe<void> FuseAddToOutputPass::Apply(const OpGraph& op_graph, JobBuilder* job_
   });
 
   auto IsReachable = op_graph.MakePredicatorIsOpNameDataOrCtrlReachable();
+  std::vector<OperatorConf> delete_ops;
   op_graph.ForEachNode([&](const OpNode* op_node) {
     const OperatorConf& op_conf = op_node->op().op_conf();
     if (!op_conf.has_user_conf()) { return; }
@@ -132,8 +133,9 @@ Maybe<void> FuseAddToOutputPass::Apply(const OpGraph& op_graph, JobBuilder* job_
         }
       }
     }
-    job_builder->DelOps({op_conf});
+    delete_ops.push_back(op_conf);
   });
+  job_builder->DelOps(delete_ops);
   for (const auto& pair : op_name2op_conf) { job_builder->MutOpsOnlyOnce({pair.second}); }
   return Maybe<void>::Ok();
 }
diff --git a/oneflow/core/job_rewriter/fuse_cast_scale_pass.cpp b/oneflow/core/job_rewriter/fuse_cast_scale_pass.cpp
index e27fb19e8d79bdcb99dfc79fbbdc7e2fc12cc9f5..c809fcc8bec1d2302eb16699a72bcf8d7c34050d 100644
--- a/oneflow/core/job_rewriter/fuse_cast_scale_pass.cpp
+++ b/oneflow/core/job_rewriter/fuse_cast_scale_pass.cpp
@@ -61,6 +61,7 @@ class FuseCastScalePass final : public JobPass {
 
 Maybe<void> FuseCastScalePass::Apply(const OpGraph& op_graph, JobBuilder* job_builder) const {
   const auto IsSafeToDelete = MakePredicatorIsSafeToDelete(op_graph);
+  std::vector<OperatorConf> delete_ops;
   op_graph.ForEachNode([&](const OpNode* op_node) {
     if (!IsUserOpWithTypeName(op_node->op().op_conf(), "cast")) { return; }
     if (!IsSafeToDelete(op_node)) { return; }
@@ -86,7 +87,6 @@ Maybe<void> FuseCastScalePass::Apply(const OpGraph& op_graph, JobBuilder* job_bu
     }
     if (op_node->parallel_desc().device_type() != DeviceType::kGPU) { return; }
     double scale = 1.0;
-    std::vector<OperatorConf> delete_ops;
     if (IsUserOpWithTypeName(sole_dst_node->op().op_conf(), "scalar_mul")) {
       const user_op::UserOpConfWrapper scalar_mul_op_conf(sole_dst_node->op().op_conf());
       if (scalar_mul_op_conf.attr<bool>("has_int_operand")) {
@@ -111,9 +111,10 @@ Maybe<void> FuseCastScalePass::Apply(const OpGraph& op_graph, JobBuilder* job_bu
 
     OperatorConf new_op_conf = sole_dst_node->op().op_conf();
     *new_op_conf.mutable_user_conf() = fused_op_builder.Build().op_conf().user_conf();
-    job_builder->DelOps(delete_ops);
+
     job_builder->MutOpsOnlyOnce({new_op_conf});
   });
+  job_builder->DelOps(delete_ops);
   return Maybe<void>::Ok();
 }