diff --git a/official/nlp/bert/src/dataset.py b/official/nlp/bert/src/dataset.py
index 01f9659e6b97ae16f9c00cf65fa5a92d68e04d6b..0c3e3e0a3ab9f864ade0fb19b91948fe9da05460 100644
--- a/official/nlp/bert/src/dataset.py
+++ b/official/nlp/bert/src/dataset.py
@@ -66,8 +66,6 @@ class BucketDatasetGenerator:
                     self.iter += 1
                     return self._package_data(data, key)
         self.stage = 1
-        for value in self.data_bucket.values():
-            self.remaining_data += list(value)
         return self._process_remaining_data()
 
     def _package_data(self, data, key):