Print backtrace when distributed CI failed (#4211)

* Print backtrace when distributed CI failed

* fix path

* typo

Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>

.dockerignore

@@ -31,3 +31,4 @@
 **/core.9*
 /.cache
 /oneflow-src.zip
+/distributed-tmp

.github/workflows/test.yml

@@ -87,6 +87,10 @@ jobs:
           --build_docker_img \
           --oneflow_wheel_path=${wheelhouse_dir} \
           --oneflow_worker_bin=${bin_dir}/oneflow_worker
+    - name: Print Backtrace (distributed test)
+      if: always()
+      run: |
+        docker run --privileged --network host --shm-size=8g --rm -v $PWD:$PWD -w $PWD oneflow-test:$USER bash ci/test/print_stack_from_core.sh python3 distributed-tmp
     - name: Upload log (distributed test)
       if: always()
       uses: ./.github/actions/upload_oss

.gitignore

@@ -28,3 +28,4 @@ compile_commands.json
 .cache
 /oneflow-src.zip
 /oneflow_temp
+/distributed-tmp

ci/test/distributed_run.py

@@ -161,6 +161,7 @@ export ONEFLOW_TEST_SSH_PORT={ssh_port}
 export ONEFLOW_TEST_LOG_DIR={log_dir}
 export ONEFLOW_TEST_NODE_LIST="{this_host},{remote_host}"
 export ONEFLOW_WORKER_KEEP_LOG=1
+export ONEFLOW_TEST_TMP_DIR="./distributed-tmp"
 export NCCL_DEBUG=INFO
 """
     if oneflow_worker_bin:

ci/test/print_stack_from_core.sh

+set -ex
+if compgen -G "$2/core.*" > /dev/null; then
+    gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $1 $2/core.*
+fi