name: Build and Test CI
on:
  pull_request:
    types: [review_requested]
    branches:
      - "*"
  workflow_dispatch:
    inputs:
      placeholder:
        description: "placeholder, no effect"
        required: false
jobs:
  cancel_previous:
    name: Cancel previous runs
    runs-on: ubuntu-latest
    steps:
      - name: Cancel previous runs of outdated commit
        uses: styfle/cancel-workflow-action@0.9.0
        with:
          access_token: ${{ github.token }}
          all_but_latest: true

  mirror_third_party:
    name: Mirror third party dependencies
    runs-on: ubuntu-18.04
    if: github.event.pull_request.draft == false && github.base_ref == 'master' && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
    steps:
      - uses: actions/checkout@v2
      - name: Mirror dependencies to aliyun
        env:
          OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
          OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
        run: |
          set -x
          if [ -z "$OSS_ACCESS_KEY_ID" ]
          then
            exit 0
          fi
          python3 -m pip install -U pip setuptools wheel
          python3 -m pip install oss2
          python3 tools/package_mirror.py -i cmake

  check_license_and_format:
    name: License and format
    runs-on: ubuntu-18.04
    if: github.event.pull_request.draft == false && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
    steps:
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{github.event.pull_request.head.repo.full_name}}
      - name: Check license
        id: license_check
        run: |
          python3 ci/check/run_license_format.py -i oneflow -c
      - name: Add license
        id: license_fmt
        if: ${{ failure() }}
        run: |
          python3 ci/check/run_license_format.py -i oneflow --fix
      - name: Check C++/CUDA format
        id: cpp_check
        run: |
          python3 ci/check/run_clang_format.py --clang_format_binary clang-format --source_dir oneflow
      - name: Run C++/CUDA format
        id: cpp_fmt
        if: ${{ failure() }}
        run: |
          python3 ci/check/run_clang_format.py --clang_format_binary clang-format --source_dir oneflow --fix
      - name: Check Python format
        id: py_check
        run: |
          python3 -m pip install black==19.10b0
          python3 ci/check/run_py_format.py --source_dir $PWD
      - name: Run Python Format
        id: py_fmt
        if: ${{ failure() }}
        run: |
          python3 -m pip install black==19.10b0
          python3 ci/check/run_py_format.py --source_dir $PWD --fix
      - name: Git push
        id: git_push
        if: ${{ failure() }}
        run: |
          git config --global user.email "ci-bot@oneflow.org"
          git config --global user.name "oneflow-ci-bot"
          git add -u
          git commit -m "auto format by CI"
          git push
      - name: Please request CI again
        if: ${{ failure() }}
        run: |
          exit 1
      - name: Check source code (prevent creating files at wrong places)
        run: |
          python3 tools/check_src.py

  static_analysis_with_clang:
    name: Static analysis with clang
    runs-on: ubuntu-20.04
    if: github.event.pull_request.draft == false && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
    steps:
      - name: Check out OneFlow
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{github.event.pull_request.head.repo.full_name}}
      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libopenblas-dev nasm python3-pip ninja-build
      - name: Download OneFlow custom clang-tidy
        run: |
          wget https://github.com/oneflow-inc/llvm-project/releases/download/latest/clang-tidy
          wget https://raw.githubusercontent.com/oneflow-inc/llvm-project/maybe/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
          chmod +x clang-tidy run-clang-tidy.py
      - name: Build third party libs and generate files
        run: |
          mkdir build
          cd build
          # clang-tidy has builtin includes, but doesn't work here
          # https://clang.llvm.org/docs/LibTooling.html#builtin-includes
          cmake .. -C ../cmake/caches/international/cpu.cmake \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTING=ON
          cmake --build . -j$(nproc) --target of_git_version oneflow_deps generate_functional of_cfgobj generate_py_cfg
      - name: Run Maybe-related checks by clang-tidy
        run: |
          cd build
          rm CMakeCache.txt
          cmake .. -C ../cmake/caches/international/cpu.cmake \
            -DCMAKE_C_COMPILER=clang-12 \
            -DCMAKE_CXX_COMPILER=clang++-12 \
            -DCMAKE_CXX_FLAGS="-isystem /usr/lib/llvm-12/lib/clang/12.0.1/include/" \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTING=ON \
            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
          cd ..
          ./run-clang-tidy.py -clang-tidy-binary ./clang-tidy -p build -quiet

  wait_for_gpu_slot:
    name: Wait for GPU slots
    runs-on: [self-hosted, scheduler]
    needs: [build]
    if: github.event.pull_request.draft == false && github.base_ref == 'master' && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
    continue-on-error: true
    steps:
      - name: Check if secrets accessible
        env:
          CI_PERSONAL_ACCESS_TOKEN: ${{ secrets.CI_PERSONAL_ACCESS_TOKEN }}
        run: |
          set -x
          if [ -z "$CI_PERSONAL_ACCESS_TOKEN" ]
          then
            exit 0
          fi
          echo "is_secrets_accessible=1" >> $GITHUB_ENV
      - uses: Oneflow-Inc/gh-action-scheduler-v2@1e45bd715c873bc3a14a87ddaf494f9b686137de
        name: Wait for GPU slot
        if: env.is_secrets_accessible == '1'
        timeout-minutes: 45
        env:
          CI_PERSONAL_ACCESS_TOKEN: ${{ secrets.CI_PERSONAL_ACCESS_TOKEN }}

  build:
    name: Build
    needs: [check_license_and_format, mirror_third_party, cancel_previous]
    continue-on-error: ${{ matrix.allow_fail }}
    runs-on: ${{ matrix.os }}
    if: github.event.pull_request.draft == false && github.base_ref == 'master' && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot') && contains(github.event.pull_request.labels.*.name, 'fmt-only') == false
    strategy:
      max-parallel: 5
      matrix:
        test_suite: ["cuda", "cpu", "xla", "xla_cpu"]
        include:
          - test_suite: cuda
            cuda_version: 10.2
            extra_flags: --extra_oneflow_cmake_args=-DCUDA_NVCC_GENCODES=arch=compute_61,code=sm_61 --extra_oneflow_cmake_args=-DRPC_BACKEND=GRPC,LOCAL --extra_oneflow_cmake_args=-DPIP_INDEX_MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple
            os: [self-hosted, linux, build]
            allow_fail: false
            python_version: 3.6,3.7
          - test_suite: cpu
            cuda_version: 10.2
            extra_flags: --extra_oneflow_cmake_args=-DBUILD_SHARED_LIBS=OFF --extra_oneflow_cmake_args=-DRPC_BACKEND=LOCAL --cpu --gcc7
            os: [self-hosted, linux, build]
            allow_fail: false
            python_version: 3.6
          - test_suite: xla
            cuda_version: 10.1
            extra_flags: --extra_oneflow_cmake_args=-DCUDA_NVCC_GENCODES=arch=compute_61,code=sm_61 --extra_oneflow_cmake_args=-DRPC_BACKEND=GRPC,LOCAL --xla --extra_oneflow_cmake_args=-DPIP_INDEX_MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple
            os: [self-hosted, linux, build]
            allow_fail: true
            python_version: 3.6
          - test_suite: xla_cpu
            cuda_version: 10.1
            extra_flags: --extra_oneflow_cmake_args=-DRPC_BACKEND=GRPC,LOCAL --xla --cpu --extra_oneflow_cmake_args=-DPIP_INDEX_MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple
            os: [self-hosted, linux, build]
            allow_fail: true
            python_version: 3.6
    steps:
      - name: Set environments from secrets
        if: contains(${{ github.repository }}, "Oneflow-Inc")
        run: |
          echo "CI_STATIC_HOST=${{ secrets.CI_STATIC_HOST }}" >> $GITHUB_ENV
          echo "CI_STATIC_PATH=${{ secrets.CI_STATIC_PATH }}" >> $GITHUB_ENV
      - name: Set environments for forks
        run: |
          CI_STATIC_HOST=${CI_STATIC_HOST:-"192.168.1.13"}
          CI_STATIC_PATH=${CI_STATIC_PATH:-"/tank/ci-static"}
          echo "CI_STATIC_HOST=${CI_STATIC_HOST}" >> $GITHUB_ENV
          echo "CI_STATIC_PATH=${CI_STATIC_PATH}" >> $GITHUB_ENV
      - name: Set environment variables
        env:
          test_suite: ${{ matrix.test_suite }}
        run: |
          set -x
          echo "test_suite=${test_suite}" >> $GITHUB_ENV
          export ci_tmp_dir=$HOME/ci-cache/test_suite/${test_suite}
          mkdir -p $ci_tmp_dir
          echo "ci_tmp_dir=${ci_tmp_dir}" >> $GITHUB_ENV
          echo "wheelhouse_dir=${ci_tmp_dir}/wheelhouse" >> $GITHUB_ENV
          sha_dir="${CI_STATIC_PATH}/${{ github.repository }}/pr/${{ github.event.pull_request.number }}/${{ github.sha }}/${{ matrix.test_suite }}"
          echo "sha_dir=${sha_dir}" >> $GITHUB_ENV
          if ssh -o StrictHostKeyChecking=no ${CI_STATIC_HOST} "test -d $sha_dir"; then
              echo "is_built=1" >> $GITHUB_ENV
          fi
          if [ "$test_suite" == "cuda" ]; then
            echo "bin_dir=${ci_tmp_dir}/manylinux2014-build-cache/10.2/build-oneflow/bin" >> $GITHUB_ENV
          elif [ "$test_suite" == "cpu" ]; then
            echo "bin_dir=${ci_tmp_dir}/manylinux2014-build-cache/10.2-gcc7-cpu/build-oneflow/bin" >> $GITHUB_ENV
          fi
          extra_docker_args=""
          extra_docker_args+=" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"

          container_name=pr-${{ github.event.pull_request.number }}-run-id-${{ github.run_id }}-${test_suite}-build
          echo "container_name=${container_name}" >> $GITHUB_ENV
          extra_docker_args+=" --name ${container_name}"
          if [ "$test_suite" == "cpu" ]; then
            extra_docker_args+=" --env ONEFLOW_TEST_CPU_ONLY=1"
            extra_docker_args+=" --env CUDA_VISIBLE_DEVICES=''"
          fi
          echo "extra_docker_args=${extra_docker_args}" >> $GITHUB_ENV

          img_cache_base_dir="$HOME/ci-cache/${{ github.repository }}/oneflow-manylinux2014-cuda10.2/0.1"
          pip_cache_docker_args="-v ${img_cache_base_dir}/dotcache/pip:/root/.cache/pip"
          echo "pip_cache_docker_args=${pip_cache_docker_args}" >> $GITHUB_ENV

          echo "pip_index_mirror=https://pypi.tuna.tsinghua.edu.cn/simple" >> $GITHUB_ENV
      - name: Fix permissions
        run: |
          docker run --rm -v $PWD:/p -w /p busybox chmod -R o+w .
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.sha }}
        if: env.is_built != '1'
      - name: Build OneFlow
        timeout-minutes: 45
        if: env.is_built != '1'
        uses: ./.github/actions/whl
        with:
          tmp_dir: ${ci_tmp_dir}
          extra_flags: ${{ matrix.extra_flags }}
          cuda_version: ${{ matrix.cuda_version }}
          extra_docker_args: $extra_docker_args
          python_version: ${{ matrix.python_version }}
      - name: Single client custom Op test (run by oneflow build docker)
        timeout-minutes: 45
        if: matrix.test_suite == 'cpu' && env.is_built != '1'
        run: |
          set -x
          docker run --shm-size=8g --rm -w $PWD -v $PWD:$PWD -v /dataset:/dataset -v /model_zoo:/model_zoo \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -v ${wheelhouse_dir}:${wheelhouse_dir} --env ONEFLOW_WHEEL_PATH=${wheelhouse_dir} \
            oneflow-manylinux2014-cuda10.2:0.1 \
            bash -c "export PATH=/opt/python/cp36-cp36m/bin:$PATH && source scl_source enable devtoolset-7 && python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_custom_op_test.sh"
      - name: Remove cache
        if: ${{ failure() }}
        run: |
          set -x
          cache_parent_dir=$(dirname ${ci_tmp_dir})
          if [ "$cache_parent_dir" == "/" ] || [ -z "$cache_parent_dir" ]; then
            exit 1
          fi
          docker run --rm -v $cache_parent_dir:$cache_parent_dir busybox rm -rf ${ci_tmp_dir}
      - name: Upload wheel via SSH
        if: env.is_built != '1'
        uses: ./.github/actions/upload_ssh
        with:
          src_path: ${wheelhouse_dir}
          dst_host: ${{ env.CI_STATIC_HOST }}
          dst_path: ${{ env.CI_STATIC_PATH }}/${{ github.repository }}/pr/${{ github.event.pull_request.number }}/${{ github.sha }}/${{ matrix.test_suite }}/whl
      - name: Upload bin via SSH
        if: contains(fromJson('["cuda", "cpu"]'), matrix.test_suite) && env.is_built != '1'
        uses: ./.github/actions/upload_ssh
        with:
          src_path: ${bin_dir}
          dst_host: ${{ env.CI_STATIC_HOST }}
          dst_path: ${{ env.CI_STATIC_PATH }}/${{ github.repository }}/pr/${{ github.event.pull_request.number }}/${{ github.sha }}/${{ matrix.test_suite }}/bin
      - name: Upload wheel to cloud, enable by adding label 'upload-wheel'
        if: contains(github.event.pull_request.labels.*.name, 'upload-wheel')
        uses: ./.github/actions/upload_oss
        with:
          src_path: ${wheelhouse_dir}
          oss_dst_path: oss://oneflow-staging/${{ github.repository }}/pr/${{ github.event.pull_request.number }}/${{ github.sha }}/${{ matrix.test_suite }}/whl
          oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
          oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
      - name: Remove container
        if: always()
        run: |
          set -x
          docker rm -f ${container_name} || true
      - name: Save shared env
        run: |
          set -x
          echo "builder_host=${CI_STATIC_HOST}" >> build.${{ matrix.test_suite }}.env
          echo "sha_dir=${sha_dir}" >> build.${{ matrix.test_suite }}.env
          echo "remote_whl_dir=${CI_STATIC_HOST}:$sha_dir/whl" >> build.${{ matrix.test_suite }}.env
          if [ "$test_suite" == "cuda" ] || [  "$test_suite" == "cpu" ]; then
            echo "remote_bin_dir=${CI_STATIC_HOST}:$sha_dir/bin" >> build.${{ matrix.test_suite }}.env
          fi
      - name: Upload shared env
        uses: actions/upload-artifact@v2
        with:
          name: build.${{ matrix.test_suite }}.env
          path: build.${{ matrix.test_suite }}.env

  test:
    name: Test suite
    needs: [build, wait_for_gpu_slot]
    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.allow_fail }}
    if: github.event.pull_request.draft == false && github.base_ref == 'master' && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
    strategy:
      max-parallel: 5
      matrix:
        test_suite:
          ["cuda", "cuda_op", "cuda_new_interface", "cpu", "xla", "xla_cpu"]
        include:
          - test_suite: "cuda"
            os: [self-hosted, linux, gpu]
            allow_fail: false
            build_env: build.cuda.env
          - test_suite: "cuda_op"
            os: [self-hosted, linux, gpu]
            allow_fail: false
            build_env: build.cuda.env
          - test_suite: "cuda_new_interface"
            os: [self-hosted, linux, gpu]
            allow_fail: false
            build_env: build.cuda.env
          - test_suite: "cpu"
            os: [self-hosted, linux, cpu]
            allow_fail: false
            build_env: build.cpu.env
          - test_suite: "xla"
            os: [self-hosted, linux, gpu]
            allow_fail: true
            build_env: build.xla.env
          - test_suite: "xla_cpu"
            os: [self-hosted, linux, cpu]
            allow_fail: true
            build_env: build.xla_cpu.env
    steps:
      - name: Fix permissions
        run: |
          docker run --rm -v $PWD:/p -w /p busybox chmod -R o+w .
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Download shared env
        uses: actions/download-artifact@v2
        with:
          name: ${{ matrix.build_env }}
      - name: Set environment variables
        env:
          test_suite: ${{matrix.test_suite }}
        run: |
          set -x
          extra_docker_args=""
          extra_docker_args+=" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined"
          extra_docker_args+=" --env ONEFLOW_CI=1"

          # load shared environment variables
          cat ${{ matrix.build_env }} >> $GITHUB_ENV
          source ${{ matrix.build_env }}
          # check directory sha_dir exists
          ssh -o StrictHostKeyChecking=no ${builder_host} "[ -d ${sha_dir} ]"

          # dowload whl and bin
          scp -r $remote_whl_dir wheel_tmp
          if [ "$test_suite" == "cuda" ] || [  "$test_suite" == "cpu" ]; then
            scp -r $remote_bin_dir bin_tmp
          fi

          # set wheelpath
          wheelhouse_dir="${PWD}/wheel_tmp"
          extra_docker_args+=" --env ONEFLOW_WHEEL_PATH=${wheelhouse_dir}"
          echo "wheelhouse_dir=${wheelhouse_dir}" >> $GITHUB_ENV

          # set matrix environment variables
          echo "test_suite=${test_suite}" >> $GITHUB_ENV
          if [ "$test_suite" == "cuda" ] || [  "$test_suite" == "cpu" ]; then
            echo "bin_dir=${PWD}/bin_tmp" >> $GITHUB_ENV
          fi
          if [ "$test_suite" == "cpu" ] || [ "$test_suite" == "xla_cpu" ]; then
            extra_docker_args+=" --env ONEFLOW_TEST_CPU_ONLY=1"
            extra_docker_args+=" --env CUDA_VISIBLE_DEVICES=-1"
          fi
          # set container_name
          container_name=pr-${{ github.event.pull_request.number }}-run-id-${{ github.run_id }}-${test_suite}-test
          extra_docker_args+=" --name ${container_name}"
          echo "container_name=${container_name}" >> $GITHUB_ENV

          extra_docker_args+=" --shm-size=8g --rm -w $PWD -v $PWD:$PWD"
          extra_docker_args+=" -v /dataset:/dataset -v /model_zoo:/model_zoo"
          echo "extra_docker_args=${extra_docker_args}" >> $GITHUB_ENV
          echo "image_name=oneflow-test:0.2" >> $GITHUB_ENV
          echo "pip_index_mirror=https://pypi.tuna.tsinghua.edu.cn/simple" >> $GITHUB_ENV

          img_cache_base_dir="$HOME/ci-cache/${{ github.repository }}/oneflow-test/0.1"
          pip_cache_docker_args="-v ${img_cache_base_dir}/dotcache/pip:/root/.cache/pip"
          echo "pip_cache_docker_args=${pip_cache_docker_args}" >> $GITHUB_ENV
      - name: Build docker image for testing
        run: |
          bash docker/ci/test/build.sh
      - name: Exe test
        timeout-minutes: 45
        if: contains(fromJson('["cuda", "cpu"]'), matrix.test_suite)
        run: |
          set -x
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            ${bin_dir}/oneflow_testexe
      - name: Build documentation
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/build_docs.sh"
      - name: Single client op test (distributed, 1st try)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        continue-on-error: true
        id: distributed_try_1
        run: |
          python3 ci/test/distributed_run.py --bash_script=ci/test/2node_op_test.sh --custom_img_tag=${{ env.image_name }} --oneflow_wheel_path=${{ env.wheelhouse_dir }} --oneflow_wheel_python_version=3.6
      - name: Single client op test (distributed, 2nd try)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda' && steps.distributed_try_1.outcome=='failure'
        continue-on-error: true
        id: distributed_try_2
        run: |
          python3 ci/test/distributed_run.py --bash_script=ci/test/2node_op_test.sh --custom_img_tag=${{ env.image_name }} --oneflow_wheel_path=${{ env.wheelhouse_dir }} --oneflow_wheel_python_version=3.6
      - name: Single client op test (distributed, 3rd try)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda' && steps.distributed_try_2.outcome=='failure'
        continue-on-error: false
        id: distributed_try_3
        run: |
          python3 ci/test/distributed_run.py --bash_script=ci/test/2node_op_test.sh --custom_img_tag=${{ env.image_name }} --oneflow_wheel_path=${{ env.wheelhouse_dir }} --oneflow_wheel_python_version=3.6
      - name: Upload log (distributed test)
        if: always() && steps.distributed_try_3.outcome=='failure' && matrix.test_suite == 'cuda'
        uses: ./.github/actions/upload_oss
        with:
          src_path: distributed-tmp
          oss_dst_path: oss://oneflow-log/${{ github.repository }}/pr/${{ github.event.pull_request.number }}/${{github.run_id}}/distributed-tmp
          oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
          oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
          upload_core: ${{ contains(github.event.pull_request.labels.*.name, 'upload-core') }}
      - name: Print backtrace (distributed test)
        if: always() && steps.distributed_try_3.outcome=='failure' && matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} bash ci/test/print_stack_from_core.sh python3 distributed-tmp
      - name: Doctest
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/doctest.sh"
      - name: Single client dry run test (run without runtime)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/dry_run_test.sh"
      - name: Set environment variables (cuda_new_interface)
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          set -x
          echo "image_tag=oneflow-test-v2:0.1" >> $GITHUB_ENV
          echo "image_url=https://oneflow-static.oss-cn-beijing.aliyuncs.com/docker_images/oneflow-test-v2.0.1.tar.gz" >> $GITHUB_ENV

          img_cache_base_dir="$HOME/ci-cache/${{ github.repository }}/pytorch/pytorch/1.9.0-cuda10.2-cudnn7-runtime"
          pip_cache_docker_args="-v ${img_cache_base_dir}/dotcache/pip:/root/.cache/pip"
          echo "pip_cache_docker_args=${pip_cache_docker_args}" >> $GITHUB_ENV
      - name: Check image (cuda_new_interface)
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          if [[ "$(docker images -q ${{ env.image_tag }} 2> /dev/null)" == "" ]]; then
            echo "should_load_img=1" >> $GITHUB_ENV
          fi
      - name: Load image (cuda_new_interface)
        if: matrix.test_suite == 'cuda_new_interface' && env.should_load_img == '1'
        run: |
          wget ${{ env.image_url }}
          docker load -i $(basename "${{ env.image_url }}")
      - name: Module API test
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/modules \
            ${{ env.image_tag }} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test_multi_client/generic_test.sh"
      - name: Dataloader API test
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/dataloader \
            ${{ env.image_tag }} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
      - name: Tensor API test
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/tensor \
            ${{ env.image_tag }} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
      - name: Graph API test
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/graph \
            ${{ env.image_tag }} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
      - name: Checkout Oneflow-Inc/models
        if: matrix.test_suite == 'cuda_new_interface'
        uses: actions/checkout@v2
        with:
          repository: Oneflow-Inc/models
          ref: 830a6b91f10c0a04a68843370cea6319a21ed9c2
          path: oneflow-models
      - name: Speed test
        id: speed
        if: matrix.test_suite == 'cuda_new_interface'
        run: |
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            -e ONEFLOW_MODELS_DIR=$PWD/oneflow-models \
            ${{ env.image_tag }} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test_multi_client/test_speed.sh"
      - name: Post speed stats
        if: matrix.test_suite == 'cuda_new_interface'
        uses: actions/github-script@v4
        with:
          script: |
            github.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: "<details>\n <summary>Speed stats:</summary>\n\n ``` \n${{ steps.speed.outputs.stats }}\n ``` \n\n</details>".replace(/\\n/g, '\n')
            })
      - name: Single client op test
        timeout-minutes: 45
        if: matrix.test_suite == 'cpu' || matrix.test_suite == 'cuda_op'
        run: |
          set -x
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_op_test.sh"
      - name: Single client model test
        timeout-minutes: 45
        if: matrix.test_suite == 'cpu' || matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run \
            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_model_test.sh"
      - name: Single client model serve test
        timeout-minutes: 45
        id: model_serve_test
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            --env ONEFLOW_TEST_TMP_DIR=$PWD/serving-tmp \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_model_serve_test.sh"
      - name: Print backtrace (serving test)
        if: always() && steps.model_serve_test.outcome=='failure'  && matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} bash ci/test/print_stack_from_core.sh python3 serving-tmp
      - name: Single client benchmark (mainly for backward compatibility)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_benchmark_test.sh"
      - name: Single client benchmark FP16 (mainly for backward compatibility)
        timeout-minutes: 45
        if: matrix.test_suite == 'cuda'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/1node_benchmark_test_fp16.sh"
      - name: Single client XLA Test
        timeout-minutes: 45
        if: contains(fromJson('["xla", "xla_cpu"]'), matrix.test_suite) && env.is_built != '1'
        run: |
          set -x
          docker run ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
            ${image_name} \
            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/test_xla.sh"
      - name: Remove automerge
        if: matrix.test_suite == 'cuda_new_interface' && failure() && cancelled() == false && contains(github.event.pull_request.labels.*.name, 'automerge')
        uses: actions/github-script@v4
        with:
          script: |
            github.issues.removeLabel({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'automerge'
            })
            github.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'CI failed, removing label automerge'
            })
      - name: Query system status
        timeout-minutes: 45
        if: failure()
        run: |
          nvidia-smi
          docker ps
      - name: Remove container
        timeout-minutes: 45
        if: always()
        run: |
          docker rm -f ${container_name} || true
