add speed test in ci (#5496)

* add speed test in ci

Signed-off-by: daquexian <daquexian566@gmail.com>

* use actions/checkout@v2

Signed-off-by: daquexian <daquexian566@gmail.com>

* add more input size

Signed-off-by: daquexian <daquexian566@gmail.com>

* fix arg order

Signed-off-by: daquexian <daquexian566@gmail.com>

* update test.yml to use oneflow-inc/models master

Signed-off-by: daquexian <daquexian566@gmail.com>

* only clone models repo in cuda_new_interface

Signed-off-by: daquexian <daquexian566@gmail.com>

* update test_speed.sh to check rel speed

Signed-off-by: daquexian <daquexian566@gmail.com>

* update yml and test_speed.sh to post speed stats

Signed-off-by: daquexian <daquexian566@gmail.com>

* escpae newline for github actions

Signed-off-by: daquexian <daquexian566@gmail.com>

* use correct escaping

Signed-off-by: daquexian <daquexian566@gmail.com>

* replace '\n' to \n in js

Signed-off-by: daquexian <daquexian566@gmail.com>

* collapse speed stats

Signed-off-by: daquexian <daquexian566@gmail.com>

* fix comments

Signed-off-by: daquexian <daquexian566@gmail.com>

* set fixed commit and refine msg

Signed-off-by: daquexian <daquexian566@gmail.com>

* test 30 times instead of 20, relax the threshold

Signed-off-by: daquexian <daquexian566@gmail.com>

* update oneflow-models

Signed-off-by: daquexian <daquexian566@gmail.com>

Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com>

.github/workflows/test.yml

@@ -550,6 +550,33 @@ jobs:
             -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/graph \
             ${{ env.image_tag }} \
             bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
+      - name: Checkout Oneflow-Inc/models
+        if: matrix.test_suite == 'cuda_new_interface'
+        uses: actions/checkout@v2
+        with:
+          repository: Oneflow-Inc/models
+          ref: 830a6b91f10c0a04a68843370cea6319a21ed9c2
+          path: oneflow-models
+      - name: Speed test
+        id: speed
+        if: matrix.test_suite == 'cuda_new_interface'
+        run: |
+          docker run \
+            ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
+            -e ONEFLOW_MODELS_DIR=$PWD/oneflow-models \
+            ${{ env.image_tag }} \
+            bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test_multi_client/test_speed.sh"
+      - name: Post speed stats
+        if: matrix.test_suite == 'cuda_new_interface'
+        uses: actions/github-script@v4
+        with:
+          script: |
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: "<details>\n <summary>Speed stats:</summary>\n\n ``` \n${{ steps.speed.outputs.stats }}\n ``` \n\n</details>".replace(/\\n/g, '\n')
+            })
       - name: Single client op test
         timeout-minutes: 45
         if: matrix.test_suite == 'cpu' || matrix.test_suite == 'cuda_op'

ci/test_multi_client/test_speed.sh

+#!/usr/bin/env bash
+
+set -uxo pipefail
+
+rc=0
+trap 'rc=$?' ERR
+
+cd $ONEFLOW_MODELS_DIR
+
+function check_relative_speed {
+  awk -F'[:(]' -v threshold=$1 'BEGIN { ret=2 } /Relative speed/{ if ($2 > threshold) { ret=0 } else { ret=1 }} {print $0} END { exit ret }'
+}
+
+function write_to_file_and_print {
+  tee -a result
+  printf "\n" >> result
+}
+
+python3 scripts/compare_speed_with_pytorch.py resnet50/models/resnet50.py resnet50 16x3x224x224 --times 30 | check_relative_speed 1.05 | write_to_file_and_print
+python3 scripts/compare_speed_with_pytorch.py resnet50/models/resnet50.py resnet50 8x3x224x224 --times 30 | check_relative_speed 1.05 | write_to_file_and_print
+python3 scripts/compare_speed_with_pytorch.py resnet50/models/resnet50.py resnet50 4x3x224x224 --times 30 | check_relative_speed 1 | write_to_file_and_print
+python3 scripts/compare_speed_with_pytorch.py resnet50/models/resnet50.py resnet50 2x3x224x224 --times 30 | check_relative_speed 0.8 | write_to_file_and_print
+python3 scripts/compare_speed_with_pytorch.py resnet50/models/resnet50.py resnet50 1x3x224x224 --times 30 | check_relative_speed 0.7 | write_to_file_and_print
+
+result="GPU Name: `nvidia-smi --query-gpu=name --format=csv,noheader -i 0` \n\n `cat result`"
+# escape newline for github actions: https://github.community/t/set-output-truncates-multiline-strings/16852/2
+# note that we escape \n and \r to \\n and \\r (i.e. raw string "\n" and "\r") instead of %0A and %0D, 
+# so that they can be correctly handled in javascript code
+result="${result//'%'/'%25'}"
+result="${result//$'\n'/'\\n'}"
+result="${result//$'\r'/'\\r'}"
+
+echo "::set-output name=stats::$result"
+
+exit $rc