diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 22952644bdac509f0199feda247eabb213d9d9ec..a931d1a95f3828701f949a5fe37c209edfe6edb5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -53,10 +53,10 @@ jobs: env: TOOLSET_DIR: /opt/vesoft/toolset/clang/9.0.0 CCACHE_DIR: /tmp/ccache/nebula-graph/${{ matrix.os }}-${{ matrix.compiler }} - CCACHE_MAXSIZE: 1G + CCACHE_MAXSIZE: 8G volumes: - /tmp/ccache/nebula-graph/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula-graph/${{ matrix.os }}-${{ matrix.compiler }} - options: --mount type=tmpfs,destination=/tmp/ccache/nebula-graph,tmpfs-size=1073741824 --cap-add=SYS_PTRACE + options: --cap-add=SYS_PTRACE steps: - name: Cleanup if: ${{ always() }} @@ -78,6 +78,7 @@ jobs: repository: ${{ github.repository_owner }}/nebula-storage path: modules/storage - name: CMake + id: cmake run: | case ${{ matrix.compiler }} in gcc-*) @@ -91,6 +92,7 @@ jobs: -DENABLE_TESTING=on \ -DENABLE_BUILD_STORAGE=on \ -B build + echo "::set-output name=j::8" ;; ubuntu1804) # build with Debug type @@ -101,6 +103,7 @@ jobs: -DENABLE_TESTING=on \ -DENABLE_BUILD_STORAGE=on \ -B build + echo "::set-output name=j::8" ;; esac ;; @@ -109,15 +112,19 @@ jobs: cmake \ -DCMAKE_CXX_COMPILER=$TOOLSET_DIR/bin/clang++ \ -DCMAKE_C_COMPILER=$TOOLSET_DIR/bin/clang \ - -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DENABLE_ASAN=on \ -DENABLE_TESTING=on \ -DENABLE_BUILD_STORAGE=on \ -B build + echo "::set-output name=j::4" ;; esac - name: Make graph - run: cmake --build build/ -j $(nproc) + run: | + ccache -z + cmake --build build/ -j $(nproc) + ccache -s - name: CTest env: ASAN_OPTIONS: fast_unwind_on_malloc=1 @@ -127,8 +134,19 @@ jobs: - name: Pytest env: NEBULA_TEST_LOGS_DIR: ${{ github.workspace }}/build - run: make RM_DIR=false -C tests test - timeout-minutes: 25 + run: make RM_DIR=false J=${{ steps.cmake.outputs.j }} test + working-directory: tests/ + timeout-minutes: 15 + - name: TCK + env: + NEBULA_TEST_LOGS_DIR: ${{ github.workspace }}/build + run: make RM_DIR=false J=${{ steps.cmake.outputs.j }} tck + working-directory: tests/ + timeout-minutes: 15 + - name: Sanitizer + if: ${{ always() }} + run: | + exit $(grep -P "SUMMARY: AddressSanitizer: \d+ byte\(s\) leaked in \d+ allocation\(s\)\." build/server_*/logs/*stderr.log | wc -l) - name: Upload logs uses: actions/upload-artifact@v2 if: ${{ failure() }} diff --git a/.gitignore b/.gitignore index 4f1ac2f58283c07b936995d13533d7019e421749..01f7ec21775f2946e6e4d1b3bab3f68f0f7a8cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,9 +38,6 @@ modules/ !tests/Makefile reformat-gherkin nebula-python -.pytest -.pytest_cache -*.lock # IDE .idea/ diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..da18baf452a225e4476e5cdf952ff9c95bc8664a --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,6 @@ +.pytest +.pytest_cache +*.lock +csv-data-* +nebula-test +tck-report.json diff --git a/tests/Makefile b/tests/Makefile index 909eaac7858b3f2a5a28a4a6dad12f4076e765bd..c559ec47379fcd19f93549fcb316d883667bf7f0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -11,6 +11,7 @@ gherkin_fmt = ~/.local/bin/reformat-gherkin RM_DIR ?= true TEST_DIR ?= $(CURR_DIR) +J ?= 8 install-deps: pip3 install --user -U setuptools wheel -i $(PYPI_MIRROR) @@ -39,10 +40,18 @@ check: @find $(CURR_DIR)/tck/features -type f -iname "*.feature" -print | xargs $(gherkin_fmt) --check test: - cd $(CURR_DIR) && python3 -m pytest -n8 --dist=loadfile --rm_dir=$(RM_DIR) -m "not skip" $(TEST_DIR) + cd $(CURR_DIR) && python3 -m pytest -n$(J) --dist=loadfile --rm_dir=$(RM_DIR) -m "not skip" -k "not tck" $(TEST_DIR) tck: - cd $(CURR_DIR) && python3 -m pytest --gherkin-terminal-reporter --gherkin-terminal-reporter-expanded --rm_dir=$(RM_DIR) -m "not skip" $(CURR_DIR)/tck/ + cd $(CURR_DIR) && python3 -m pytest --cucumber-json=$(CURR_DIR)/tck-report.json --cucumber-json-expanded -n$(J) --rm_dir=$(RM_DIR) -m "not skip" $(CURR_DIR)/tck/ + +report: + @mv $(CURR_DIR)/tck-report.json $(CURR_DIR)/tck-report-bak.json + @jq . $(CURR_DIR)/tck-report-bak.json > tck-report.json + @rm -rf $(CURR_DIR)/tck-report-bak.json clean: @rm -rf $(CURR_DIR)/nebula-python $(CURR_DIR)/reformat-gherkin $(CURR_DIR)/.pytest/* $(CURR_DIR)/.pytest_cache + +kill: + ps -ef | grep -P '\sbin/nebula-' | grep "$$(whoami)" | sed 's/\s\s*/ /g' | cut -f2 -d' ' | xargs kill -9 diff --git a/tests/common/dataset_printer.py b/tests/common/dataset_printer.py index 2c59e3fab33eacf4f6ea3acc32b9bea85bd723be..14b336044cd8fe51540deb724c380f198d8de6f3 100644 --- a/tests/common/dataset_printer.py +++ b/tests/common/dataset_printer.py @@ -5,12 +5,13 @@ from typing import List -from nebula2.common.ttypes import DataSet, Edge, NullType, Path, Value, Vertex, Value +from nebula2.common.ttypes import DataSet, Edge, NullType, Path, Value, Vertex class DataSetPrinter: - def __init__(self, decode_type='utf-8'): + def __init__(self, decode_type='utf-8', vid_fn=None): self._decode_type = decode_type + self._vid_fn = vid_fn def sstr(self, b) -> str: if not type(b) == bytes: @@ -18,11 +19,13 @@ class DataSetPrinter: return b.decode(self._decode_type) def vid(self, v) -> str: - if type(v) == str: - return f'"{v}"' - if type(v) == bytes: - return f'"{self.sstr(v)}"' - if type(v) == int: + if type(v) is str: + return f'"{v}"' if self._vid_fn is None else f"{self._vid_fn(v)}" + if type(v) is bytes: + if self._vid_fn is None: + return f'"{self.sstr(v)}"' + return f"{self._vid_fn(v)}" + if type(v) is int: return f'{v}' if isinstance(v, Value): return self.vid(self.to_string(v)) diff --git a/tests/common/types.py b/tests/common/types.py index 3686a12a481c6e77ba54410331f71b0a0f27745f..404666ce70f6d3cc3b278a56c7f62903709c87da 100644 --- a/tests/common/types.py +++ b/tests/common/types.py @@ -19,13 +19,16 @@ class SpaceDesc: self.charset = charset self.collate = collate + def __str__(self): + return str(self.__dict__) + @staticmethod def from_json(obj: dict): return SpaceDesc( name=obj.get('name', None), - vid_type=obj.get('vidType', 'FIXED_STRING(32)'), - partition_num=obj.get('partitionNum', 7), - replica_factor=obj.get('replicaFactor', 1), + vid_type=obj.get('vid_type', 'FIXED_STRING(32)'), + partition_num=obj.get('partition_num', 7), + replica_factor=obj.get('replica_factor', 1), charset=obj.get('charset', 'utf8'), collate=obj.get('collate', 'utf8_bin'), ) diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index a4f565ef0df418a5bc6bd02efb62c3557bff296a..b9aa49dbd7cf3d34ba9a6b601d190f601281a26f 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -124,11 +124,22 @@ def wait(secs): time.sleep(secs) -def cmp_dataset(graph_spaces, - result, - order: bool, - strict: bool, - included=False) -> None: +def line_number(steps, result): + for step in steps: + res_lines = result.split('\n') + if all(l in r for (l, r) in zip(res_lines, step.lines)): + return step.line_number + return -1 + + +def cmp_dataset( + request, + graph_spaces, + result, + order: bool, + strict: bool, + included=False, +) -> None: rs = graph_spaces['result_set'] ngql = graph_spaces['ngql'] check_resp(rs, ngql) @@ -144,7 +155,7 @@ def cmp_dataset(graph_spaces, vid_fn=vid_fn) def dsp(ds): - printer = DataSetPrinter(rs._decode_type) + printer = DataSetPrinter(rs._decode_type, vid_fn=vid_fn) return printer.ds_to_string(ds) def rowp(ds, i): @@ -152,7 +163,7 @@ def cmp_dataset(graph_spaces, return "" assert i < len(ds.rows), f"{i} out of range {len(ds.rows)}" row = ds.rows[i].values - printer = DataSetPrinter(rs._decode_type) + printer = DataSetPrinter(rs._decode_type, vid_fn=vid_fn) ss = printer.list_to_string(row, delimiter='|') return f'{i}: |' + ss + '|' @@ -161,7 +172,20 @@ def cmp_dataset(graph_spaces, rds = rs._data_set_wrapper._data_set res, i = dscmp(rds, ds) - assert res, f"Fail to exec: {ngql}\nResponse: {dsp(rds)}\nExpected: {dsp(ds)}\nNotFoundRow: {rowp(ds, i)}" + if not res: + scen = request.function.__scenario__ + feature = scen.feature.rel_filename + location = f"{feature}:{line_number(scen._steps, result)}" + msg = [ + f"Fail to exec: {ngql}", + f"Response: {dsp(rds)}", + f"Expected: {dsp(ds)}", + f"NotFoundRow: {rowp(ds, i)}", + f"Location: {location}", + f"Space: {str(space_desc)}", + f"vid_fn: {vid_fn}", + ] + assert res, "\n".join(msg) @then(parse("define some list variables:\n{text}")) @@ -174,28 +198,33 @@ def define_list_var_alias(text, graph_spaces): @then(parse("the result should be, in order:\n{result}")) -def result_should_be_in_order(result, graph_spaces): - cmp_dataset(graph_spaces, result, order=True, strict=True) +def result_should_be_in_order(request, result, graph_spaces): + cmp_dataset(request, graph_spaces, result, order=True, strict=True) @then(parse("the result should be, in order, with relax comparison:\n{result}")) -def result_should_be_in_order_relax_cmp(result, graph_spaces): - cmp_dataset(graph_spaces, result, order=True, strict=False) +def result_should_be_in_order_relax_cmp(request, result, graph_spaces): + cmp_dataset(request, graph_spaces, result, order=True, strict=False) @then(parse("the result should be, in any order:\n{result}")) -def result_should_be(result, graph_spaces): - cmp_dataset(graph_spaces, result, order=False, strict=True) +def result_should_be(request, result, graph_spaces): + cmp_dataset(request, graph_spaces, result, order=False, strict=True) @then(parse("the result should be, in any order, with relax comparison:\n{result}")) -def result_should_be_relax_cmp(result, graph_spaces): - cmp_dataset(graph_spaces, result, order=False, strict=False) +def result_should_be_relax_cmp(request, result, graph_spaces): + cmp_dataset(request, graph_spaces, result, order=False, strict=False) @then(parse("the result should include:\n{result}")) -def result_should_include(result, graph_spaces): - cmp_dataset(graph_spaces, result, order=False, strict=True, included=True) +def result_should_include(request, result, graph_spaces): + cmp_dataset(request, + graph_spaces, + result, + order=False, + strict=True, + included=True) @then("no side effects") diff --git a/tests/tck/features/delete/DeleteEdge.IntVid.feature b/tests/tck/features/delete/DeleteEdge.IntVid.feature index 1e59f872b42508c09bb12bb7f24684f295847430..7d2f000c7c8d0b3447815ccc7ded65ba902c2ec2 100644 --- a/tests/tck/features/delete/DeleteEdge.IntVid.feature +++ b/tests/tck/features/delete/DeleteEdge.IntVid.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@delete_e_int Feature: Delete int vid of edge Background: Prepare space diff --git a/tests/tck/features/delete/DeleteEdge.feature b/tests/tck/features/delete/DeleteEdge.feature index 518e2b233e1b240b9eff3d006a89468ff2cf2cc8..c1d2fc323a8eb0ff5ff643fdf8333477e76c6a8b 100644 --- a/tests/tck/features/delete/DeleteEdge.feature +++ b/tests/tck/features/delete/DeleteEdge.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@delete_e_string Feature: Delete string vid of edge Scenario: delete edges diff --git a/tests/tck/features/delete/DeleteVertex.IntVid.feature b/tests/tck/features/delete/DeleteVertex.IntVid.feature index 7d4e18c935380819957bffc2da0a383f7bd0e5a4..8d7f76a1f212ae282d2b48693d8d4cac3c6fa37e 100644 --- a/tests/tck/features/delete/DeleteVertex.IntVid.feature +++ b/tests/tck/features/delete/DeleteVertex.IntVid.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@delete_v_int Feature: Delete int vid of vertex Background: Prepare space diff --git a/tests/tck/features/delete/DeleteVertex.feature b/tests/tck/features/delete/DeleteVertex.feature index cf866bae4e0814f9b8466b6a124eb2f3b1f89612..d6f79d0134b8c1b3b052429737ead6107f341fae 100644 --- a/tests/tck/features/delete/DeleteVertex.feature +++ b/tests/tck/features/delete/DeleteVertex.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@delete_v_string Feature: Delete string vid of vertex Scenario: delete string vertex diff --git a/tests/tck/features/insert/Insert.IntVid.feature b/tests/tck/features/insert/Insert.IntVid.feature index b4242d0f2b056cdbf6f54c0b21fbce94ec64f3dc..5cf107efcc1e944977c0d9dc44286a6e91cd133e 100644 --- a/tests/tck/features/insert/Insert.IntVid.feature +++ b/tests/tck/features/insert/Insert.IntVid.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@insert_int Feature: Insert int vid of vertex and edge Background: Prepare space diff --git a/tests/tck/features/insert/Insert.feature b/tests/tck/features/insert/Insert.feature index 292cdfb6e57f8f5eff2fa51bf28651af18fe539c..5e4698c870fc5c08b5bfd64b2eca53794e9c6e4b 100644 --- a/tests/tck/features/insert/Insert.feature +++ b/tests/tck/features/insert/Insert.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@insert_string Feature: Insert string vid of vertex and edge Scenario: insert vertex and edge test diff --git a/tests/tck/features/update/Update.IntVid.feature b/tests/tck/features/update/Update.IntVid.feature index 9a469cb84028d9d76bada78e4b3878a088608c76..16b77939ef3489f8d7973189bd8f1a4f7067f4db 100644 --- a/tests/tck/features/update/Update.IntVid.feature +++ b/tests/tck/features/update/Update.IntVid.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@update_int Feature: Update int vid of vertex and edge Background: Prepare space diff --git a/tests/tck/features/update/Update.feature b/tests/tck/features/update/Update.feature index bebc425a0a07757e65400b8cdac147ba200eedb4..1b690d578ffbf094d795fc5ba259f17055b1a7e1 100644 --- a/tests/tck/features/update/Update.feature +++ b/tests/tck/features/update/Update.feature @@ -2,7 +2,6 @@ # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -@update_string Feature: Update string vid of vertex and edge Background: Prepare space