Skip to content
Snippets Groups Projects
Unverified Commit dd4cbaad authored by Yee's avatar Yee Committed by GitHub
Browse files

Support more features in test framework (#507)

* Fix error handle steps

* Fix error message

* Print row index

* Print row index in assert msg

* Fix workflow

* Fix row index and cleanup

* Fix Makefile

* Load nba int vid space

* Support int vid hash
parent 374cf58b
No related branches found
No related tags found
No related merge requests found
# Copyright (c) 2020 vesoft inc. All rights reserved.
#
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
if(NOT ${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
find_program(PYTHON_EXECUTABLE python3
/usr/bin /usr/local/bin
NO_CMAKE_ENVIRONMENT_PATH
NO_SYSTEM_ENVIRONMENT_PATH
)
find_program(PYTHON_EXECUTABLE python)
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/ntr.out-of-source
${CMAKE_CURRENT_BINARY_DIR}/nebula-test-run.py
@ONLY
)
endif()
execute_process(
COMMAND chmod +x nebula-test-run.py
COMMAND ${CMAKE_COMMAND} -E create_symlink ./nebula-test-run.py ntr
COMMAND ${CMAKE_COMMAND} -E create_symlink ./nebula-test-run.py nebula-test-run
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
......@@ -3,13 +3,14 @@
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
.PHONY: fmt check init clean test
.PHONY: fmt check init init-all clean test tck
PYPI_MIRROR = https://mirrors.aliyun.com/pypi/simple/
CURR_DIR = $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
gherkin_fmt = ~/.local/bin/reformat-gherkin
RM_DIR ?= true
TEST_DIR ?= $(CURR_DIR)
install-deps:
pip3 install --user -U setuptools wheel -i $(PYPI_MIRROR)
......@@ -38,10 +39,10 @@ check:
@find $(CURR_DIR)/tck/features -type f -iname "*.feature" -print | xargs $(gherkin_fmt) --check
test:
cd $(CURR_DIR) && python3 -m pytest -n8 --dist=loadfile --rm_dir=$(RM_DIR) -m "not skip"
cd $(CURR_DIR) && python3 -m pytest -n8 --dist=loadfile --rm_dir=$(RM_DIR) -m "not skip" $(TEST_DIR)
tck:
cd $(CURR_DIR) && python3 -m pytest -n8 --dist=loadfile --rm_dir=$(RM_DIR) -m "not skip" $(CURR_DIR)/tck/
cd $(CURR_DIR) && python3 -m pytest --gherkin-terminal-reporter --gherkin-terminal-reporter-expanded --rm_dir=$(RM_DIR) -m "not skip" $(CURR_DIR)/tck/
clean:
@rm -rf $(CURR_DIR)/nebula-python $(CURR_DIR)/reformat-gherkin $(CURR_DIR)/.pytest $(CURR_DIR)/.pytest_cache
......@@ -26,11 +26,13 @@ class DataSetComparator:
strict=True,
order=False,
included=False,
decode_type: str = 'utf-8'):
decode_type: str = 'utf-8',
vid_fn=None):
self._strict = strict
self._order = order
self._included = included
self._decode_type = decode_type
self._vid_fn = vid_fn
def __call__(self, resp: DataSet, expect: DataSet):
return self.compare(resp, expect)
......@@ -43,20 +45,23 @@ class DataSetComparator:
def compare(self, resp: DataSet, expect: DataSet):
if all(x is None for x in [expect, resp]):
return True
return True, None
if None in [expect, resp]:
return False
return False, -1
if len(resp.rows) < len(expect.rows):
return False
return False, -1
if len(resp.column_names) != len(expect.column_names):
return False
return False, -1
for (ln, rn) in zip(resp.column_names, expect.column_names):
if ln != self.bstr(rn):
return False
return False, -2
if self._order:
return all(
self.compare_row(l, r)
for (l, r) in zip(resp.rows, expect.rows))
for i in range(0, len(expect.rows)):
if not self.compare_row(resp.rows[i], expect.rows[i]):
return False, i
if self._included:
return True, None
return len(resp.rows) == len(expect.rows), -1
return self._compare_list(resp.rows, expect.rows, self.compare_row,
self._included)
......@@ -130,7 +135,8 @@ class DataSetComparator:
return False
lvals = lhs.get_uVal().values
rvals = rhs.get_uVal().values
return self._compare_list(lvals, rvals, self.compare_value)
res, _ = self._compare_list(lvals, rvals, self.compare_value)
return res
if lhs.getType() == Value.MVAL:
if not rhs.getType() == Value.MVAL:
return False
......@@ -202,14 +208,16 @@ class DataSetComparator:
if not lhs.ranking == rhs.ranking:
return False
rsrc, rdst = self.eid(rhs, lhs.type)
if lhs.src != rsrc or lhs.dst != rdst:
if not (self.compare_vid(lhs.src, rsrc)
and self.compare_vid(lhs.dst, rdst)):
return False
if rhs.props is None or len(lhs.props) != len(rhs.props):
return False
else:
if rhs.src is not None and rhs.dst is not None:
rsrc, rdst = self.eid(rhs, lhs.type)
if lhs.src != rsrc or lhs.dst != rdst:
if not (self.compare_vid(lhs.src, rsrc)
and self.compare_vid(lhs.dst, rdst)):
return False
if rhs.ranking is not None:
if lhs.ranking != rhs.ranking:
......@@ -224,18 +232,33 @@ class DataSetComparator:
def bstr(self, vid) -> bytes:
return self.b(vid) if type(vid) == str else vid
def compare_vid(
self,
lid: Union[int, bytes],
rid: Union[int, bytes, str],
) -> bool:
if type(lid) is bytes:
return type(rid) in [str, bytes] and lid == self.bstr(rid)
if type(lid) is int:
if type(rid) is int:
return lid == rid
if type(rid) not in [str, bytes] or self._vid_fn is None:
return False
return lid == self._vid_fn(rid)
return False
def compare_node(self, lhs: Vertex, rhs: Vertex):
rtags = []
if self._strict:
assert rhs.vid is not None
if not lhs.vid == self.bstr(rhs.vid):
if not self.compare_vid(lhs.vid, rhs.vid):
return False
if rhs.tags is None or len(lhs.tags) != len(rhs.tags):
return False
rtags = rhs.tags
else:
if rhs.vid is not None:
if not lhs.vid == self.bstr(rhs.vid):
if not self.compare_vid(lhs.vid, rhs.vid):
return False
if rhs.tags is not None and len(lhs.tags) < len(rhs.tags):
return False
......@@ -268,8 +291,10 @@ class DataSetComparator:
return True
def compare_list(self, lhs: List[Value], rhs: List[Value]):
return len(lhs) == len(rhs) and \
self._compare_list(lhs, rhs, self.compare_value)
if len(lhs) != len(rhs):
return False
res, _ = self._compare_list(lhs, rhs, self.compare_value)
return res
def compare_row(self, lhs: Row, rhs: Row):
if not len(lhs.values) == len(rhs.values):
......@@ -279,7 +304,7 @@ class DataSetComparator:
def _compare_list(self, lhs, rhs, cmp_fn, included=False):
visited = []
for rr in rhs:
for j, rr in enumerate(rhs):
found = False
for i, lr in enumerate(lhs):
if i not in visited and cmp_fn(lr, rr):
......@@ -287,8 +312,8 @@ class DataSetComparator:
found = True
break
if not found:
return False
return False, j
size = len(lhs)
if included:
return len(visited) <= size
return len(visited) == size
return len(visited) <= size, -1
return len(visited) == size, -1
......@@ -27,10 +27,11 @@ class DataSetPrinter:
return str(v)
def ds_to_string(self, ds: DataSet) -> str:
col_names = '[' + ','.join(self.sstr(col)
for col in ds.column_names) + ']'
data_rows = '\n'.join('[' + self.list_to_string(row.values) + ']'
for row in ds.rows)
col_names = '|' + '|'.join(self.sstr(col)
for col in ds.column_names) + '|'
data_rows = '\n'.join(
f'{i}: |' + self.list_to_string(row.values, delimiter='|') + '|'
for (i, row) in enumerate(ds.rows))
return '\n'.join([col_names, data_rows])
def to_string(self, val: Value):
......@@ -74,19 +75,19 @@ class DataSetPrinter:
return self.ds_to_string(val.get_gVal())
return ""
def list_to_string(self, lst: List[Value]) -> str:
return ','.join(self.to_string(val) for val in lst)
def list_to_string(self, lst: List[Value], delimiter: str = ",") -> str:
return delimiter.join(self.to_string(val) for val in lst)
def vertex_to_string(self, v: Vertex):
if v.vid is None:
return "()"
if v.tags is None:
return f'("{self.vid(v.vid)}")'
return f'({self.vid(v.vid)})'
tags = []
for tag in v.tags:
name = self.sstr(tag.name)
tags.append(f":{name}{self.map_to_string(tag.props)}")
return f'("{self.vid(v.vid)}"{"".join(tags)})'
return f'({self.vid(v.vid)}{"".join(tags)})'
def map_to_string(self, m: dict) -> str:
if m is None:
......@@ -97,7 +98,7 @@ class DataSetPrinter:
def edge_to_string(self, e: Edge) -> str:
name = "" if e.name is None else ":" + self.sstr(e.name)
arrow = "->" if e.type is None or e.type > 0 else "<-"
direct = f'"{self.vid(e.src)}"{arrow}"{self.vid(e.dst)}"'
direct = f'{self.vid(e.src)}{arrow}{self.vid(e.dst)}'
rank = "" if e.ranking is None else f"@{e.ranking}"
return f"[{name} {direct}{rank}{self.map_to_string(e.props)}]"
......
......@@ -123,7 +123,7 @@ class NebulaService(object):
for port in ports:
ports_status[port] = False
for i in range(0, 30):
for i in range(0, 20):
for port in ports_status:
if ports_status[port]:
continue
......@@ -191,7 +191,7 @@ class NebulaService(object):
print("try to stop nebula services...")
self.kill_all(signal.SIGTERM)
max_retries = 30
max_retries = 20
while self.is_procs_alive() and max_retries >= 0:
time.sleep(1)
max_retries = max_retries-1
......
......@@ -19,6 +19,17 @@ class SpaceDesc:
self.charset = charset
self.collate = collate
@staticmethod
def from_json(obj: dict):
return SpaceDesc(
name=obj.get('name', None),
vid_type=obj.get('vidType', 'FIXED_STRING(32)'),
partition_num=obj.get('partitionNum', 7),
replica_factor=obj.get('replicaFactor', 1),
charset=obj.get('charset', 'utf8'),
collate=obj.get('collate', 'utf8_bin'),
)
def create_stmt(self) -> str:
return f"""CREATE SPACE IF NOT EXISTS `{self.name}`( \
partition_num={self.partition_num}, \
......
......@@ -9,10 +9,9 @@ import os
import random
import string
import time
from pathlib import Path
import yaml
from typing import Pattern
import yaml
from nebula2.common import ttypes as CommonTtypes
from nebula2.gclient.net import Session
......@@ -322,10 +321,15 @@ def space_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
def check_resp(resp, stmt):
msg = f"Fail to exec: {stmt}, error: {resp.error_msg()}"
assert resp.is_succeeded(), msg
def create_space(space_desc: SpaceDesc, sess: Session):
def exec(stmt):
resp = sess.execute(stmt)
assert resp.is_succeeded(), f"Fail to exec: {stmt}, {resp.error_msg()}"
check_resp(resp, stmt)
exec(space_desc.drop_stmt())
exec(space_desc.create_stmt())
......@@ -336,11 +340,15 @@ def create_space(space_desc: SpaceDesc, sess: Session):
def _load_data_from_file(sess, data_dir, fd):
for stmt in CSVImporter(fd, data_dir):
rs = sess.execute(stmt)
assert rs.is_succeeded(), \
f"fail to exec: {stmt}, error: {rs.error_msg()}"
check_resp(rs, stmt)
def load_csv_data(pytestconfig, sess: Session, data_dir: str):
def load_csv_data(
pytestconfig,
sess: Session,
data_dir: str,
space_name: str = "",
):
"""
Before loading CSV data files, you must create and select a graph
space. The `config.yaml' file only create schema about tags and
......@@ -350,12 +358,30 @@ def load_csv_data(pytestconfig, sess: Session, data_dir: str):
with open(config_path, 'r') as f:
config = yaml.full_load(f)
space = config.get('space', None)
assert space is not None
if not space_name:
space_name = space.get('name', "A" + space_generator())
space_desc = SpaceDesc(
name=space_name,
vid_type=space.get('vidType', 'FIXED_STRING(32)'),
partition_num=space.get('partitionNum', 7),
replica_factor=space.get('replicaFactor', 1),
charset=space.get('charset', 'utf8'),
collate=space.get('collate', 'utf8_bin'),
)
create_space(space_desc, sess)
schemas = config['schema']
stmts = ' '.join(map(lambda x: x.strip(), schemas.splitlines()))
rs = sess.execute(stmts)
assert rs.is_succeeded()
check_resp(rs, stmts)
time.sleep(3)
for fd in config["files"]:
_load_data_from_file(sess, data_dir, fd)
return space_desc
......@@ -18,7 +18,7 @@ from nebula2.gclient.net import ConnectionPool
from tests.common.configs import all_configs
from tests.common.nebula_service import NebulaService
from tests.common.types import SpaceDesc
from tests.common.utils import create_space, load_csv_data
from tests.common.utils import load_csv_data
tests_collected = set()
tests_executed = set()
......@@ -152,46 +152,73 @@ def session(conn_pool, pytestconfig):
sess.release()
def load_csv_data_once(tmp_path_factory, pytestconfig, worker_id, conn_pool,
space_desc: SpaceDesc):
space_name = space_desc.name
def load_csv_data_once(
tmp_path_factory,
pytestconfig,
worker_id,
conn_pool: ConnectionPool,
space: str,
):
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / f"csv-data-{space_name}"
fn = root_tmp_dir / f"csv-data-{space}"
is_file = True
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
data_dir = os.path.join(CURR_PATH, "data", space_name)
data_dir = os.path.join(CURR_PATH, "data", space)
user = pytestconfig.getoption("user")
password = pytestconfig.getoption("password")
sess = conn_pool.get_session(user, password)
create_space(space_desc, sess)
load_csv_data(pytestconfig, sess, data_dir)
space_desc = load_csv_data(pytestconfig, sess, data_dir)
sess.release()
fn.write_text(space_name)
fn.write_text(json.dumps(space_desc.__dict__))
is_file = False
else:
space_desc = SpaceDesc.from_json(json.loads(fn.read_text()))
if is_file:
logging.info(
f"session-{worker_id} need not to load {space_name} csv data")
logging.info(f"session-{worker_id} need not to load {space} csv data")
yield space_desc
else:
logging.info(f"session-{worker_id} load {space_name} csv data")
logging.info(f"session-{worker_id} load {space} csv data")
yield space_desc
os.remove(str(fn))
# TODO(yee): optimize data load fixtures
@pytest.fixture(scope="session")
def load_nba_data(conn_pool, pytestconfig, tmp_path_factory, worker_id):
space_desc = SpaceDesc(name="nba", vid_type="FIXED_STRING(30)")
yield from load_csv_data_once(tmp_path_factory, pytestconfig, worker_id,
conn_pool, space_desc)
yield from load_csv_data_once(
tmp_path_factory,
pytestconfig,
worker_id,
conn_pool,
"nba",
)
# @pytest.fixture(scope="session")
# def load_nba_int_vid_data(
# conn_pool,
# pytestconfig,
# tmp_path_factory,
# worker_id,
# ):
# yield from load_csv_data_once(
# tmp_path_factory,
# pytestconfig,
# worker_id,
# conn_pool,
# "nba_int_vid",
# )
@pytest.fixture(scope="session")
def load_student_data(conn_pool, pytestconfig, tmp_path_factory, worker_id):
space_desc = SpaceDesc(name="student", vid_type="FIXED_STRING(8)")
yield from load_csv_data_once(tmp_path_factory, pytestconfig, worker_id,
conn_pool, space_desc)
yield from load_csv_data_once(
tmp_path_factory,
pytestconfig,
worker_id,
conn_pool,
"student",
)
# TODO(yee): Delete this when we migrate all test cases
......
#!@PYTHON_EXECUTABLE@
# Call ntr in out-of-source build
import os, sys
os.environ['NEBULA_BUILD_DIR'] = '@CMAKE_BINARY_DIR@'
os.environ['NEBULA_SOURCE_DIR'] = '@CMAKE_SOURCE_DIR@'
exec(open("@CMAKE_SOURCE_DIR@/tests/nebula-test-run.py").read())
......@@ -20,6 +20,7 @@ from tests.common.configs import DATA_DIR
from tests.common.types import SpaceDesc
from tests.common.utils import create_space, load_csv_data, space_generator
from tests.tck.utils.table import dataset, table
from tests.tck.utils.nbv import murmurhash2
parse = functools.partial(parsers.parse)
rparse = functools.partial(parsers.re)
......@@ -31,10 +32,18 @@ def graph_spaces():
@given(parse('a graph with space named "{space}"'))
def preload_space(space, load_nba_data, load_student_data, session,
graph_spaces):
def preload_space(
space,
load_nba_data,
# load_nba_int_vid_data,
load_student_data,
session,
graph_spaces,
):
if space == "nba":
graph_spaces["space_desc"] = load_nba_data
# elif space == "nba_int_vid":
# graph_spaces["space_desc"] = load_nba_int_vid_data
elif space == "student":
graph_spaces["space_desc"] = load_student_data
else:
......@@ -74,21 +83,24 @@ def new_space(options, session, graph_spaces):
graph_spaces["drop_space"] = True
@given(parse('import "{data}" csv data'))
@given(parse('load "{data}" csv data to a new space'))
def import_csv_data(data, graph_spaces, session, pytestconfig):
data_dir = os.path.join(DATA_DIR, data)
space_desc = graph_spaces["space_desc"]
space_desc = load_csv_data(
pytestconfig,
session,
data_dir,
"I" + space_generator(),
)
assert space_desc is not None
resp = session.execute(space_desc.use_stmt())
assert resp.is_succeeded(), \
f"Fail to use {space_desc.name}, {resp.error_msg()}"
load_csv_data(pytestconfig, session, data_dir)
graph_spaces["space_desc"] = space_desc
@when(parse("executing query:\n{query}"))
def executing_query(query, graph_spaces, session):
ngql = " ".join(query.splitlines())
graph_spaces['result_set'] = session.execute(ngql)
graph_spaces['ngql'] = ngql
@given(parse("wait {secs:d} seconds"))
......@@ -103,16 +115,36 @@ def cmp_dataset(graph_spaces,
strict: bool,
included=False) -> None:
rs = graph_spaces['result_set']
ngql = graph_spaces['ngql']
space_desc = graph_spaces['space_desc']
assert rs.is_succeeded(), f"Response failed: {rs.error_msg()}"
vid_fn = murmurhash2 if space_desc.vid_type == 'int' else None
ds = dataset(table(result))
dscmp = DataSetComparator(strict=strict,
order=order,
included=included,
decode_type=rs._decode_type)
dsp = DataSetPrinter(rs._decode_type)
resp_ds = rs._data_set_wrapper._data_set
assert dscmp(resp_ds, ds), \
f"Response: {dsp.ds_to_string(resp_ds)} vs. Expected: {dsp.ds_to_string(ds)}"
decode_type=rs._decode_type,
vid_fn=vid_fn)
def dsp(ds):
printer = DataSetPrinter(rs._decode_type)
return printer.ds_to_string(ds)
def rowp(ds, i):
if i is None or i < 0:
return ""
assert i < len(ds.rows), f"{i} out of range {len(ds.rows)}"
row = ds.rows[i].values
printer = DataSetPrinter(rs._decode_type)
ss = printer.list_to_string(row, delimiter='|')
return f'{i}: |' + ss + '|'
if rs._data_set_wrapper is None:
assert not ds.column_names and not ds.rows, f"Expected result must be empty table: ||"
rds = rs._data_set_wrapper._data_set
res, i = dscmp(rds, ds)
assert res, f"Fail to exec: {ngql}\nResponse: {dsp(rds)}\nExpected: {dsp(ds)}\nNotFoundRow: {rowp(ds, i)}"
@then(parse("the result should be, in order:\n{result}"))
......@@ -151,10 +183,9 @@ def execution_should_be_succ(graph_spaces):
assert rs is not None, "Please execute a query at first"
assert rs.is_succeeded(), f"Response failed: {rs.error_msg()}"
@then(
rparse("a (?P<err_type>\w+) should be raised at (?P<time>runtime|compile time):(?P<msg>.*)")
)
def raised_type_error(err_type, time, msg, graph_spaces):
@then(rparse(r"a (?P<err_type>\w+) should be raised at (?P<time>runtime|compile time)(?P<sym>:|.)(?P<msg>.*)"))
def raised_type_error(err_type, time, sym, msg, graph_spaces):
res = graph_spaces["result_set"]
assert not res.is_succeeded(), "Response should be failed"
err_type = err_type.strip()
......
......@@ -2,13 +2,7 @@ Feature: Feature examples
Scenario: Supported features
Given an empty graph
And create a space with following options:
| partition_num | 9 |
| replica_factor | 1 |
| vid_type | FIXED_STRING(30) |
| charset | utf8 |
| collate | utf8_bin |
And import "nba" csv data
And load "nba" csv data to a new space
And having executed:
"""
CREATE TAG IF NOT EXISTS `test_tag`(name string)
......@@ -43,5 +37,20 @@ Feature: Feature examples
"""
CREATE TAG player(name string);
"""
Then a ExecutionError should be raised at runtime:
Then a ExecutionError should be raised at runtime.
Then drop the used space
Scenario: Supported space creation
Given an empty graph
And create a space with following options:
| partition_num | 9 |
| replica_factor | 1 |
| vid_type | FIXED_STRING(30) |
| charset | utf8 |
| collate | utf8_bin |
When executing query:
"""
SHOW SPACES
"""
Then the execution should be successful
Then drop the used space
# Copyright (c) 2020 vesoft inc. All rights reserved.
#
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
def __bytes2ul(b):
return int.from_bytes(b, byteorder='little', signed=False)
def mmh2(bstr, seed = 0xc70f6907, signed = True):
def mmh2(bstr, seed=0xc70f6907, signed=True):
MASK = 2 ** 64 - 1
size = len(bstr)
m = 0xc6a4a7935bd1e995
......@@ -42,6 +48,7 @@ def mmh2(bstr, seed = 0xc70f6907, signed = True):
return h
if __name__ == '__main__':
assert mmh2(b'hello') == 2762169579135187400
assert mmh2(b'World') == -295471233978816215
......
......@@ -7,7 +7,11 @@
import re
import ply.lex as lex
import ply.yacc as yacc
from tests.tck.utils.mmh2 import mmh2
if __name__ == "__main__":
from mmh2 import mmh2
else:
from tests.tck.utils.mmh2 import mmh2
from nebula2.common.ttypes import (
Value,
......@@ -528,12 +532,15 @@ lexer = lex.lex()
parser = yacc.yacc()
functions = {}
def murmurhash2(v):
if isinstance(v, Value):
v = v.get_sVal()
else:
assert isinstance(v, str)
return mmh2(bytes(v, 'utf-8'))
if type(v) is str:
return mmh2(bytes(v, 'utf-8'))
if type(v) is bytes:
return mmh2(v)
raise ValueError(f"Invalid value type: {type(v)}")
def register_function(name, func):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment