Skip to content
Snippets Groups Projects
Unverified Commit 9c8bba4d authored by Yee's avatar Yee Committed by GitHub
Browse files

Replace behave framework with pytest-bdd and add DataSetComparator (#447)

* Replace behave with pytest_bdd

* Add value comparator

* Finish value compare

* Fix set compare

* Execute query and check result steps

* Make parser return bytes array rather than string

* Add NBA CSV data files

* Add types

* Generate insert stmt string

* Fix row comparing

* Generate create stmt string

* Use pytest-mproc's global scope to init nebula servers

* Fix compare bug

* Add student data

* Cleanup steps

* Fix parser bug

* Update parser

* Remove pytest-mproc

* Fix cleanup nebula services

* Fix ci test run parallel

* Replace conn pool creation with fixture

* Improve the free ports get function

* Fix workarround for class fixture usage

* improve free port find method

* Fix test spaces

* Change pytest temp base dir

* Improve kill nebula service logics

* Update workflow

* Fix github workflow step

* Address comments
parent 2e9d27ec
No related branches found
No related tags found
No related merge requests found
Showing
with 954 additions and 113 deletions
......@@ -131,8 +131,9 @@ jobs:
- name: Pytest
env:
NEBULA_TEST_LOGS_DIR: ${{ github.workspace }}/build
run: ./ci/test.sh test --rm_dir=false
run: python3 -m pytest --dist=loadfile -n8 --rm_dir=false --build_dir="$PWD/../build" -m "not skip"
timeout-minutes: 25
working-directory: tests
- name: Upload logs
uses: actions/upload-artifact@v2
if: ${{ failure() }}
......
......@@ -181,4 +181,3 @@ nebula_add_subdirectory(src)
nebula_add_subdirectory(conf)
nebula_add_subdirectory(resources)
nebula_add_subdirectory(scripts)
nebula_add_subdirectory(tests)
......@@ -91,16 +91,12 @@ function run_ctest() {
function run_test() {
export PYTHONPATH=$PROJ_DIR:$PYTHONPATH
testpath=$(cat $PROJ_DIR/ci/tests.txt | sed "s|\(.*\)|$PROJ_DIR/tests/\1|g" | tr '\n' ' ')
$BUILD_DIR/tests/ntr \
-n=8 \
pytest -n 8 --build_dir=$BUILD_DIR \
--dist=loadfile \
--debug_log=false \
${@:1} \
$testpath
${@:1}
$BUILD_DIR/tests/ntr --debug_log=false ${@:1} $PROJ_DIR/tests/job/*
# $BUILD_DIR/tests/ntr --debug_log=false ${@:1} $PROJ_DIR/tests/job/*
}
function test_in_cluster() {
......
.pytest
.pytest_cache
......@@ -7,11 +7,8 @@
import time
import pytest
from tests.common.nebula_test_suite import NebulaTestSuite
from nebula2.graph import ttypes
from tests.common.nebula_test_suite import NebulaTestSuite
class TestPermission(NebulaTestSuite):
......
......@@ -6,21 +6,12 @@
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
import time
import re
from tests.common.nebula_test_suite import NebulaTestSuite
class TestSpace(NebulaTestSuite):
@classmethod
def prepare(self):
pass
@classmethod
def cleanup(self):
pass
def test_space(self):
# not exist
resp = self.client.execute('USE not_exist_space')
......@@ -74,7 +65,6 @@ class TestSpace(NebulaTestSuite):
resp = self.client.execute('DROP SPACE default_space')
self.check_resp_succeeded(resp)
resp = self.client.execute(create_space_str)
self.check_resp_succeeded(resp)
......
......@@ -9,6 +9,7 @@ import time
from tests.common.nebula_test_suite import NebulaTestSuite
class TestUsers(NebulaTestSuite):
@classmethod
def prepare(self):
......
# Copyright (c) 2020 vesoft inc. All rights reserved.
#
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
import csv
import re
from tests.common.types import (
VID,
Rank,
Prop,
Tag,
Edge,
Vertex,
)
class CSVImporter:
_SRC_VID = ':SRC_VID'
_DST_VID = ':DST_VID'
_VID = ':VID'
_RANK = ':RANK'
def __init__(self, filepath):
self._filepath = filepath
self._insert_stmt = ""
self._create_stmt = ""
self._type = None
def __iter__(self):
with open(self._filepath, 'r') as f:
for i, row in enumerate(csv.reader(f)):
if i == 0:
yield self.parse_header(row)
else:
yield self.process(row)
def process(self, row: list):
if isinstance(self._type, Vertex):
return self.build_vertex_insert_stmt(row)
return self.build_edge_insert_stmt(row)
def build_vertex_insert_stmt(self, row: list):
props = []
for p in self._type.tags[0].props:
col = row[p.index]
props.append(self.value(p.ptype, col))
vid = self._type.vid
id_val = self.value(vid.id_type, row[vid.index])
return f'{self._insert_stmt} {id_val}:({",".join(props)});'
def build_edge_insert_stmt(self, row: list):
props = []
for p in self._type.props:
col = row[p.index]
props.append(self.value(p.ptype, col))
src = self._type.src
dst = self._type.dst
src_vid = self.value(src.id_type, row[src.index])
dst_vid = self.value(dst.id_type, row[dst.index])
if self._type.rank is None:
return f'{self._insert_stmt} {src_vid}->{dst_vid}:({",".join(props)});'
rank = row[self._type.rank.index]
return f'{self._insert_stmt} {src_vid}->{dst_vid}@{rank}:({",".join(props)});'
def value(self, ptype: str, col):
return f'"{col}"' if ptype == 'string' else f'{col}'
def parse_header(self, row):
"""
Only parse the scenario that one tag in each file
"""
for col in row:
if self._SRC_VID in col or self._DST_VID in col:
self._type = Edge()
self.parse_edge(row)
break
if self._VID in col:
self._type = Vertex()
self.parse_vertex(row)
break
if self._type is None:
raise ValueError(f'Invalid csv header: {",".join(row)}')
return self._create_stmt
def parse_edge(self, row):
props = []
name = ''
for i, col in enumerate(row):
if col == self._RANK:
self._type.rank = Rank(i)
continue
m = re.search(r':SRC_VID\((.*)\)', col)
if m:
self._type.src = VID(i, m.group(1))
continue
m = re.search(r':DST_VID\((.*)\)', col)
if m:
self._type.dst = VID(i, m.group(1))
continue
m = re.search(r'(\w+)\.(\w+):(\w+)', col)
if not m:
raise ValueError(f'Invalid csv header format {col}')
g1 = m.group(1)
if not name:
name = g1
assert name == g1, f'Different edge type {g1}'
props.append(Prop(i, m.group(2), m.group(3)))
self._type.name = name
self._type.props = props
pdecl = ','.join(p.name for p in props)
self._insert_stmt = f"INSERT EDGE {name}({pdecl}) VALUES"
pdecl = ','.join(f"`{p.name}` {p.ptype}" for p in props)
self._create_stmt = f"CREATE EDGE IF NOT EXISTS `{name}`({pdecl});"
def parse_vertex(self, row):
tag = Tag()
props = []
for i, col in enumerate(row):
m = re.search(r':VID\((.*)\)', col)
if m:
self._type.vid = VID(i, m.group(1))
continue
m = re.search(r'(\w+)\.(\w+):(\w+)', col)
if not m:
raise ValueError(f'Invalid csv header format {col}')
g1 = m.group(1)
if not tag.name:
tag.name = g1
assert tag.name == g1, f'Different tag name {g1}'
props.append(Prop(i, m.group(2), m.group(3)))
tag.props = props
self._type.tags = [tag]
pdecl = ','.join(p.name for p in tag.props)
self._insert_stmt = f"INSERT VERTEX {tag.name}({pdecl}) VALUES"
pdecl = ','.join(f"`{p.name}` {p.ptype}" for p in tag.props)
self._create_stmt = f"CREATE TAG IF NOT EXISTS `{tag.name}`({pdecl});"
if __name__ == '__main__':
for row in CSVImporter('../data/nba/player.csv'):
print(row)
......@@ -10,7 +10,6 @@ import time
from nebula2.gclient.net import ConnectionPool
from nebula2.Config import Config
from nebula2.graph import ttypes
from tests.common.configs import get_delay_time
......@@ -83,7 +82,7 @@ class GlobalDataLoader(object):
# The whole test will load once, for the only read tests
def load_student(self):
resp = self.client.execute(
'CREATE SPACE IF NOT EXISTS student_space(partition_num=10, replica_factor=1, vid_type = fixed_string(8)); USE student_space;')
'CREATE SPACE IF NOT EXISTS student(partition_num=10, replica_factor=1, vid_type = fixed_string(8)); USE student;')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute('CREATE TAG IF NOT EXISTS person(name string, age int, gender string);')
......@@ -209,5 +208,5 @@ class GlobalDataLoader(object):
assert resp.is_succeeded(), resp.error_msg()
def drop_data(self):
resp = self.client.execute('DROP SPACE nba; DROP SPACE student_space;')
resp = self.client.execute('DROP SPACE nba; DROP SPACE student;')
assert resp.is_succeeded(), resp.error_msg()
......@@ -18,11 +18,19 @@ NEBULA_START_COMMAND_FORMAT = "bin/nebula-{} --flagfile conf/nebula-{}.conf {}"
class NebulaService(object):
def __init__(self, build_dir, src_dir):
def __init__(self, build_dir, src_dir, cleanup=True):
self.build_dir = build_dir
self.src_dir = src_dir
self.work_dir = os.path.join(self.build_dir, 'server_' + time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime()))
self.pids = {}
self._cleanup = cleanup
def __enter__(self):
self.install()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stop(cleanup=self._cleanup)
def set_work_dir(self, work_dir):
self.work_dir = work_dir
......@@ -65,14 +73,32 @@ class NebulaService(object):
command = NEBULA_START_COMMAND_FORMAT.format(name, name, param)
return command
def _find_free_port(self):
@staticmethod
def is_port_in_use(port):
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
return s.connect_ex(('localhost', port)) == 0
@staticmethod
def get_free_port():
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
s.bind(('', 0))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
return s.getsockname()[1]
# TODO(yee): Find free port range
@staticmethod
def _find_free_port():
# tcp_port, http_port, https_port
ports = []
for i in range(0, 3):
with closing(socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as s:
s.bind(('', 0))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ports.append(s.getsockname()[1])
for i in range(0, 2):
ports.append(NebulaService.get_free_port())
while True:
port = NebulaService.get_free_port()
if port not in ports and all(
not NebulaService.is_port_in_use(port + i)
for i in range(-2, 3)):
ports.insert(0, port)
break
return ports
def _telnet_port(self, port):
......@@ -116,13 +142,19 @@ class NebulaService(object):
os.chdir(self.work_dir)
metad_ports = self._find_free_port()
all_ports = [metad_ports[0]]
command = ''
graph_port = 0
server_ports = []
for server_name in ['metad', 'storaged', 'graphd']:
ports = []
if server_name != 'metad':
ports = self._find_free_port()
while True:
ports = self._find_free_port()
if all((ports[0] + i) not in all_ports
for i in range(-2, 3)):
all_ports += [ports[0]]
break
else:
ports = metad_ports
server_ports.append(ports[0])
......@@ -141,8 +173,9 @@ class NebulaService(object):
# wait nebula start
start_time = time.time()
if not self._check_servers_status(server_ports):
raise Exception('nebula servers not ready in {}s'.format(time.time() - start_time))
print('nebula servers start ready in {}s'.format(time.time() - start_time))
raise Exception(
f'nebula servers not ready in {time.time() - start_time}s')
print(f'nebula servers start ready in {time.time() - start_time}s')
for pf in glob.glob(self.work_dir + '/pids/*.pid'):
with open(pf) as f:
......@@ -150,30 +183,42 @@ class NebulaService(object):
return graph_port
def stop(self, cleanup):
def stop(self):
print("try to stop nebula services...")
for p in self.pids:
try:
os.kill(self.pids[p], signal.SIGTERM)
except OSError as err:
print("nebula stop {} failed: {}".format(p, str(err)))
self.kill_all(signal.SIGTERM)
max_retries = 30
while self.check_procs_alive() and max_retries >= 0:
while self.is_procs_alive() and max_retries >= 0:
time.sleep(1)
max_retries = max_retries-1
if cleanup:
self.kill_all(signal.SIGKILL)
if self._cleanup:
shutil.rmtree(self.work_dir, ignore_errors=True)
def check_procs_alive(self):
def kill_all(self, sig):
for p in self.pids:
self.kill(p, sig)
def kill(self, pid, sig):
if not self.is_proc_alive(pid):
return
try:
os.kill(self.pids[pid], sig)
except OSError as err:
print("stop nebula {} failed: {}".format(pid, str(err)))
def is_procs_alive(self):
return any(self.is_proc_alive(pid) for pid in self.pids)
def is_proc_alive(self, pid):
process = subprocess.Popen(['ps', '-eo', 'pid,args'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout = process.communicate()
for line in bytes.decode(stdout[0]).splitlines():
pid = line.lstrip().split(' ', 1)[0]
for p in self.pids:
if str(self.pids[p]) == str(pid):
return True
p = line.lstrip().split(' ', 1)[0]
if str(p) == str(self.pids[pid]):
return True
return False
......@@ -5,21 +5,25 @@
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
import pytest
import time
import datetime
import pytest
import re
from pathlib import Path
from typing import Pattern, Set
from nebula2.common import ttypes as CommonTtypes
from nebula2.gclient.net import ConnectionPool
from nebula2.Config import Config
# from nebula2.gclient.net import ConnectionPool
# from nebula2.Config import Config
from nebula2.graph import ttypes
from tests.common.configs import get_delay_time
from tests.common.utils import compare_value, \
row_to_string, to_value, value_to_string, find_in_rows
from tests.common.utils import (
compare_value,
row_to_string,
to_value,
value_to_string,
find_in_rows,
)
T_EMPTY = CommonTtypes.Value()
......@@ -37,32 +41,31 @@ T_NULL_UNKNOWN_DIV_BY_ZERO = CommonTtypes.Value()
T_NULL_UNKNOWN_DIV_BY_ZERO.set_nVal(CommonTtypes.NullType.DIV_BY_ZERO)
@pytest.mark.usefixtures("workarround_for_class")
class NebulaTestSuite(object):
@classmethod
def set_delay(self):
self.delay = get_delay_time(self.client)
@classmethod
def setup_class(self):
self.spaces = []
address = pytest.cmdline.address.split(':')
self.host = address[0]
self.port = address[1]
self.user = pytest.cmdline.user
self.password = pytest.cmdline.password
self.replica_factor = pytest.cmdline.replica_factor
self.partition_num = pytest.cmdline.partition_num
self.check_format_str = 'result: {}, expect: {}'
self.data_dir = pytest.cmdline.data_dir
self.data_loaded = False
self.create_nebula_clients()
self.set_delay()
self.prepare()
# @classmethod
# def setup_class(self):
# self.spaces = []
# self.user = pytest.cmdline.user
# self.password = pytest.cmdline.password
# self.replica_factor = pytest.cmdline.replica_factor
# self.partition_num = pytest.cmdline.partition_num
# self.check_format_str = 'result: {}, expect: {}'
# self.data_dir = pytest.cmdline.data_dir
# self.data_loaded = False
# self.create_nebula_clients()
# self.set_delay()
# self.prepare()
@classmethod
def load_data(self):
self.data_loaded = True
pathlist = Path(self.data_dir).rglob('*.ngql')
# pathlist = Path(self.data_dir).rglob('*.ngql')
pathlist = [Path(self.data_dir).joinpath("data/nba.ngql")]
for path in pathlist:
print("open: ", path)
with open(path, 'r') as data_file:
......@@ -115,20 +118,20 @@ class NebulaTestSuite(object):
@classmethod
def use_student_space(self):
resp = self.execute('USE student_space;')
resp = self.execute('USE student;')
self.check_resp_succeeded(resp)
@classmethod
def create_nebula_clients(self):
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
self.client_pool = ConnectionPool()
assert self.client_pool.init([(self.host, self.port)], config)
# @classmethod
# def create_nebula_clients(self):
# config = Config()
# config.max_connection_pool_size = 20
# config.timeout = 60000
# # init connection pool
# self.client_pool = ConnectionPool()
# assert self.client_pool.init([(self.host, self.port)], config)
# get session from the pool
self.client = self.client_pool.get_session(self.user, self.password)
# # get session from the pool
# self.client = self.client_pool.get_session(self.user, self.password)
@classmethod
def spawn_nebula_client(self, user, password):
......@@ -138,22 +141,17 @@ class NebulaTestSuite(object):
def release_nebula_client(self, client):
client.release()
@classmethod
def close_nebula_clients(self):
self.client_pool.close()
@classmethod
def teardown_class(self):
if self.client is not None:
self.cleanup()
self.drop_data()
self.client.release()
self.close_nebula_clients()
@classmethod
def execute(self, ngql, profile=True):
return self.client.execute(
'PROFILE {{{}}}'.format(ngql) if profile else ngql)
# @classmethod
# def close_nebula_clients(self):
# self.client_pool.close()
# @classmethod
# def teardown_class(self):
# if self.client is not None:
# self.cleanup()
# self.drop_data()
# self.client.release()
# self.close_nebula_clients()
@classmethod
def execute(self, ngql, profile=True):
......
# Copyright (c) 2020 vesoft inc. All rights reserved.
#
# This source code is licensed under Apache 2.0 License,
# attached with Common Clause Condition 1.0, found in the LICENSES directory.
class Column:
def __init__(self, index: int):
if index < 0:
raise ValueError(f"Invalid index of vid: {index}")
self._index = index
@property
def index(self):
return self._index
class VID(Column):
def __init__(self, index: int, vtype: str):
super().__init__(index)
if vtype not in ['int', 'string']:
raise ValueError(f'Invalid vid type: {vtype}')
self._type = vtype
@property
def id_type(self):
return self._type
class Rank(Column):
def __init__(self, index: int):
super().__init__(index)
class Prop(Column):
def __init__(self, index: int, name: str, ptype: str):
super().__init__(index)
self._name = name
if ptype not in ['string', 'int', 'double']:
raise ValueError(f'Invalid prop type: {ptype}')
self._type = ptype
@property
def name(self):
return self._name
@property
def ptype(self):
return self._type
class Properties:
def __init__(self):
self._name = ''
self._props = []
@property
def name(self):
return self._name
@name.setter
def name(self, name: str):
self._name = name
@property
def props(self):
return self._props
@props.setter
def props(self, props: list):
if any(not isinstance(p, Prop) for p in props):
raise ValueError("Invalid prop type in props")
self._props = props
class Tag(Properties):
def __init__(self):
super().__init__()
class Edge(Properties):
def __init__(self):
super().__init__()
self._src = None
self._dst = None
self._rank = None
@property
def src(self):
return self._src
@src.setter
def src(self, src: VID):
self._src = src
@property
def dst(self):
return self._dst
@dst.setter
def dst(self, dst: VID):
self._dst = dst
@property
def rank(self):
return self._rank
@rank.setter
def rank(self, rank: VID):
self._rank = rank
class Vertex:
def __init__(self):
self._vid = None
self.tags = []
@property
def vid(self):
return self._vid
@vid.setter
def vid(self, vid: VID):
self._vid = vid
@property
def tags(self):
return self._tags
@tags.setter
def tags(self, tags: list):
if any(not isinstance(t, Tag) for t in tags):
raise ValueError('Invalid tag type of vertex')
self._tags = tags
......@@ -7,28 +7,31 @@
import pytest
import os
import time
import logging
from tests.common.configs import all_configs
import json
from filelock import FileLock
from pathlib import Path
from nebula2.gclient.net import ConnectionPool
from nebula2.Config import Config
DOCKER_GRAPHD_DIGESTS = os.getenv('NEBULA_GRAPHD_DIGESTS')
if DOCKER_GRAPHD_DIGESTS is None:
DOCKER_GRAPHD_DIGESTS = '0'
DOCKER_METAD_DIGESTS = os.getenv('NEBULA_METAD_DIGESTS')
if DOCKER_METAD_DIGESTS is None:
DOCKER_METAD_DIGESTS = '0'
DOCKER_STORAGED_DIGESTS = os.getenv('NEBULA_STORAGED_DIGESTS')
if DOCKER_STORAGED_DIGESTS is None:
DOCKER_STORAGED_DIGESTS = '0'
from tests.common.configs import all_configs
from tests.common.nebula_service import NebulaService
from tests.common.csv_import import CSVImporter
tests_collected = set()
tests_executed = set()
data_dir = os.getenv('NEBULA_DATA_DIR')
CURR_PATH = os.path.dirname(os.path.abspath(__file__))
# pytest hook to handle test collection when xdist is used (parallel tests)
# https://github.com/pytest-dev/pytest-xdist/pull/35/commits (No official documentation available)
def pytest_xdist_node_collection_finished(node, ids):
tests_collected.update(set(ids))
# link to pytest_collection_modifyitems
# https://docs.pytest.org/en/5.3.2/writing_plugins.html#hook-function-validation-and-execution
@pytest.hookimpl(tryfirst=True)
......@@ -36,12 +39,14 @@ def pytest_collection_modifyitems(items):
for item in items:
tests_collected.add(item.nodeid)
# link to pytest_runtest_logreport
# https://docs.pytest.org/en/5.3.2/reference.html#_pytest.hookspec.pytest_runtest_logreport
def pytest_runtest_logreport(report):
if report.passed:
tests_executed.add(report.nodeid)
def pytest_addoption(parser):
for config in all_configs:
parser.addoption(config,
......@@ -49,6 +54,12 @@ def pytest_addoption(parser):
default=all_configs[config][1],
help=all_configs[config][2])
parser.addoption("--build_dir",
dest="build_dir",
default="",
help="Nebula Graph CMake build directory")
def pytest_configure(config):
pytest.cmdline.address = config.getoption("address")
pytest.cmdline.user = config.getoption("user")
......@@ -62,6 +73,184 @@ def pytest_configure(config):
pytest.cmdline.stop_nebula = config.getoption("stop_nebula")
pytest.cmdline.rm_dir = config.getoption("rm_dir")
pytest.cmdline.debug_log = config.getoption("debug_log")
config._metadata['graphd digest'] = DOCKER_GRAPHD_DIGESTS
config._metadata['metad digest'] = DOCKER_METAD_DIGESTS
config._metadata['storaged digest'] = DOCKER_STORAGED_DIGESTS
def get_conn_pool(host: str, port: int):
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
pool = ConnectionPool()
if not pool.init([(host, port)], config):
raise Exception("Fail to init connection pool.")
return pool
@pytest.fixture(scope="session")
def conn_pool(pytestconfig, worker_id, tmp_path_factory):
addr = pytestconfig.getoption("address")
if addr:
addrsplit = addr.split(":")
assert len(addrsplit) == 2
pool = get_conn_pool(addrsplit[0], addrsplit[1])
yield pool
pool.close()
return
build_dir = pytestconfig.getoption("build_dir")
rm_dir = pytestconfig.getoption("rm_dir")
project_dir = os.path.dirname(CURR_PATH)
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / "nebula-test"
nb = None
with FileLock(str(fn) + ".lock"):
if fn.is_file():
data = json.loads(fn.read_text())
port = data["port"]
logging.info(f"session-{worker_id} read the port: {port}")
pool = get_conn_pool("localhost", port)
data["num_workers"] += 1
fn.write_text(json.dumps(data))
else:
nb = NebulaService(build_dir, project_dir, rm_dir.lower() == "true")
nb.install()
port = nb.start()
pool = get_conn_pool("localhost", port)
data = dict(port=port, num_workers=1, finished=0)
fn.write_text(json.dumps(data))
logging.info(f"session-{worker_id} write the port: {port}")
yield pool
pool.close()
if nb is None:
with FileLock(str(fn) + ".lock"):
data = json.loads(fn.read_text())
data["finished"] += 1
fn.write_text(json.dumps(data))
else:
# TODO(yee): improve this option format, only specify it by `--stop_nebula`
stop_nebula = pytestconfig.getoption("stop_nebula")
while stop_nebula.lower() == "true":
data = json.loads(fn.read_text())
if data["finished"] + 1 == data["num_workers"]:
nb.stop()
break
time.sleep(1)
os.remove(str(fn))
@pytest.fixture(scope="session")
def session(conn_pool, pytestconfig):
user = pytestconfig.getoption("user")
password = pytestconfig.getoption("password")
sess = conn_pool.get_session(user, password)
yield sess
sess.release()
def load_csv_data(pytestconfig, conn_pool, folder: str):
data_dir = os.path.join(CURR_PATH, 'data', folder)
schema_path = os.path.join(data_dir, 'schema.ngql')
user = pytestconfig.getoption("user")
password = pytestconfig.getoption("password")
sess = conn_pool.get_session(user, password)
with open(schema_path, 'r') as f:
stmts = []
for line in f.readlines():
ln = line.strip()
if ln.startswith('--'):
continue
stmts.append(ln)
rs = sess.execute(' '.join(stmts))
assert rs.is_succeeded()
time.sleep(3)
for path in Path(data_dir).rglob('*.csv'):
for stmt in CSVImporter(path):
rs = sess.execute(stmt)
assert rs.is_succeeded()
sess.release()
# TODO(yee): optimize data load fixtures
@pytest.fixture(scope="session")
def load_nba_data(conn_pool, pytestconfig, tmp_path_factory, worker_id):
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / "csv-data-nba"
load = False
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
load_csv_data(pytestconfig, conn_pool, "nba")
fn.write_text("nba")
logging.info(f"session-{worker_id} load nba csv data")
load = True
else:
logging.info(f"session-{worker_id} need not to load nba csv data")
yield
if load:
os.remove(str(fn))
@pytest.fixture(scope="session")
def load_student_data(conn_pool, pytestconfig, tmp_path_factory, worker_id):
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / "csv-data-student"
load = False
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
load_csv_data(pytestconfig, conn_pool, "student")
fn.write_text("student")
logging.info(f"session-{worker_id} load student csv data")
load = True
else:
logging.info(
f"session-{worker_id} need not to load student csv data")
yield
if load:
os.remove(str(fn))
# TODO(yee): Delete this when we migrate all test cases
@pytest.fixture(scope="class")
def workarround_for_class(request, pytestconfig, tmp_path_factory, conn_pool,
session, load_nba_data, load_student_data):
if request.cls is None:
return
addr = pytestconfig.getoption("address")
if addr:
ss = addr.split(':')
request.cls.host = ss[0]
request.cls.port = ss[1]
else:
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / "nebula-test"
data = json.loads(fn.read_text())
request.cls.host = "localhost"
request.cls.port = data["port"]
request.cls.data_dir = os.path.dirname(os.path.abspath(__file__))
request.cls.spaces = []
request.cls.user = pytestconfig.getoption("user")
request.cls.password = pytestconfig.getoption("password")
request.cls.replica_factor = pytestconfig.getoption("replica_factor")
request.cls.partition_num = pytestconfig.getoption("partition_num")
request.cls.check_format_str = 'result: {}, expect: {}'
request.cls.data_loaded = False
request.cls.client_pool = conn_pool
request.cls.client = session
request.cls.set_delay()
request.cls.prepare()
yield
if request.cls.client is not None:
request.cls.cleanup()
request.cls.drop_data()
:VID(string),bachelor.name:string,bachelor.speciality:string
Tim Duncan,Tim Duncan,psychology
:SRC_VID(string),:DST_VID(string),like.likeness:int
Amar'e Stoudemire,Steve Nash,90
Russell Westbrook,Paul George,90
Russell Westbrook,James Harden,90
James Harden,Russell Westbrook,80
Tracy McGrady,Kobe Bryant,90
Tracy McGrady,Grant Hill,90
Tracy McGrady,Rudy Gay,90
Chris Paul,LeBron James,90
Chris Paul,Carmelo Anthony,90
Chris Paul,Dwyane Wade,90
Boris Diaw,Tony Parker,80
Boris Diaw,Tim Duncan,80
LeBron James,Ray Allen,100
Klay Thompson,Stephen Curry,90
Kristaps Porzingis,Luka Doncic,90
Marco Belinelli,Tony Parker,50
Marco Belinelli,Tim Duncan,55
Marco Belinelli,Danny Green,60
Luka Doncic,Dirk Nowitzki,90
Luka Doncic,Kristaps Porzingis,90
Luka Doncic,James Harden,80
Tony Parker,Tim Duncan,95
Tony Parker,Manu Ginobili,95
Tony Parker,LaMarcus Aldridge,90
Danny Green,Marco Belinelli,83
Danny Green,Tim Duncan,70
Danny Green,LeBron James,80
Rudy Gay,LaMarcus Aldridge,70
LaMarcus Aldridge,Tony Parker,75
LaMarcus Aldridge,Tim Duncan,75
Tim Duncan,Tony Parker,95
Tim Duncan,Manu Ginobili,95
Ray Allen,Rajon Rondo,9
Tiago Splitter,Tim Duncan,80
Tiago Splitter,Manu Ginobili,90
Paul Gasol,Kobe Bryant,90
Paul Gasol,Marc Gasol,99
Aron Baynes,Tim Duncan,80
Vince Carter,Tracy McGrady,90
Vince Carter,Jason Kidd,70
Marc Gasol,Paul Gasol,99
Ben Simmons,Joel Embiid,80
Rajon Rondo,Ray Allen,-1
Manu Ginobili,Tim Duncan,90
Kyrie Irving,LeBron James,13
Carmelo Anthony,LeBron James,90
Carmelo Anthony,Chris Paul,90
Carmelo Anthony,Dwyane Wade,90
Dwyane Wade,LeBron James,90
Dwyane Wade,Chris Paul,90
Dwyane Wade,Carmelo Anthony,90
Joel Embiid,Ben Simmons,80
Damian Lillard,LaMarcus Aldridge,80
Yao Ming,Tracy McGrady,90
Yao Ming,Shaquile O'Neal,90
Dejounte Murray,Tim Duncan,99
Dejounte Murray,Tony Parker,99
Dejounte Murray,Manu Ginobili,99
Dejounte Murray,Marco Belinelli,99
Dejounte Murray,Danny Green,99
Dejounte Murray,LeBron James,99
Dejounte Murray,Russell Westbrook,99
Dejounte Murray,Chris Paul,99
Dejounte Murray,Kyle Anderson,99
Dejounte Murray,Kevin Durant,99
Dejounte Murray,James Harden,99
Dejounte Murray,Tony Parker,99
Blake Griffin,Chris Paul,-1
Steve Nash,Amar'e Stoudemire,90
Steve Nash,Dirk Nowitzki,88
Steve Nash,Stephen Curry,90
Steve Nash,Jason Kidd,85
Jason Kidd,Vince Carter,80
Jason Kidd,Steve Nash,90
Jason Kidd,Dirk Nowitzki,85
Dirk Nowitzki,Steve Nash,80
Dirk Nowitzki,Jason Kidd,80
Dirk Nowitzki,Dwyane Wade,10
Paul George,Russell Westbrook,95
Grant Hill,Tracy McGrady,90
Shaquile O'Neal,JaVale McGee,100
Shaquile O'Neal,Tim Duncan,80
:VID(string),player.name:string,player.age:int
Nobody,Nobody,0
Amar'e Stoudemire,Amar'e Stoudemire,36
Russell Westbrook,Russell Westbrook,30
James Harden,James Harden,29
Kobe Bryant,Kobe Bryant,40
Tracy McGrady,Tracy McGrady,39
Chris Paul,Chris Paul,33
Boris Diaw,Boris Diaw,36
LeBron James,LeBron James,34
Klay Thompson,Klay Thompson,29
Kristaps Porzingis,Kristaps Porzingis,23
Jonathon Simmons,Jonathon Simmons,29
Marco Belinelli,Marco Belinelli,32
Luka Doncic,Luka Doncic,20
David West,David West,38
Tony Parker,Tony Parker,36
Danny Green,Danny Green,31
Rudy Gay,Rudy Gay,32
LaMarcus Aldridge,LaMarcus Aldridge,33
Tim Duncan,Tim Duncan,42
Kevin Durant,Kevin Durant,30
Stephen Curry,Stephen Curry,31
Ray Allen,Ray Allen,43
Tiago Splitter,Tiago Splitter,34
DeAndre Jordan,DeAndre Jordan,30
Paul Gasol,Paul Gasol,38
Aron Baynes,Aron Baynes,32
Cory Joseph,Cory Joseph,27
Vince Carter,Vince Carter,42
Marc Gasol,Marc Gasol,34
Ricky Rubio,Ricky Rubio,28
Ben Simmons,Ben Simmons,22
Giannis Antetokounmpo,Giannis Antetokounmpo,24
Rajon Rondo,Rajon Rondo,33
Manu Ginobili,Manu Ginobili,41
Kyrie Irving,Kyrie Irving,26
Carmelo Anthony,Carmelo Anthony,34
Dwyane Wade,Dwyane Wade,37
Joel Embiid,Joel Embiid,25
Damian Lillard,Damian Lillard,28
Yao Ming,Yao Ming,38
Kyle Anderson,Kyle Anderson,25
Dejounte Murray,Dejounte Murray,29
Blake Griffin,Blake Griffin,30
Steve Nash,Steve Nash,45
Jason Kidd,Jason Kidd,45
Dirk Nowitzki,Dirk Nowitzki,40
Paul George,Paul George,28
Grant Hill,Grant Hill,46
Shaquile O'Neal,Shaquile O'Neal,47
JaVale McGee,JaVale McGee,31
Dwight Howard,Dwight Howard,33
DROP SPACE IF EXISTS nba;
CREATE SPACE nba(partition_num=7, replica_factor=1, vid_type=FIXED_STRING(30));
USE nba;
CREATE TAG IF NOT EXISTS player(name string, age int);
CREATE TAG IF NOT EXISTS team(name string);
CREATE TAG IF NOT EXISTS bachelor(name string, speciality string);
CREATE EDGE IF NOT EXISTS like(likeness int);
CREATE EDGE IF NOT EXISTS serve(start_year int, end_year int);
CREATE EDGE IF NOT EXISTS teammate(start_year int, end_year int);
CREATE TAG INDEX IF NOT EXISTS player_name_index ON player(name(64));
CREATE TAG INDEX IF NOT EXISTS player_age_index ON player(age);
CREATE TAG INDEX IF NOT EXISTS team_name_index ON team(name(64));
:SRC_VID(string),:DST_VID(string),:RANK,serve.start_year:int,serve.end_year:int
Amar'e Stoudemire,Suns,0,2002,2010
Amar'e Stoudemire,Knicks,0,2010,2015
Amar'e Stoudemire,Heat,0,2015,2016
Russell Westbrook,Thunders,0,2008,2019
James Harden,Thunders,0,2009,2012
James Harden,Rockets,0,2012,2019
Kobe Bryant,Lakers,0,1996,2016
Tracy McGrady,Raptors,0,1997,2000
Tracy McGrady,Magic,0,2000,2004
Tracy McGrady,Rockets,0,2004,2010
Tracy McGrady,Spurs,0,2013,2013
Chris Paul,Hornets,0,2005,2011
Chris Paul,Clippers,0,2011,2017
Chris Paul,Rockets,0,2017,2021
Boris Diaw,Hawks,0,2003,2005
Boris Diaw,Suns,0,2005,2008
Boris Diaw,Hornets,0,2008,2012
Boris Diaw,Spurs,0,2012,2016
Boris Diaw,Jazz,0,2016,2017
LeBron James,Cavaliers,0,2003,2010
LeBron James,Heat,0,2010,2014
LeBron James,Cavaliers,1,2014,2018
LeBron James,Lakers,0,2018,2019
Klay Thompson,Warriors,0,2011,2019
Kristaps Porzingis,Knicks,0,2015,2019
Kristaps Porzingis,Mavericks,0,2019,2020
Jonathon Simmons,Spurs,0,2015,2017
Jonathon Simmons,Magic,0,2017,2019
Jonathon Simmons,76ers,0,2019,2019
Marco Belinelli,Warriors,0,2007,2009
Marco Belinelli,Raptors,0,2009,2010
Marco Belinelli,Hornets,0,2010,2012
Marco Belinelli,Bulls,0,2012,2013
Marco Belinelli,Spurs,0,2013,2015
Marco Belinelli,Kings,0,2015,2016
Marco Belinelli,Hornets,1,2016,2017
Marco Belinelli,Hawks,0,2017,2018
Marco Belinelli,76ers,0,2018,2018
Marco Belinelli,Spurs,1,2018,2019
Luka Doncic,Mavericks,0,2018,2019
David West,Hornets,0,2003,2011
David West,Pacers,0,2011,2015
David West,Spurs,0,2015,2016
David West,Warriors,0,2016,2018
Tony Parker,Spurs,0,1999,2018
Tony Parker,Hornets,0,2018,2019
Danny Green,Cavaliers,0,2009,2010
Danny Green,Spurs,0,2010,2018
Danny Green,Raptors,0,2018,2019
Rudy Gay,Grizzlies,0,2006,2013
Rudy Gay,Raptors,0,2013,2013
Rudy Gay,Kings,0,2013,2017
Rudy Gay,Spurs,0,2017,2019
LaMarcus Aldridge,Trail Blazers,0,2006,2015
LaMarcus Aldridge,Spurs,0,2015,2019
Tim Duncan,Spurs,0,1997,2016
Kevin Durant,Thunders,0,2007,2016
Kevin Durant,Warriors,0,2016,2019
Stephen Curry,Warriors,0,2009,2019
Ray Allen,Bucks,0,1996,2003
Ray Allen,Thunders,0,2003,2007
Ray Allen,Celtics,0,2007,2012
Ray Allen,Heat,0,2012,2014
Tiago Splitter,Spurs,0,2010,2015
Tiago Splitter,Hawks,0,2015,2017
Tiago Splitter,76ers,0,2017,2017
DeAndre Jordan,Clippers,0,2008,2018
DeAndre Jordan,Mavericks,0,2018,2019
DeAndre Jordan,Knicks,0,2019,2019
Paul Gasol,Grizzlies,0,2001,2008
Paul Gasol,Lakers,0,2008,2014
Paul Gasol,Bulls,0,2014,2016
Paul Gasol,Spurs,0,2016,2019
Paul Gasol,Bucks,0,2019,2020
Aron Baynes,Spurs,0,2013,2015
Aron Baynes,Pistons,0,2015,2017
Aron Baynes,Celtics,0,2017,2019
Cory Joseph,Spurs,0,2011,2015
Cory Joseph,Raptors,0,2015,2017
Cory Joseph,Pacers,0,2017,2019
Vince Carter,Raptors,0,1998,2004
Vince Carter,Nets,0,2004,2009
Vince Carter,Magic,0,2009,2010
Vince Carter,Suns,0,2010,2011
Vince Carter,Mavericks,0,2011,2014
Vince Carter,Grizzlies,0,2014,2017
Vince Carter,Kings,0,2017,2018
Vince Carter,Hawks,0,2018,2019
Marc Gasol,Grizzlies,0,2008,2019
Marc Gasol,Raptors,0,2019,2019
Ricky Rubio,Timberwolves,0,2011,2017
Ricky Rubio,Jazz,0,2017,2019
Ben Simmons,76ers,0,2016,2019
Giannis Antetokounmpo,Bucks,0,2013,2019
Rajon Rondo,Celtics,0,2006,2014
Rajon Rondo,Mavericks,0,2014,2015
Rajon Rondo,Kings,0,2015,2016
Rajon Rondo,Bulls,0,2016,2017
Rajon Rondo,Pelicans,0,2017,2018
Rajon Rondo,Lakers,0,2018,2019
Manu Ginobili,Spurs,0,2002,2018
Kyrie Irving,Cavaliers,0,2011,2017
Kyrie Irving,Celtics,0,2017,2019
Carmelo Anthony,Nuggets,0,2003,2011
Carmelo Anthony,Knicks,0,2011,2017
Carmelo Anthony,Thunders,0,2017,2018
Carmelo Anthony,Rockets,0,2018,2019
Dwyane Wade,Heat,0,2003,2016
Dwyane Wade,Bulls,0,2016,2017
Dwyane Wade,Cavaliers,0,2017,2018
Dwyane Wade,Heat,1,2018,2019
Joel Embiid,76ers,0,2014,2019
Damian Lillard,Trail Blazers,0,2012,2019
Yao Ming,Rockets,0,2002,2011
Kyle Anderson,Spurs,0,2014,2018
Kyle Anderson,Grizzlies,0,2018,2019
Dejounte Murray,Spurs,0,2016,2019
Blake Griffin,Clippers,0,2009,2018
Blake Griffin,Pistons,0,2018,2019
Steve Nash,Suns,0,1996,1998
Steve Nash,Mavericks,0,1998,2004
Steve Nash,Suns,1,2004,2012
Steve Nash,Lakers,0,2012,2015
Jason Kidd,Mavericks,0,1994,1996
Jason Kidd,Suns,0,1996,2001
Jason Kidd,Nets,0,2001,2008
Jason Kidd,Mavericks,1,2008,2012
Jason Kidd,Knicks,0,2012,2013
Dirk Nowitzki,Mavericks,0,1998,2019
Paul George,Pacers,0,2010,2017
Paul George,Thunders,0,2017,2019
Grant Hill,Pistons,0,1994,2000
Grant Hill,Magic,0,2000,2007
Grant Hill,Suns,0,2007,2012
Grant Hill,Clippers,0,2012,2013
Shaquile O'Neal,Magic,0,1992,1996
Shaquile O'Neal,Lakers,0,1996,2004
Shaquile O'Neal,Heat,0,2004,2008
Shaquile O'Neal,Suns,0,2008,2009
Shaquile O'Neal,Cavaliers,0,2009,2010
Shaquile O'Neal,Celtics,0,2010,2011
JaVale McGee,Wizards,0,2008,2012
JaVale McGee,Nuggets,0,2012,2015
JaVale McGee,Mavericks,0,2015,2016
JaVale McGee,Warriors,0,2016,2018
JaVale McGee,Lakers,0,2018,2019
Dwight Howard,Magic,0,2004,2012
Dwight Howard,Lakers,0,2012,2013
Dwight Howard,Rockets,0,2013,2016
Dwight Howard,Hawks,0,2016,2017
Dwight Howard,Hornets,0,2017,2018
Dwight Howard,Wizards,0,2018,2019
:VID(string),team.name:string
Nets,Nets
Pistons,Pistons
Bucks,Bucks
Mavericks,Mavericks
Clippers,Clippers
Thunders,Thunders
Lakers,Lakers
Jazz,Jazz
Nuggets,Nuggets
Wizards,Wizards
Pacers,Pacers
Timberwolves,Timberwolves
Hawks,Hawks
Warriors,Warriors
Magic,Magic
Rockets,Rockets
Pelicans,Pelicans
Raptors,Raptors
Spurs,Spurs
Heat,Heat
Grizzlies,Grizzlies
Knicks,Knicks
Suns,Suns
Hornets,Hornets
Cavaliers,Cavaliers
Kings,Kings
Celtics,Celtics
76ers,76ers
Trail Blazers,Trail Blazers
Bulls,Bulls
:SRC_VID(string),:DST_VID(string),teammate.start_year:int,teammate.end_year:int
Tony Parker,Tim Duncan,2001,2016
Tony Parker,Manu Ginobili,2002,2018
Tony Parker,LaMarcus Aldridge,2015,2018
Tony Parker,Kyle Anderson,2014,2016
Tim Duncan,Tony Parker,2001,2016
Tim Duncan,Manu Ginobili,2002,2016
Tim Duncan,LaMarcus Aldridge,2015,2016
Tim Duncan,Danny Green,2010,2016
Manu Ginobili,Tim Duncan,2002,2016
Manu Ginobili,Tony Parker,2002,2016
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment