From ec5e8ee9cfc58a667b3b87ac40bdd61df122801e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 2 Jan 2023 13:31:19 +0000 Subject: [PATCH] Done --- .pylintrc | 4 + tests/integration/test_library_bridge/test.py | 62 +--------- .../test_library_bridge/test_exiled.py | 97 ++++++++++++++++ .../integration/test_odbc_interaction/test.py | 55 +-------- .../test_odbc_interaction/test_exiled.py | 106 ++++++++++++++++++ 5 files changed, 213 insertions(+), 111 deletions(-) create mode 100644 tests/integration/test_library_bridge/test_exiled.py create mode 100644 tests/integration/test_odbc_interaction/test_exiled.py diff --git a/.pylintrc b/.pylintrc index b672cbfdfad..3fd991a53b8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -12,6 +12,10 @@ max-statements=200 [FORMAT] ignore-long-lines = (# )??$ +[LOGGING] +# Not to warn about lazy formatting in logging functions +logging-format-style=fstr + [MESSAGES CONTROL] disable = missing-docstring, too-few-public-methods, diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index e50c89f9567..a4dca545d44 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -16,9 +16,9 @@ instance = cluster.add_instance( ) -def create_dict_simple(): - instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") - instance.query( +def create_dict_simple(ch_instance): + ch_instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") + ch_instance.query( """ CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) @@ -242,7 +242,7 @@ def test_recover_after_bridge_crash(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - create_dict_simple() + create_dict_simple(instance) result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(0));""") assert result.strip() == "100" @@ -269,7 +269,7 @@ def test_server_restart_bridge_might_be_stil_alive(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - create_dict_simple() + create_dict_simple(instance) result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") assert result.strip() == "101" @@ -290,58 +290,6 @@ def test_server_restart_bridge_might_be_stil_alive(ch_cluster): instance.query("DROP DICTIONARY lib_dict_c") -def test_bridge_dies_with_parent(ch_cluster): - if instance.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - if instance.is_built_with_address_sanitizer(): - pytest.skip( - "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" - ) - - create_dict_simple() - result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") - assert result.strip() == "101" - - clickhouse_pid = instance.get_process_pid("clickhouse server") - bridge_pid = instance.get_process_pid("library-bridge") - assert clickhouse_pid is not None - assert bridge_pid is not None - - try: - instance.exec_in_container( - ["kill", str(clickhouse_pid)], privileged=True, user="root" - ) - except: - pass - - for i in range(30): - time.sleep(1) - clickhouse_pid = instance.get_process_pid("clickhouse server") - if clickhouse_pid is None: - break - - for i in range(30): - time.sleep(1) - bridge_pid = instance.get_process_pid("library-bridge") - if bridge_pid is None: - break - - if bridge_pid: - out = instance.exec_in_container( - ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], - privileged=True, - user="root", - ) - logging.debug(f"Bridge is running, gdb output:\n{out}") - - try: - assert clickhouse_pid is None - assert bridge_pid is None - finally: - instance.start_clickhouse(20) - instance.query("DROP DICTIONARY lib_dict_c") - - def test_path_validation(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") diff --git a/tests/integration/test_library_bridge/test_exiled.py b/tests/integration/test_library_bridge/test_exiled.py new file mode 100644 index 00000000000..1a7ead8d50b --- /dev/null +++ b/tests/integration/test_library_bridge/test_exiled.py @@ -0,0 +1,97 @@ +import os +import os.path as p +import pytest +import time +import logging + +from helpers.cluster import ClickHouseCluster, run_and_check +from test_library_bridge.test import create_dict_simple + +cluster = ClickHouseCluster(__file__) + +instance = cluster.add_instance( + "instance", + dictionaries=["configs/dictionaries/dict1.xml"], + main_configs=["configs/config.d/config.xml"], + stay_alive=True, +) + +@pytest.fixture(scope="module") +def ch_cluster(): + try: + cluster.start() + instance.query("CREATE DATABASE test") + + instance.copy_file_to_container( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs/dict_lib.cpp" + ), + "/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp", + ) + + instance.query("SYSTEM RELOAD CONFIG") + + instance.exec_in_container( + [ + "bash", + "-c", + "/usr/bin/g++ -shared -o /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp", + ], + user="root", + ) + yield cluster + + finally: + cluster.shutdown() + + +def test_bridge_dies_with_parent(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + if instance.is_built_with_address_sanitizer(): + pytest.skip( + "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" + ) + + create_dict_simple(instance) + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" + + clickhouse_pid = instance.get_process_pid("clickhouse server") + bridge_pid = instance.get_process_pid("library-bridge") + assert clickhouse_pid is not None + assert bridge_pid is not None + + try: + instance.exec_in_container( + ["kill", str(clickhouse_pid)], privileged=True, user="root" + ) + except: + pass + + for i in range(30): + time.sleep(1) + clickhouse_pid = instance.get_process_pid("clickhouse server") + if clickhouse_pid is None: + break + + for i in range(30): + time.sleep(1) + bridge_pid = instance.get_process_pid("library-bridge") + if bridge_pid is None: + break + + if bridge_pid: + out = instance.exec_in_container( + ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, + user="root", + ) + logging.debug(f"Bridge is running, gdb output:\n{out}") + + try: + assert clickhouse_pid is None + assert bridge_pid is None + finally: + instance.start_clickhouse(20) + instance.query("DROP DICTIONARY lib_dict_c") diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 743e4ecd68a..e2cf4bdab8a 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -22,8 +22,7 @@ node1 = cluster.add_instance( "configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml", "configs/dictionaries/sqlite3_odbc_cached_dictionary.xml", "configs/dictionaries/postgres_odbc_hashed_dictionary.xml", - ], - stay_alive=True, + ] ) @@ -617,58 +616,6 @@ def test_postgres_insert(started_cluster): ) -def test_bridge_dies_with_parent(started_cluster): - skip_test_msan(node1) - - if node1.is_built_with_address_sanitizer(): - # TODO: Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge in this test and - # that's linked somehow with that we have replaced getauxval() in glibc-compatibility. - # The leak sanitizer calls getauxval() for its own purposes, and our replaced version doesn't seem to be equivalent in that case. - pytest.skip( - "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" - ) - - node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") - - clickhouse_pid = node1.get_process_pid("clickhouse server") - bridge_pid = node1.get_process_pid("odbc-bridge") - assert clickhouse_pid is not None - assert bridge_pid is not None - - try: - node1.exec_in_container( - ["kill", str(clickhouse_pid)], privileged=True, user="root" - ) - except: - pass - - for i in range(30): - time.sleep(1) - clickhouse_pid = node1.get_process_pid("clickhouse server") - if clickhouse_pid is None: - break - - for i in range(30): - time.sleep(1) # just for sure, that odbc-bridge caught signal - bridge_pid = node1.get_process_pid("odbc-bridge") - if bridge_pid is None: - break - - if bridge_pid: - out = node1.exec_in_container( - ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], - privileged=True, - user="root", - ) - logging.debug(f"Bridge is running, gdb output:\n{out}") - - try: - assert clickhouse_pid is None - assert bridge_pid is None - finally: - node1.start_clickhouse(20) - - def test_odbc_postgres_date_data_type(started_cluster): skip_test_msan(node1) diff --git a/tests/integration/test_odbc_interaction/test_exiled.py b/tests/integration/test_odbc_interaction/test_exiled.py new file mode 100644 index 00000000000..b0174a5b8de --- /dev/null +++ b/tests/integration/test_odbc_interaction/test_exiled.py @@ -0,0 +1,106 @@ +import time +import logging +import pytest + +from helpers.cluster import ClickHouseCluster, assert_eq_with_retry +from test_odbc_interaction.test import create_mysql_db, create_mysql_table, get_mysql_conn, skip_test_msan + + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", + with_odbc_drivers=True, + main_configs=["configs/openssl.xml", "configs/odbc_logging.xml"], + stay_alive=True, + dictionaries=["configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] + logging.debug(f"sqlite data received: {sqlite_db}") + node1.exec_in_container( + [ + "sqlite3", + sqlite_db, + "CREATE TABLE t2(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);", + ], + privileged=True, + user="root", + ) + + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) + + node1.query("SYSTEM RELOAD DICTIONARY sqlite3_odbc_hashed") + + yield cluster + except Exception as ex: + logging.exception(ex) + raise ex + finally: + cluster.shutdown() + + +# This test kills ClickHouse server and ODBC bridge and in worst scenario +# may cause group test crashes. Thus, this test is executed in a separate "module" +# with separate environment. +def test_bridge_dies_with_parent(started_cluster): + skip_test_msan(node1) + + if node1.is_built_with_address_sanitizer(): + # TODO: Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge in this test and + # that's linked somehow with that we have replaced getauxval() in glibc-compatibility. + # The leak sanitizer calls getauxval() for its own purposes, and our replaced version doesn't seem to be equivalent in that case. + pytest.skip( + "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" + ) + + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3" + ) + + clickhouse_pid = node1.get_process_pid("clickhouse server") + bridge_pid = node1.get_process_pid("odbc-bridge") + assert clickhouse_pid is not None + assert bridge_pid is not None + + try: + node1.exec_in_container( + ["kill", str(clickhouse_pid)], privileged=True, user="root" + ) + except: + pass + + for _ in range(30): + time.sleep(1) + clickhouse_pid = node1.get_process_pid("clickhouse server") + if clickhouse_pid is None: + break + + for _ in range(30): + time.sleep(1) # just for sure, that odbc-bridge caught signal + bridge_pid = node1.get_process_pid("odbc-bridge") + if bridge_pid is None: + break + + if bridge_pid: + out = node1.exec_in_container( + ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, + user="root", + ) + logging.debug(f"Bridge is running, gdb output:\n{out}") + + try: + assert clickhouse_pid is None + assert bridge_pid is None + finally: + node1.start_clickhouse(20)