mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-17 03:42:48 +00:00
1277 lines
45 KiB
Python
1277 lines
45 KiB
Python
import datetime
|
|
import json
|
|
|
|
import bson
|
|
import pymongo
|
|
import pytest
|
|
|
|
from helpers.client import QueryRuntimeException
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster(request):
|
|
try:
|
|
cluster = ClickHouseCluster(__file__)
|
|
cluster.add_instance(
|
|
"node",
|
|
main_configs=[
|
|
"configs/named_collections.xml",
|
|
],
|
|
user_configs=["configs/users.xml"],
|
|
with_mongo=True,
|
|
)
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def get_mongo_connection(started_cluster, secure=False, with_credentials=True):
|
|
if secure:
|
|
return pymongo.MongoClient(
|
|
"mongodb://root:clickhouse@localhost:{}/?tls=true&tlsAllowInvalidCertificates=true&tlsAllowInvalidHostnames=true".format(
|
|
started_cluster.mongo_secure_port
|
|
)
|
|
)
|
|
if with_credentials:
|
|
return pymongo.MongoClient(
|
|
"mongodb://root:clickhouse@localhost:{}".format(started_cluster.mongo_port)
|
|
)
|
|
|
|
return pymongo.MongoClient(
|
|
"mongodb://localhost:{}".format(started_cluster.mongo_no_cred_port)
|
|
)
|
|
|
|
|
|
def test_simple_select(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM simple_mongo_table")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from simple_mongo_table where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE simple_mongo_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_simple_select_uri(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table_uri"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_table_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://root:clickhouse@mongo1:27017/test', 'simple_table_uri')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM simple_table_uri") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM simple_table_uri")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from simple_table_uri where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE simple_table_uri")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_simple_select_from_view(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
simple_mongo_table_view = db.create_collection(
|
|
"simple_table_view", viewOn="simple_table"
|
|
)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table_view', 'root', 'clickhouse')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM simple_mongo_table")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from simple_mongo_table where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE simple_mongo_table")
|
|
simple_mongo_table_view.drop()
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_arrays(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
arrays_mongo_table = db["arrays_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append(
|
|
{
|
|
"key": i,
|
|
"arr_int64": [-(i + 1), -(i + 2), -(i + 3)],
|
|
"arr_int32": [-(i + 1), -(i + 2), -(i + 3)],
|
|
"arr_int16": [-(i + 1), -(i + 2), -(i + 3)],
|
|
"arr_int8": [-(i + 1), -(i + 2), -(i + 3)],
|
|
"arr_uint64": [i + 1, i + 2, i + 3],
|
|
"arr_uint32": [i + 1, i + 2, i + 3],
|
|
"arr_uint16": [i + 1, i + 2, i + 3],
|
|
"arr_uint8": [i + 1, i + 2, i + 3],
|
|
"arr_float32": [i + 1.125, i + 2.5, i + 3.750],
|
|
"arr_float64": [i + 1.125, i + 2.5, i + 3.750],
|
|
"arr_date": [
|
|
datetime.datetime(2002, 10, 27),
|
|
datetime.datetime(2024, 1, 8, 23, 59, 59),
|
|
],
|
|
"arr_date32": [
|
|
datetime.datetime(2002, 10, 27),
|
|
datetime.datetime(2024, 1, 8, 23, 59, 59),
|
|
],
|
|
"arr_datetime": [
|
|
datetime.datetime(2023, 3, 31, 6, 3, 12),
|
|
datetime.datetime(1999, 2, 28, 12, 46, 34),
|
|
],
|
|
"arr_datetime64": [
|
|
datetime.datetime(2023, 3, 31, 6, 3, 12),
|
|
datetime.datetime(1999, 2, 28, 12, 46, 34),
|
|
],
|
|
"arr_string": [str(i + 1), str(i + 2), str(i + 3)],
|
|
"arr_uuid": [
|
|
"f0e77736-91d1-48ce-8f01-15123ca1c7ed",
|
|
"93376a07-c044-4281-a76e-ad27cf6973c5",
|
|
],
|
|
"arr_arr_bool": [
|
|
[True, False, True],
|
|
[True],
|
|
[None],
|
|
[False],
|
|
],
|
|
"arr_arr_bool_nullable": [
|
|
[True, False, True],
|
|
[True, None],
|
|
[None],
|
|
[False],
|
|
],
|
|
"arr_empty": [],
|
|
"arr_null": None,
|
|
"arr_nullable": None,
|
|
}
|
|
)
|
|
|
|
arrays_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE arrays_mongo_table("
|
|
"key UInt64,"
|
|
"arr_int64 Array(Int64),"
|
|
"arr_int32 Array(Int32),"
|
|
"arr_int16 Array(Int16),"
|
|
"arr_int8 Array(Int8),"
|
|
"arr_uint64 Array(UInt64),"
|
|
"arr_uint32 Array(UInt32),"
|
|
"arr_uint16 Array(UInt16),"
|
|
"arr_uint8 Array(UInt8),"
|
|
"arr_float32 Array(Float32),"
|
|
"arr_float64 Array(Float64),"
|
|
"arr_date Array(Date),"
|
|
"arr_date32 Array(Date32),"
|
|
"arr_datetime Array(DateTime),"
|
|
"arr_datetime64 Array(DateTime64),"
|
|
"arr_string Array(String),"
|
|
"arr_uuid Array(UUID),"
|
|
"arr_arr_bool Array(Array(Bool)),"
|
|
"arr_arr_bool_nullable Array(Array(Nullable(Bool))),"
|
|
"arr_empty Array(UInt64),"
|
|
"arr_null Array(UInt64),"
|
|
"arr_arr_null Array(Array(UInt64)),"
|
|
"arr_nullable Array(Nullable(UInt64))"
|
|
") ENGINE = MongoDB('mongo1:27017', 'test', 'arrays_table', 'root', 'clickhouse')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM arrays_mongo_table") == "100\n"
|
|
|
|
for column_name in ["arr_int64", "arr_int32", "arr_int16", "arr_int8"]:
|
|
assert (
|
|
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
|
|
== "[-43,-44,-45]\n"
|
|
)
|
|
|
|
for column_name in ["arr_uint64", "arr_uint32", "arr_uint16", "arr_uint8"]:
|
|
assert (
|
|
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
|
|
== "[43,44,45]\n"
|
|
)
|
|
|
|
for column_name in ["arr_float32", "arr_float64"]:
|
|
assert (
|
|
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
|
|
== "[43.125,44.5,45.75]\n"
|
|
)
|
|
|
|
assert (
|
|
node.query(f"SELECT arr_date FROM arrays_mongo_table WHERE key = 42")
|
|
== "['2002-10-27','2024-01-08']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_date32 FROM arrays_mongo_table WHERE key = 42")
|
|
== "['2002-10-27','2024-01-08']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_datetime FROM arrays_mongo_table WHERE key = 42")
|
|
== "['2023-03-31 06:03:12','1999-02-28 12:46:34']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_datetime64 FROM arrays_mongo_table WHERE key = 42")
|
|
== "['2023-03-31 06:03:12.000','1999-02-28 12:46:34.000']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_string FROM arrays_mongo_table WHERE key = 42")
|
|
== "['43','44','45']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_uuid FROM arrays_mongo_table WHERE key = 42")
|
|
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42")
|
|
== "[[true,false,true],[true],[false],[false]]\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
f"SELECT arr_arr_bool_nullable FROM arrays_mongo_table WHERE key = 42"
|
|
)
|
|
== "[[true,false,true],[true,NULL],[NULL],[false]]\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_empty FROM arrays_mongo_table WHERE key = 42") == "[]\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_null FROM arrays_mongo_table WHERE key = 42") == "[]\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_arr_null FROM arrays_mongo_table WHERE key = 42")
|
|
== "[]\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT arr_nullable FROM arrays_mongo_table WHERE key = 42")
|
|
== "[]\n"
|
|
)
|
|
|
|
node.query("DROP TABLE arrays_mongo_table")
|
|
arrays_mongo_table.drop()
|
|
|
|
|
|
def test_complex_data_type(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
incomplete_mongo_table = db["complex_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i), "dict": {"a": i, "b": str(i)}})
|
|
incomplete_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM incomplete_mongo_table")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from incomplete_mongo_table where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE incomplete_mongo_table")
|
|
incomplete_mongo_table.drop()
|
|
|
|
|
|
def test_secure_connection(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, secure=True)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo_secure:27017', 'test', 'simple_table', 'root', 'clickhouse', 'tls=true&tlsAllowInvalidCertificates=true&tlsAllowInvalidHostnames=true')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM simple_mongo_table")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from simple_mongo_table where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE simple_mongo_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_secure_connection_with_validation(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, secure=True)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo_secure:27017', 'test', 'simple_table', 'root', 'clickhouse', 'tls=true')"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT COUNT() FROM simple_mongo_table")
|
|
|
|
node.query("DROP TABLE simple_mongo_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_secure_connection_uri(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, secure=True)
|
|
db = mongo_connection["test"]
|
|
simple_mongo_table = db["test_secure_connection_uri"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE test_secure_connection_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://root:clickhouse@mongo_secure:27017/test?tls=true&tlsAllowInvalidCertificates=true&tlsAllowInvalidHostnames=true&authSource=admin', 'test_secure_connection_uri')"
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM test_secure_connection_uri") == "100\n"
|
|
assert (
|
|
node.query("SELECT sum(key) FROM test_secure_connection_uri")
|
|
== str(sum(range(0, 100))) + "\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT data from test_secure_connection_uri where key = 42")
|
|
== hex(42 * 42) + "\n"
|
|
)
|
|
|
|
node.query("DROP TABLE test_secure_connection_uri")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_secure_connection_uri_with_validation(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, secure=True)
|
|
db = mongo_connection["test"]
|
|
simple_mongo_table = db["test_secure_connection_uri"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE test_secure_connection_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://root:clickhouse@mongo_secure:27017/test?tls=true', 'test_secure_connection_uri')"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT COUNT() FROM test_secure_connection_uri")
|
|
|
|
node.query("DROP TABLE test_secure_connection_uri")
|
|
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_predefined_connection_configuration(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB(mongo1)"
|
|
)
|
|
|
|
assert node.query("SELECT count() FROM simple_mongo_table") == "100\n"
|
|
|
|
node.query("DROP TABLE simple_mongo_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_predefined_connection_configuration_uri(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table_uri"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_table_uri(key UInt64, data String) ENGINE = MongoDB(mongo1_uri)"
|
|
)
|
|
|
|
assert node.query("SELECT count() FROM simple_table_uri") == "100\n"
|
|
|
|
node.query("DROP TABLE simple_table_uri")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_no_credentials(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, with_credentials=False)
|
|
db = mongo_connection["test"]
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_table(key UInt64, data String) ENGINE = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', '', '')"
|
|
)
|
|
|
|
assert node.query("SELECT count() FROM simple_table") == "100\n"
|
|
|
|
node.query("DROP TABLE simple_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_no_credentials_uri(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, with_credentials=False)
|
|
db = mongo_connection["test"]
|
|
simple_mongo_table = db["simple_table_uri"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_table_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://mongo_no_cred:27017/test', 'simple_table_uri')"
|
|
)
|
|
|
|
assert node.query("SELECT count() FROM simple_table_uri") == "100\n"
|
|
|
|
node.query("DROP TABLE simple_table_uri")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_auth_source(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster, with_credentials=False)
|
|
admin_db = mongo_connection["admin"]
|
|
admin_db.command("dropAllUsersFromDatabase")
|
|
admin_db.command(
|
|
"createUser",
|
|
"root",
|
|
pwd="clickhouse",
|
|
roles=[{"role": "userAdminAnyDatabase", "db": "admin"}, "readWriteAnyDatabase"],
|
|
)
|
|
simple_mongo_table = admin_db["simple_table"]
|
|
data = []
|
|
for i in range(0, 50):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
db = mongo_connection["test"]
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 100):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table_fail(key UInt64, data String) ENGINE = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', 'root', 'clickhouse')"
|
|
)
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT count() FROM simple_mongo_table_fail")
|
|
|
|
node.query(
|
|
"CREATE OR REPLACE TABLE simple_mongo_table_ok(key UInt64, data String) ENGINE = MongoDB('mongo_no_cred:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')"
|
|
)
|
|
assert node.query("SELECT count() FROM simple_mongo_table_ok") == "100\n"
|
|
|
|
node.query("DROP TABLE simple_mongo_table_fail")
|
|
node.query("DROP TABLE simple_mongo_table_ok")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_missing_columns(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
simple_mongo_table = db["simple_table"]
|
|
data = []
|
|
for i in range(0, 10):
|
|
data.append({"key": i, "data": hex(i * i)})
|
|
for i in range(0, 10):
|
|
data.append({"key": i})
|
|
simple_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""CREATE OR REPLACE TABLE simple_mongo_table(
|
|
key UInt64,
|
|
data Nullable(String),
|
|
not_exists Int64,
|
|
not_exists_nullable Nullable(Int64)
|
|
) ENGINE = MongoDB(mongo1)"""
|
|
)
|
|
|
|
assert (
|
|
node.query("SELECT count() FROM simple_mongo_table WHERE isNull(data)")
|
|
== "10\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT count() FROM simple_mongo_table WHERE isNull(not_exists)")
|
|
== "0\n"
|
|
)
|
|
|
|
node.query("DROP TABLE IF EXISTS simple_mongo_table")
|
|
simple_mongo_table.drop()
|
|
|
|
|
|
def test_string_casting(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
string_mongo_table = db["strings_table"]
|
|
data = {
|
|
"k_boolT": True,
|
|
"k_boolF": False,
|
|
"k_int32P": 100,
|
|
"k_int32N": -100,
|
|
"k_int64P": bson.int64.Int64(100),
|
|
"k_int64N": bson.int64.Int64(-100),
|
|
"k_doubleP": 6.66,
|
|
"k_doubleN": -6.66,
|
|
"k_date": datetime.datetime(1999, 2, 28, 0, 0, 0),
|
|
"k_timestamp": datetime.datetime(1999, 2, 28, 12, 46, 34),
|
|
"k_string": "ClickHouse",
|
|
"k_document": {
|
|
"Hello": "world!",
|
|
"meow123": True,
|
|
"number": 321,
|
|
"doc": {"Hello": "world!"},
|
|
"arr": [{"Hello": "world!"}, 1, "c"],
|
|
},
|
|
"k_array": [
|
|
"Hello",
|
|
"world!",
|
|
{"cat": "meow!"},
|
|
[1, 2, 3],
|
|
],
|
|
}
|
|
string_mongo_table.insert_one(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""CREATE OR REPLACE TABLE strings_table (
|
|
_id String,
|
|
k_boolT String,
|
|
k_boolF String,
|
|
k_int32P String,
|
|
k_int32N String,
|
|
k_int64P String,
|
|
k_int64N String,
|
|
k_doubleP String,
|
|
k_doubleN String,
|
|
k_date String,
|
|
k_timestamp String,
|
|
k_string String,
|
|
k_document String,
|
|
k_array String
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'strings_table', 'root', 'clickhouse')"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM strings_table") == "1\n"
|
|
assert node.query("SELECT _id FROM strings_table") != ""
|
|
assert node.query("SELECT k_boolT FROM strings_table") == "true\n"
|
|
assert node.query("SELECT k_boolF FROM strings_table") == "false\n"
|
|
assert node.query("SELECT k_int32P FROM strings_table") == "100\n"
|
|
assert node.query("SELECT k_int32N FROM strings_table") == "-100\n"
|
|
assert node.query("SELECT k_int64P FROM strings_table") == "100\n"
|
|
assert node.query("SELECT k_int64N FROM strings_table") == "-100\n"
|
|
assert node.query("SELECT k_doubleP FROM strings_table") == "6.660000\n"
|
|
assert node.query("SELECT k_doubleN FROM strings_table") == "-6.660000\n"
|
|
assert node.query("SELECT k_date FROM strings_table") == "1999-02-28 00:00:00\n"
|
|
assert (
|
|
node.query("SELECT k_timestamp FROM strings_table") == "1999-02-28 12:46:34\n"
|
|
)
|
|
assert node.query("SELECT k_string FROM strings_table") == "ClickHouse\n"
|
|
assert json.dumps(
|
|
json.loads(node.query("SELECT k_document FROM strings_table"))
|
|
) == json.dumps(data["k_document"])
|
|
assert json.dumps(
|
|
json.loads(node.query("SELECT k_array FROM strings_table"))
|
|
) == json.dumps(data["k_array"])
|
|
|
|
node.query("DROP TABLE strings_table")
|
|
string_mongo_table.drop()
|
|
|
|
|
|
def test_dates_casting(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
dates_mongo_table = db["dates_table"]
|
|
data = {
|
|
"k_dateTime": datetime.datetime(1999, 2, 28, 11, 23, 16),
|
|
"k_dateTime64": datetime.datetime(1999, 2, 28, 11, 23, 16),
|
|
"k_date": datetime.datetime(1999, 2, 28, 11, 23, 16),
|
|
"k_date32": datetime.datetime(1999, 2, 28, 11, 23, 16),
|
|
}
|
|
dates_mongo_table.insert_one(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""CREATE OR REPLACE TABLE dates_table (
|
|
k_dateTime DateTime,
|
|
k_dateTime64 DateTime64,
|
|
k_date Date,
|
|
k_date32 Date32
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'dates_table', 'root', 'clickhouse')"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM dates_table") == "1\n"
|
|
assert (
|
|
node.query("SELECT * FROM dates_table")
|
|
== "1999-02-28 11:23:16\t1999-02-28 11:23:16.000\t1999-02-28\t1999-02-28\n"
|
|
)
|
|
|
|
node.query("DROP TABLE dates_table")
|
|
dates_mongo_table.drop()
|
|
|
|
|
|
def test_order_by(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
sort_mongo_table = db["sort_table"]
|
|
data = []
|
|
for i in range(1, 31):
|
|
for d in range(1, 31):
|
|
data.append(
|
|
{
|
|
"keyInt": i,
|
|
"keyFloat": i + (d * 0.001),
|
|
"keyDateTime": datetime.datetime(1999, 12, i, 11, 23, 16),
|
|
"keyDate": datetime.datetime(1999, 12, i, 11, 23, 16),
|
|
}
|
|
)
|
|
sort_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""CREATE OR REPLACE TABLE sort_table (
|
|
keyInt Int,
|
|
keyFloat Float64,
|
|
keyDateTime DateTime,
|
|
keyDate Date
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'sort_table', 'root', 'clickhouse')"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM sort_table") == "900\n"
|
|
assert node.query("SELECT keyInt FROM sort_table ORDER BY keyInt LIMIT 1") == "1\n"
|
|
assert (
|
|
node.query("SELECT keyInt FROM sort_table ORDER BY keyInt DESC LIMIT 1")
|
|
== "30\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT keyInt, keyFloat FROM sort_table ORDER BY keyInt, keyFloat DESC LIMIT 1"
|
|
)
|
|
== "1\t1.03\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT keyDateTime FROM sort_table ORDER BY keyDateTime DESC LIMIT 1"
|
|
)
|
|
== "1999-12-30 11:23:16\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT keyDate FROM sort_table ORDER BY keyDate DESC LIMIT 1")
|
|
== "1999-12-30\n"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM sort_table ORDER BY keyInt WITH FILL")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM sort_table ORDER BY keyInt WITH FILL TO sort_table")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM sort_table ORDER BY keyInt WITH FILL FROM sort_table")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM sort_table ORDER BY keyInt WITH FILL STEP 1")
|
|
|
|
node.query("DROP TABLE sort_table")
|
|
sort_mongo_table.drop()
|
|
|
|
|
|
def test_where(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
where_mongo_table = db["where_table"]
|
|
data = []
|
|
for i in range(1, 3):
|
|
for d in range(1, 3):
|
|
data.append(
|
|
{
|
|
"id": str(i) + str(d),
|
|
"keyInt": i,
|
|
"keyFloat": i + (d * 0.001),
|
|
"keyString": str(d) + "string",
|
|
"keyDateTime": datetime.datetime(1999, d, i, 11, 23, 16),
|
|
"keyDate": datetime.datetime(1999, d, i, 11, 23, 16),
|
|
"keyNull": None,
|
|
}
|
|
)
|
|
where_mongo_table.insert_many(data)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""CREATE OR REPLACE TABLE where_table (
|
|
id String,
|
|
keyInt Int,
|
|
keyFloat Float64,
|
|
keyString String,
|
|
keyDateTime DateTime,
|
|
keyDate Date,
|
|
keyNull Nullable(UInt8),
|
|
keyNotExists Nullable(Int)
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'where_table', 'root', 'clickhouse')"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM where_table") == "4\n"
|
|
|
|
assert (
|
|
node.query("SELECT keyString FROM where_table WHERE id = '11'") == "1string\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT keyString FROM where_table WHERE id != '11' ORDER BY keyFloat"
|
|
)
|
|
== "2string\n1string\n2string\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT keyString FROM where_table WHERE id = '11' AND keyString = '1string'"
|
|
)
|
|
== "1string\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE keyInt = 1 AND keyFloat = 1.001")
|
|
== "11\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE keyInt = 0 OR keyFloat = 1.001")
|
|
== "11\n"
|
|
)
|
|
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE keyInt BETWEEN 1 AND 2")
|
|
== "11\n12\n21\n22\n"
|
|
)
|
|
assert node.query("SELECT id FROM where_table WHERE keyInt > 10") == ""
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE keyInt < 10.1 ORDER BY keyFloat")
|
|
== "11\n12\n21\n22\n"
|
|
)
|
|
|
|
assert node.query("SELECT id FROM where_table WHERE id IN ('11')") == "11\n"
|
|
assert node.query("SELECT id FROM where_table WHERE id IN ['11']") == "11\n"
|
|
assert node.query("SELECT id FROM where_table WHERE id IN ('11', 100)") == "11\n"
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id IN ('11', '22') ORDER BY keyFloat"
|
|
)
|
|
== "11\n22\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id IN ['11', '22'] ORDER BY keyFloat"
|
|
)
|
|
== "11\n22\n"
|
|
)
|
|
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id NOT IN ('11') ORDER BY keyFloat"
|
|
)
|
|
== "12\n21\n22\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id NOT IN ['11'] ORDER BY keyFloat"
|
|
)
|
|
== "12\n21\n22\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id NOT IN ('11', 100) ORDER BY keyFloat"
|
|
)
|
|
== "12\n21\n22\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE id NOT IN ('11') AND id IN ('12')")
|
|
== "12\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT id FROM where_table WHERE id NOT IN ['11'] AND id IN ('12')")
|
|
== "12\n"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
assert (
|
|
node.query(
|
|
"SELECT id FROM where_table WHERE id NOT IN ['11', 100] ORDER BY keyFloat"
|
|
)
|
|
== "12\n21\n22\n"
|
|
)
|
|
|
|
assert node.query("SELECT id FROM where_table WHERE keyDateTime > now()") == ""
|
|
assert (
|
|
node.query(
|
|
"SELECT keyInt FROM where_table WHERE keyDateTime < now() AND keyString = '1string' AND keyInt IS NOT NULL ORDER BY keyInt"
|
|
)
|
|
== "1\n2\n"
|
|
)
|
|
|
|
assert node.query("SELECT count() FROM where_table WHERE isNotNull(id)") == "4\n"
|
|
assert (
|
|
node.query("SELECT count() FROM where_table WHERE isNotNull(keyNull)") == "0\n"
|
|
)
|
|
assert node.query("SELECT count() FROM where_table WHERE isNull(keyNull)") == "4\n"
|
|
assert (
|
|
node.query("SELECT count() FROM where_table WHERE isNotNull(keyNotExists)")
|
|
== "0\n"
|
|
)
|
|
assert (
|
|
node.query("SELECT count() FROM where_table WHERE isNull(keyNotExists)")
|
|
== "4\n"
|
|
)
|
|
assert node.query("SELECT count() FROM where_table WHERE keyNotExists = 0") == "0\n"
|
|
assert (
|
|
node.query("SELECT count() FROM where_table WHERE keyNotExists != 0") == "0\n"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM where_table WHERE keyInt = keyFloat")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM where_table WHERE equals(keyInt, keyFloat)")
|
|
|
|
node.query("DROP TABLE where_table")
|
|
where_mongo_table.drop()
|
|
|
|
|
|
def test_defaults(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
defaults_mongo_table = db["defaults_table"]
|
|
defaults_mongo_table.insert_one({"key": "key"})
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE defaults_table(
|
|
_id String,
|
|
k_int64 Int64,
|
|
k_int32 Int32,
|
|
k_int16 Int16,
|
|
k_int8 Int8,
|
|
k_uint64 UInt64,
|
|
k_uint32 UInt32,
|
|
k_uint16 UInt16,
|
|
k_uint8 UInt8,
|
|
k_float32 Float32,
|
|
k_float64 Float64,
|
|
k_date Date,
|
|
k_date32 Date32,
|
|
k_datetime DateTime,
|
|
k_datetime64 DateTime64,
|
|
k_string String,
|
|
k_uuid UUID,
|
|
k_arr Array(Bool)
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'defaults_table', 'root', 'clickhouse')
|
|
"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM defaults_table") == "1\n"
|
|
|
|
assert (
|
|
node.query(
|
|
"SELECT k_int64, k_int32, k_int16, k_int8, k_uint64, k_uint32, k_uint16, k_uint8, k_float32, k_float64 FROM defaults_table"
|
|
)
|
|
== "0\t0\t0\t0\t0\t0\t0\t0\t0\t0\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT k_date, k_date32, k_datetime, k_datetime64, k_string, k_uuid, k_arr FROM defaults_table"
|
|
)
|
|
== "1970-01-01\t1900-01-01\t1970-01-01 00:00:00\t1970-01-01 00:00:00.000\t\t00000000-0000-0000-0000-000000000000\t[]\n"
|
|
)
|
|
|
|
node.query("DROP TABLE defaults_table")
|
|
defaults_mongo_table.drop()
|
|
|
|
|
|
def test_nulls(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
nulls_mongo_table = db["nulls_table"]
|
|
nulls_mongo_table.insert_one({"key": "key"})
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE nulls_table(
|
|
_id String,
|
|
k_int64 Nullable(Int64),
|
|
k_int32 Nullable(Int32),
|
|
k_int16 Nullable(Int16),
|
|
k_int8 Nullable(Int8),
|
|
k_uint64 Nullable(UInt64),
|
|
k_uint32 Nullable(UInt32),
|
|
k_uint16 Nullable(UInt16),
|
|
k_uint8 Nullable(UInt8),
|
|
k_float32 Nullable(Float32),
|
|
k_float64 Nullable(Float64),
|
|
k_date Nullable(Date),
|
|
k_date32 Nullable(Date32),
|
|
k_datetime Nullable(DateTime),
|
|
k_datetime64 Nullable(DateTime64),
|
|
k_string Nullable(String),
|
|
k_uuid Nullable(UUID)
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'nulls_table', 'root', 'clickhouse')
|
|
"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM nulls_table") == "1\n"
|
|
|
|
assert (
|
|
node.query(
|
|
"SELECT k_int64, k_int32, k_int16, k_int8, k_uint64, k_uint32, k_uint16, k_uint8, k_float32, k_float64 FROM nulls_table"
|
|
)
|
|
== "\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"SELECT k_date, k_date32, k_datetime, k_datetime64, k_string, k_uuid FROM nulls_table"
|
|
)
|
|
== "\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n"
|
|
)
|
|
|
|
node.query("DROP TABLE nulls_table")
|
|
nulls_mongo_table.drop()
|
|
|
|
|
|
def test_oid(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
oid_mongo_table = db["oid_table"]
|
|
inserted_result = oid_mongo_table.insert_many(
|
|
[
|
|
{"key": "a"},
|
|
{"key": "b"},
|
|
{"key": "c"},
|
|
{"key": "d"},
|
|
{"key": "e"},
|
|
]
|
|
)
|
|
oid = inserted_result.inserted_ids
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE oid_table(
|
|
_id String,
|
|
key String
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'oid_table', 'root', 'clickhouse')
|
|
"""
|
|
)
|
|
|
|
assert node.query("SELECT COUNT() FROM oid_table") == "5\n"
|
|
|
|
assert node.query(f"SELECT key FROM oid_table WHERE _id = '{oid[0]}'") == "a\n"
|
|
assert (
|
|
node.query(f"SELECT * FROM oid_table WHERE _id = '{oid[2]}'")
|
|
== f"{oid[2]}\tc\n"
|
|
)
|
|
assert node.query(f"SELECT COUNT() FROM oid_table WHERE _id != '{oid[0]}'") == "4\n"
|
|
|
|
assert (
|
|
node.query(
|
|
f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}', '{oid[1]}') ORDER BY key"
|
|
)
|
|
== "a\nb\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
f"SELECT key FROM oid_table WHERE _id in ['{oid[0]}', '{oid[1]}'] ORDER BY key"
|
|
)
|
|
== "a\nb\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}') ORDER BY key")
|
|
== "a\n"
|
|
)
|
|
assert (
|
|
node.query(f"SELECT key FROM oid_table WHERE _id in ['{oid[1]}'] ORDER BY key")
|
|
== "b\n"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM oid_table WHERE _id = 'invalidOID'")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM oid_table WHERE _id = 123123")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM oid_table WHERE _id in (123123, 123)")
|
|
|
|
node.query("DROP TABLE oid_table")
|
|
oid_mongo_table.drop()
|
|
|
|
|
|
def test_uuid(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
uuid_mongo_table = db["uuid_table"]
|
|
uuid_mongo_table.insert_many(
|
|
[
|
|
{"isValid": 0, "kUUID": "bad_uuid"},
|
|
{"isValid": 1, "kUUID": "f0e77736-91d1-48ce-8f01-15123ca1c7ed"},
|
|
]
|
|
)
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE uuid_table(
|
|
isValid UInt8,
|
|
kUUID UUID
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'uuid_table', 'root', 'clickhouse')
|
|
"""
|
|
)
|
|
|
|
assert (
|
|
node.query(f"SELECT kUUID FROM uuid_table WHERE isValid = 1")
|
|
== "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
|
|
)
|
|
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM uuid_table WHERE isValid = 0")
|
|
with pytest.raises(QueryRuntimeException):
|
|
node.query("SELECT * FROM uuid_table")
|
|
|
|
node.query("DROP TABLE uuid_table")
|
|
uuid_mongo_table.drop()
|
|
|
|
|
|
def test_no_fail_on_unsupported_clauses(started_cluster):
|
|
mongo_connection = get_mongo_connection(started_cluster)
|
|
db = mongo_connection["test"]
|
|
db.command("dropAllUsersFromDatabase")
|
|
db.command("createUser", "root", pwd="clickhouse", roles=["readWrite"])
|
|
unsupported_clauses_table = db["unsupported_clauses"]
|
|
|
|
node = started_cluster.instances["node"]
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE unsupported_clauses(
|
|
a UInt64,
|
|
b UInt64
|
|
) ENGINE = MongoDB('mongo1:27017', 'test', 'unsupported_clauses', 'root', 'clickhouse')
|
|
"""
|
|
)
|
|
|
|
node.query(
|
|
f"SELECT * FROM unsupported_clauses WHERE a > rand() SETTINGS mongodb_throw_on_unsupported_query = 0"
|
|
)
|
|
node.query(
|
|
f"SELECT * FROM unsupported_clauses WHERE a / 1000 > 0 SETTINGS mongodb_throw_on_unsupported_query = 0"
|
|
)
|
|
node.query(
|
|
f"SELECT * FROM unsupported_clauses WHERE toFloat64(a) < 6.66 > rand() SETTINGS mongodb_throw_on_unsupported_query = 0"
|
|
)
|
|
node.query(
|
|
f"SELECT * FROM unsupported_clauses ORDER BY a, b LIMIT 2 BY a SETTINGS mongodb_throw_on_unsupported_query = 0"
|
|
)
|
|
|
|
node.query("DROP TABLE unsupported_clauses")
|
|
unsupported_clauses_table.drop()
|
|
|
|
|
|
def test_password_masking(started_cluster):
|
|
node = started_cluster.instances["node"]
|
|
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE TABLE mongodb_uri_password_masking (_id String)
|
|
ENGINE = MongoDB('mongodb://testuser:mypassword@127.0.0.1:27017/example', 'test_clickhouse');
|
|
"""
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(create_table_query, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_uri_password_masking' AND database = currentDatabase();
|
|
"""
|
|
)
|
|
== "CREATE TABLE default.mongodb_uri_password_masking (`_id` String) ENGINE = MongoDB(\\'mongodb://testuser:[HIDDEN]@127.0.0.1:27017/example\\', \\'test_clickhouse\\')\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(engine_full, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_uri_password_masking' AND database = currentDatabase();
|
|
"""
|
|
)
|
|
== "MongoDB(\\'mongodb://testuser:[HIDDEN]@127.0.0.1:27017/example\\', \\'test_clickhouse\\')\n"
|
|
)
|
|
node.query("DROP TABLE IF EXISTS mongodb_uri_password_masking;")
|
|
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE DICTIONARY mongodb_dictionary_uri_password_masking (_id String)
|
|
PRIMARY KEY _id
|
|
SOURCE(MONGODB(uri 'mongodb://testuser:mypassword@127.0.0.1:27017/example' collection 'test_clickhouse'))
|
|
LAYOUT(FLAT())
|
|
LIFETIME(0);
|
|
"""
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(create_table_query, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_dictionary_uri_password_masking' AND database = currentDatabase();"""
|
|
)
|
|
== "CREATE DICTIONARY default.mongodb_dictionary_uri_password_masking (`_id` String) PRIMARY KEY _id SOURCE(MONGODB(URI \\'mongodb://testuser:[HIDDEN]@127.0.0.1:27017/example\\' COLLECTION \\'test_clickhouse\\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())\n"
|
|
)
|
|
node.query("DROP DICTIONARY IF EXISTS mongodb_dictionary_uri_password_masking;")
|
|
|
|
node.query(
|
|
"""
|
|
CREATE TABLE mongodb_password_masking (_id String)
|
|
ENGINE = MongoDB('127.0.0.1:27017', 'example', 'test_clickhouse', 'testuser', 'mypassword');
|
|
"""
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(create_table_query, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_password_masking' AND database = currentDatabase();
|
|
"""
|
|
)
|
|
== "CREATE TABLE default.mongodb_password_masking (`_id` String) ENGINE = MongoDB(\\'127.0.0.1:27017\\', \\'example\\', \\'test_clickhouse\\', \\'testuser\\', \\'[HIDDEN]\\')\n"
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(engine_full, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_password_masking' AND database = currentDatabase();
|
|
"""
|
|
)
|
|
== "MongoDB(\\'127.0.0.1:27017\\', \\'example\\', \\'test_clickhouse\\', \\'testuser\\', \\'[HIDDEN]\\')\n"
|
|
)
|
|
node.query("DROP TABLE IF EXISTS mongodb_password_masking;")
|
|
|
|
node.query(
|
|
"""
|
|
CREATE OR REPLACE DICTIONARY mongodb_dictionary_password_masking (_id String)
|
|
PRIMARY KEY _id
|
|
SOURCE(MONGODB(
|
|
host '127.0.0.1'
|
|
port 27017
|
|
user 'testuser'
|
|
password 'mypassword'
|
|
db 'example'
|
|
collection 'test_clickhouse'
|
|
options 'ssl=true'
|
|
))
|
|
LAYOUT(FLAT())
|
|
LIFETIME(0);
|
|
"""
|
|
)
|
|
assert (
|
|
node.query(
|
|
"""
|
|
SELECT replaceAll(create_table_query, currentDatabase(), 'default') FROM system.tables
|
|
WHERE table = 'mongodb_dictionary_password_masking' AND database = currentDatabase();
|
|
"""
|
|
)
|
|
== "CREATE DICTIONARY default.mongodb_dictionary_password_masking (`_id` String) PRIMARY KEY _id SOURCE(MONGODB(HOST \\'127.0.0.1\\' PORT 27017 USER \\'testuser\\' PASSWORD \\'[HIDDEN]\\' DB \\'example\\' COLLECTION \\'test_clickhouse\\' OPTIONS \\'ssl=true\\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())\n"
|
|
)
|
|
node.query("DROP DICTIONARY IF EXISTS mongodb_dictionary_password_masking;")
|