fix style, errors after merge

This commit is contained in:
Kirill Nikiforov 2024-07-27 19:23:22 +03:00
parent c23e3d8663
commit 538836f669
No known key found for this signature in database
9 changed files with 360 additions and 296 deletions

View File

@ -15,26 +15,40 @@ Keep in mind that it is deprecated, and will be removed in next releases.**
## Types mappings
| MongoDB | ClickHouse |
|--------------------|--------------------------------------------|
| bool, int32, int64 | *any numeric type*, String |
| int32 | Int32, String |
| int64 | Int64, String |
| double | Float64, String |
| date | Date, Date32, DateTime, DateTime64, String |
| string | String, UUID |
| document | String(as JSON) |
| array | Array, String(as JSON) |
| oid | String |
| *any other* | String |
| MongoDB | ClickHouse |
|--------------------|-----------------------------------------------------------------------|
| bool, int32, int64 | *any numeric type*, String |
| int32 | Int32, String |
| int64 | Int64, String |
| double | Float64, String |
| date | Date, Date32, DateTime, DateTime64, String |
| string | String, UUID |
| document | String(as JSON) |
| array | Array, String(as JSON) |
| oid | String |
| binary | String if in column, base64 encoded string if in an array or document |
| *any other* | String |
If key not found in MongoDB document, default value or null(if the column is nullable) will be inserted.
## Supported clauses
**You can disable all these restriction, see [mongodb_fail_on_query_build_error](../../../operations/settings/settings.md#mongodb_fail_on_query_build_error).**
**You can disable all these restriction, see [mongodb_fail_on_query_build_error](../../../operations/settings/settings.md#mongodb_fail_on_query_build_error).**\
*If `allow_experimental_analyzer=0`, ClickHouse will not try to build MongoDB query, sort and limit.*
*Hint: you can use MongoDB table in CTE to perform any clauses, but be aware, that in some cases, performance will be significantly degraded.*
#### You can use MongoDB table in CTE to perform any clauses, but be aware, that in some cases, performance will be significantly degraded.
For example, you want to query count() with GROUP BY(which is not supported by MongoDB engine):
```sql
SELECT count(), name FROM mongo_table WHERE name IN ('clickhouse', 'mongodb') GROUP BY name;
```
You can set `mongodb_fail_on_query_build_error=0`, but this will cause poor performance, because all data will be read from `mongo_table`
before filtering by `name`. \
So, there is a solution:
```sql
SELECT count(), name
FROM (SELECT name FROM mongo_table WHERE name in ('clickhouse', 'mongodb'))
GROUP BY name;
```
### WHERE
Only constant literals are allowed.
@ -56,12 +70,6 @@ Not supported.
### Aggregation functions
Not supported.
## Notes
### Situation with bool
In ClickHouse `boolean` is an alias for `UInt8`, but in MongoDB it's a type.
So, not in all cases it's possible to determine, is UInt8 supposed to be bool, and filters may not work correctly.
But there is a hack: use `x = toBool(true)` instead of `x = true`.
## Creating a Table {#creating-a-table}
``` sql

View File

@ -5,9 +5,6 @@
#if USE_MONGODB
#include <Common/Base64.h>
#include <DataTypes/FieldToDataType.h>
#include <bsoncxx/builder/basic/array.hpp>
#include <bsoncxx/builder/basic/document.hpp>
#include <bsoncxx/json.hpp>
using bsoncxx::builder::basic::array;
using bsoncxx::builder::basic::document;
@ -95,7 +92,7 @@ static JSONBuilder::ItemPtr BSONElementAsJSON(const T & value)
return std::make_unique<JSONBuilder::JSONString>(value.get_oid().value.to_string());
case bsoncxx::type::k_binary:
return std::make_unique<JSONBuilder::JSONString>(
std::string(reinterpret_cast<const char *>(value.get_binary().bytes), value.get_binary().size));
base64Encode(std::string(reinterpret_cast<const char *>(value.get_binary().bytes), value.get_binary().size)));
case bsoncxx::type::k_bool:
return std::make_unique<JSONBuilder::JSONBool>(value.get_bool());
case bsoncxx::type::k_int32:
@ -147,7 +144,7 @@ static std::string BSONElementAsString(const T & value, const JSONBuilder::Forma
case bsoncxx::type::k_oid:
return value.get_oid().value.to_string();
case bsoncxx::type::k_binary:
return base64Encode(std::string(reinterpret_cast<const char *>(value.get_binary().bytes), value.get_binary().size));
return std::string(reinterpret_cast<const char *>(value.get_binary().bytes), value.get_binary().size);
case bsoncxx::type::k_bool:
return value.get_bool().value ? "true" : "false";
case bsoncxx::type::k_int32:
@ -172,7 +169,6 @@ static std::string BSONElementAsString(const T & value, const JSONBuilder::Forma
WriteBufferFromOwnString buf;
auto format_context = JSONBuilder::FormatContext{.out = buf};
BSONElementAsJSON(value)->format(json_format_settings, format_context);
std::cout << buf.str() << std::endl;
return buf.str();
}
case bsoncxx::type::k_undefined:

View File

@ -604,7 +604,7 @@
M(723, PARQUET_EXCEPTION) \
M(724, TOO_MANY_TABLES) \
M(725, TOO_MANY_DATABASES) \
M(725, FAILED_TO_BUILD_MONGODB_QUERY) \
M(726, FAILED_TO_BUILD_MONGODB_QUERY) \
\
M(900, DISTRIBUTED_CACHE_ERROR) \
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \

View File

@ -18,8 +18,6 @@
#include <Common/BSONCXXHelper.h>
#include <base/range.h>
#include <bsoncxx/document/element.hpp>
namespace DB
{
@ -208,14 +206,7 @@ Chunk MongoDBSource::generate()
auto & sample_column = sample_block.getByPosition(idx);
auto value = doc[sample_column.name];
if (!value || value.type() == bsoncxx::type::k_null)
{
insertDefaultValue(*columns[idx], *sample_column.column);
if (sample_column.type->isNullable())
assert_cast<ColumnNullable &>(*columns[idx]).getNullMapData().back() = true;
}
else
if (value && value.type() != bsoncxx::type::k_null)
{
if (sample_column.type->isNullable())
{
@ -223,11 +214,13 @@ Chunk MongoDBSource::generate()
const auto & type_nullable = assert_cast<const DataTypeNullable &>(*sample_column.type);
insertValue(column_nullable.getNestedColumn(), idx, type_nullable.getNestedType(), sample_column.name, value);
column_nullable.getNullMapData().emplace_back(false);
column_nullable.getNullMapData().emplace_back(0);
}
else
insertValue(*columns[idx], idx, sample_column.type, sample_column.name, value);
}
else
insertDefaultValue(*columns[idx], *sample_column.column);
}
if (++num_rows == max_block_size)

View File

@ -20,14 +20,15 @@
#include <Common/parseAddress.h>
#include <Common/ErrorCodes.h>
#include <Common/BSONCXXHelper.h>
#include <Core/Settings.h>
#include <bsoncxx/builder/basic/document.hpp>
#include <bsoncxx/json.hpp>
using bsoncxx::builder::basic::document;
using bsoncxx::builder::basic::make_document;
using bsoncxx::builder::basic::make_array;
using bsoncxx::builder::basic::kvp;
using bsoncxx::to_json;
namespace DB
{
@ -181,44 +182,16 @@ bsoncxx::document::value StorageMongoDB::visitWhereFunction(const ContextPtr & c
{
if (!func->getArguments().getNodes().empty())
{
const auto & column = func->getArguments().getNodes().at(0)->as<ColumnNode>();
if (!column)
{
auto arr = bsoncxx::builder::basic::array{};
for (const auto & elem : func->getArguments().getNodes())
{
if (const auto & elem_func = elem->as<FunctionNode>())
arr.append(visitWhereFunction(context, elem_func));
}
if (!arr.view().empty())
return make_document(kvp(mongoFuncName(func->getFunctionName()), arr));
}
else
if (const auto & column = func->getArguments().getNodes().at(0)->as<ColumnNode>())
{
if (func->getFunctionName() == "isNull")
return make_document(kvp(
column->getColumnName(),
make_document(kvp(
"$eq",
bsoncxx::types::b_null{}))));
return make_document(kvp(column->getColumnName(), make_document(kvp("$eq", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "isNotNull")
return make_document(kvp(
column->getColumnName(),
make_document(kvp(
"$ne",
bsoncxx::types::b_null{}))));
return make_document(kvp(column->getColumnName(), make_document(kvp("$ne", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "empty")
return make_document(kvp(
column->getColumnName(),
make_document(kvp(
"$in",
make_array(bsoncxx::types::b_null{}, "")))));
return make_document(kvp(column->getColumnName(), make_document(kvp("$in", make_array(bsoncxx::types::b_null{}, "")))));
if (func->getFunctionName() == "notEmpty")
return make_document(kvp(
column->getColumnName(),
make_document(kvp(
"$nin",
make_array(bsoncxx::types::b_null{}, "")))));
return make_document(kvp(column->getColumnName(), make_document(kvp("$nin", make_array(bsoncxx::types::b_null{}, "")))));
if (func->getArguments().getNodes().size() == 2)
{
@ -238,17 +211,25 @@ bsoncxx::document::value StorageMongoDB::visitWhereFunction(const ContextPtr & c
if (func_name == "$nin" && func_value->view().type() != bsoncxx::v_noabi::type::k_array)
func_name = "$ne";
return make_document(kvp(
column->getColumnName(),
make_document(kvp(
func_name,
std::move(*func_value)))));
return make_document(kvp(column->getColumnName(), make_document(kvp(func_name, std::move(*func_value)))));
}
if (const auto & func_value = value->as<FunctionNode>())
return make_document(kvp(column->getColumnName(), make_document(kvp(func_name, visitWhereFunction(context, func_value)))));
return make_document(
kvp(column->getColumnName(), make_document(kvp(func_name, visitWhereFunction(context, func_value)))));
}
}
else
{
auto arr = bsoncxx::builder::basic::array{};
for (const auto & elem : func->getArguments().getNodes())
{
if (const auto & elem_func = elem->as<FunctionNode>())
arr.append(visitWhereFunction(context, elem_func));
}
if (!arr.view().empty())
return make_document(kvp(mongoFuncName(func->getFunctionName()), arr));
}
}
throw Exception(ErrorCodes::FAILED_TO_BUILD_MONGODB_QUERY, "Only constant expressions are supported in WHERE section. You can disable this error with 'SET mongodb_fail_on_query_build_error=0', but this may cause poor performance, and is highly not recommended.");
@ -265,7 +246,6 @@ bsoncxx::document::value StorageMongoDB::buildMongoDBQuery(const ContextPtr & co
if (!context->getSettingsRef().allow_experimental_analyzer)
return make_document();
auto & query_tree = query.query_tree->as<QueryNode &>();
std::cout << query_tree.dumpTree() << std::endl << std::endl;
if (context->getSettingsRef().mongodb_fail_on_query_build_error)
{
@ -337,10 +317,13 @@ bsoncxx::document::value StorageMongoDB::buildMongoDBQuery(const ContextPtr & co
std::optional<bsoncxx::document::value> filter{};
if (const auto & func = query_tree.getWhere()->as<FunctionNode>())
filter = visitWhereFunction(context, func);
else if (const auto & const_expr = query_tree.getWhere()->as<ConstantNode>(); const_expr->hasSourceExpression())
else if (const auto & const_expr = query_tree.getWhere()->as<ConstantNode>())
{
if (const auto & func_expr = const_expr->getSourceExpression()->as<FunctionNode>())
filter = visitWhereFunction(context, func_expr);
if (const_expr->hasSourceExpression())
{
if (const auto & func_expr = const_expr->getSourceExpression()->as<FunctionNode>())
filter = visitWhereFunction(context, func_expr);
}
}
if (!filter.has_value())

View File

@ -10,20 +10,18 @@
#include <Poco/MongoDB/Connection.h>
#include <Poco/MongoDB/Cursor.h>
#include <Poco/MongoDB/Database.h>
#include <Poco/URI.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <Common/parseAddress.h>
#include <Common/NamedCollections/NamedCollections.h>
#include <IO/Operators.h>
#include <Parsers/ASTLiteral.h>
#include <QueryPipeline/Pipe.h>
#include <Processors/Sources/MongoDBPocoLegacySource.h>
#include <Processors/Sinks/SinkToStorage.h>
#include <base/range.h>
#include <unordered_set>
#include <DataTypes/DataTypeArray.h>
namespace DB
{
@ -92,133 +90,6 @@ void StorageMongoDBPocoLegacy::connectIfNotConnected()
}
}
class StorageMongoDBPocoLegacySink : public SinkToStorage
{
public:
explicit StorageMongoDBPocoLegacySink(
const std::string & collection_name_,
const std::string & db_name_,
const StorageMetadataPtr & metadata_snapshot_,
std::shared_ptr<Poco::MongoDB::Connection> connection_)
: SinkToStorage(metadata_snapshot_->getSampleBlock())
, collection_name(collection_name_)
, db_name(db_name_)
, metadata_snapshot{metadata_snapshot_}
, connection(connection_)
, is_wire_protocol_old(isMongoDBWireProtocolOld(*connection_, db_name))
{
}
String getName() const override { return "StorageMongoDBSink"; }
void consume(Chunk chunk) override
{
Poco::MongoDB::Database db(db_name);
Poco::MongoDB::Document::Vector documents;
auto block = getHeader().cloneWithColumns(chunk.detachColumns());
size_t num_rows = block.rows();
size_t num_cols = block.columns();
const auto columns = block.getColumns();
const auto data_types = block.getDataTypes();
const auto data_names = block.getNames();
documents.reserve(num_rows);
for (const auto i : collections::range(0, num_rows))
{
Poco::MongoDB::Document::Ptr document = new Poco::MongoDB::Document();
for (const auto j : collections::range(0, num_cols))
{
insertValueIntoMongoDB(*document, data_names[j], *data_types[j], *columns[j], i);
}
documents.push_back(std::move(document));
}
if (is_wire_protocol_old)
{
Poco::SharedPtr<Poco::MongoDB::InsertRequest> insert_request = db.createInsertRequest(collection_name);
insert_request->documents() = std::move(documents);
connection->sendRequest(*insert_request);
}
else
{
Poco::SharedPtr<Poco::MongoDB::OpMsgMessage> insert_request = db.createOpMsgMessage(collection_name);
insert_request->setCommandName(Poco::MongoDB::OpMsgMessage::CMD_INSERT);
insert_request->documents() = std::move(documents);
connection->sendRequest(*insert_request);
}
}
private:
void insertValueIntoMongoDB(
Poco::MongoDB::Document & document,
const std::string & name,
const IDataType & data_type,
const IColumn & column,
size_t idx)
{
WhichDataType which(data_type);
if (which.isArray())
{
const ColumnArray & column_array = assert_cast<const ColumnArray &>(column);
const ColumnArray::Offsets & offsets = column_array.getOffsets();
size_t offset = offsets[idx - 1];
size_t next_offset = offsets[idx];
const IColumn & nested_column = column_array.getData();
const auto * array_type = assert_cast<const DataTypeArray *>(&data_type);
const DataTypePtr & nested_type = array_type->getNestedType();
Poco::MongoDB::Array::Ptr array = new Poco::MongoDB::Array();
for (size_t i = 0; i + offset < next_offset; ++i)
{
insertValueIntoMongoDB(*array, Poco::NumberFormatter::format(i), *nested_type, nested_column, i + offset);
}
document.add(name, array);
return;
}
/// MongoDB does not support UInt64 type, so just cast it to Int64
if (which.isNativeUInt())
document.add(name, static_cast<Poco::Int64>(column.getUInt(idx)));
else if (which.isNativeInt())
document.add(name, static_cast<Poco::Int64>(column.getInt(idx)));
else if (which.isFloat32())
document.add(name, static_cast<Float64>(column.getFloat32(idx)));
else if (which.isFloat64())
document.add(name, column.getFloat64(idx));
else if (which.isDate())
document.add(name, Poco::Timestamp(DateLUT::instance().fromDayNum(DayNum(column.getUInt(idx))) * 1000000));
else if (which.isDateTime())
document.add(name, Poco::Timestamp(column.getUInt(idx) * 1000000));
else
{
WriteBufferFromOwnString ostr;
data_type.getDefaultSerialization()->serializeText(column, idx, ostr, FormatSettings{});
document.add(name, ostr.str());
}
}
String collection_name;
String db_name;
StorageMetadataPtr metadata_snapshot;
std::shared_ptr<Poco::MongoDB::Connection> connection;
const bool is_wire_protocol_old;
};
Pipe StorageMongoDBPocoLegacy::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
@ -242,12 +113,6 @@ Pipe StorageMongoDBPocoLegacy::read(
return Pipe(std::make_shared<MongoDBPocoLegacySource>(connection, database_name, collection_name, Poco::MongoDB::Document{}, sample_block, max_block_size));
}
SinkToStoragePtr StorageMongoDBPocoLegacy::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr /* context */, bool /*async_insert*/)
{
connectIfNotConnected();
return std::make_shared<StorageMongoDBPocoLegacySink>(collection_name, database_name, metadata_snapshot, connection);
}
StorageMongoDBPocoLegacy::Configuration StorageMongoDBPocoLegacy::getConfiguration(ASTs engine_args, ContextPtr context)
{
Configuration configuration;

View File

@ -42,12 +42,6 @@ public:
size_t max_block_size,
size_t num_streams) override;
SinkToStoragePtr write(
const ASTPtr & query,
const StorageMetadataPtr & /*metadata_snapshot*/,
ContextPtr context,
bool async_insert) override;
struct Configuration
{
std::string host;

View File

@ -35,9 +35,13 @@ def get_mongo_connection(started_cluster, secure=False, with_credentials=True):
)
)
if with_credentials:
return pymongo.MongoClient("mongodb://root:clickhouse@localhost:{}".format(started_cluster.mongo_port))
return pymongo.MongoClient(
"mongodb://root:clickhouse@localhost:{}".format(started_cluster.mongo_port)
)
return pymongo.MongoClient("mongodb://localhost:{}".format(started_cluster.mongo_no_cred_port))
return pymongo.MongoClient(
"mongodb://localhost:{}".format(started_cluster.mongo_no_cred_port)
)
def test_simple_select(started_cluster):
@ -56,8 +60,14 @@ def test_simple_select(started_cluster):
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table.drop()
@ -79,8 +89,14 @@ def test_simple_select_uri(started_cluster):
)
assert node.query("SELECT COUNT() FROM simple_table_uri") == "100\n"
assert node.query("SELECT sum(key) FROM simple_table_uri") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from simple_table_uri where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM simple_table_uri")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_table_uri where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_table_uri")
simple_mongo_table.drop()
@ -105,8 +121,14 @@ def test_simple_select_from_view(started_cluster):
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table_view.drop()
@ -206,26 +228,71 @@ def test_arrays(started_cluster):
assert node.query("SELECT COUNT() FROM arrays_mongo_table") == "100\n"
for column_name in ["arr_int64", "arr_int32", "arr_int16", "arr_int8"]:
assert node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42") == "[-43,-44,-45]\n"
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[-43,-44,-45]\n"
)
for column_name in ["arr_uint64", "arr_uint32", "arr_uint16", "arr_uint8"]:
assert node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42") == "[43,44,45]\n"
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[43,44,45]\n"
)
for column_name in ["arr_float32", "arr_float64"]:
assert node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42") == "[43.125,44.5,45.75]\n"
assert (
node.query(f"SELECT {column_name} FROM arrays_mongo_table WHERE key = 42")
== "[43.125,44.5,45.75]\n"
)
assert node.query(f"SELECT arr_date FROM arrays_mongo_table WHERE key = 42") == "['2002-10-27','2024-01-08']\n"
assert node.query(f"SELECT arr_date32 FROM arrays_mongo_table WHERE key = 42") == "['2002-10-27','2024-01-08']\n"
assert node.query(f"SELECT arr_datetime FROM arrays_mongo_table WHERE key = 42") == "['2023-03-31 06:03:12','1999-02-28 12:46:34']\n"
assert node.query(f"SELECT arr_datetime64 FROM arrays_mongo_table WHERE key = 42") == "['2023-03-31 06:03:12.000','1999-02-28 12:46:34.000']\n"
assert node.query(f"SELECT arr_string FROM arrays_mongo_table WHERE key = 42") == "['43','44','45']\n"
assert node.query(f"SELECT arr_uuid FROM arrays_mongo_table WHERE key = 42") == "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
assert node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42") == "[[true,false,true],[true],[false],[false]]\n"
assert node.query(f"SELECT arr_arr_bool_nullable FROM arrays_mongo_table WHERE key = 42") == "[[true,false,true],[true,NULL],[NULL],[false]]\n"
assert node.query(f"SELECT arr_empty FROM arrays_mongo_table WHERE key = 42") == "[]\n"
assert node.query(f"SELECT arr_null FROM arrays_mongo_table WHERE key = 42") == "[]\n"
assert node.query(f"SELECT arr_arr_null FROM arrays_mongo_table WHERE key = 42") == "[]\n"
assert node.query(f"SELECT arr_nullable FROM arrays_mongo_table WHERE key = 42") == "[]\n"
assert (
node.query(f"SELECT arr_date FROM arrays_mongo_table WHERE key = 42")
== "['2002-10-27','2024-01-08']\n"
)
assert (
node.query(f"SELECT arr_date32 FROM arrays_mongo_table WHERE key = 42")
== "['2002-10-27','2024-01-08']\n"
)
assert (
node.query(f"SELECT arr_datetime FROM arrays_mongo_table WHERE key = 42")
== "['2023-03-31 06:03:12','1999-02-28 12:46:34']\n"
)
assert (
node.query(f"SELECT arr_datetime64 FROM arrays_mongo_table WHERE key = 42")
== "['2023-03-31 06:03:12.000','1999-02-28 12:46:34.000']\n"
)
assert (
node.query(f"SELECT arr_string FROM arrays_mongo_table WHERE key = 42")
== "['43','44','45']\n"
)
assert (
node.query(f"SELECT arr_uuid FROM arrays_mongo_table WHERE key = 42")
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
)
assert (
node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42")
== "[[true,false,true],[true],[false],[false]]\n"
)
assert (
node.query(
f"SELECT arr_arr_bool_nullable FROM arrays_mongo_table WHERE key = 42"
)
== "[[true,false,true],[true,NULL],[NULL],[false]]\n"
)
assert (
node.query(f"SELECT arr_empty FROM arrays_mongo_table WHERE key = 42") == "[]\n"
)
assert (
node.query(f"SELECT arr_null FROM arrays_mongo_table WHERE key = 42") == "[]\n"
)
assert (
node.query(f"SELECT arr_arr_null FROM arrays_mongo_table WHERE key = 42")
== "[]\n"
)
assert (
node.query(f"SELECT arr_nullable FROM arrays_mongo_table WHERE key = 42")
== "[]\n"
)
node.query("DROP TABLE arrays_mongo_table")
arrays_mongo_table.drop()
@ -247,8 +314,14 @@ def test_complex_data_type(started_cluster):
)
assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == "100\n"
assert node.query("SELECT sum(key) FROM incomplete_mongo_table") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from incomplete_mongo_table where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM incomplete_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from incomplete_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE incomplete_mongo_table")
incomplete_mongo_table.drop()
@ -269,8 +342,14 @@ def test_secure_connection(started_cluster):
)
assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n"
assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM simple_mongo_table")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from simple_mongo_table where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE simple_mongo_table")
simple_mongo_table.drop()
@ -312,8 +391,14 @@ def test_secure_connection_uri(started_cluster):
)
assert node.query("SELECT COUNT() FROM test_secure_connection_uri") == "100\n"
assert node.query("SELECT sum(key) FROM test_secure_connection_uri") == str(sum(range(0, 100))) + "\n"
assert node.query("SELECT data from test_secure_connection_uri where key = 42") == hex(42 * 42) + "\n"
assert (
node.query("SELECT sum(key) FROM test_secure_connection_uri")
== str(sum(range(0, 100))) + "\n"
)
assert (
node.query("SELECT data from test_secure_connection_uri where key = 42")
== hex(42 * 42) + "\n"
)
node.query("DROP TABLE test_secure_connection_uri")
simple_mongo_table.drop()
@ -482,8 +567,14 @@ def test_missing_columns(started_cluster):
) ENGINE = MongoDB(mongo1)"""
)
assert node.query("SELECT count() FROM simple_mongo_table WHERE isNull(data)") == "10\n"
assert node.query("SELECT count() FROM simple_mongo_table WHERE isNull(not_exists)") == "0\n"
assert (
node.query("SELECT count() FROM simple_mongo_table WHERE isNull(data)")
== "10\n"
)
assert (
node.query("SELECT count() FROM simple_mongo_table WHERE isNull(not_exists)")
== "0\n"
)
node.query("DROP TABLE IF EXISTS simple_mongo_table")
simple_mongo_table.drop()
@ -553,10 +644,16 @@ def test_string_casting(started_cluster):
assert node.query("SELECT k_doubleP FROM strings_table") == "6.660000\n"
assert node.query("SELECT k_doubleN FROM strings_table") == "-6.660000\n"
assert node.query("SELECT k_date FROM strings_table") == "1999-02-28 00:00:00\n"
assert node.query("SELECT k_timestamp FROM strings_table") == "1999-02-28 12:46:34\n"
assert (
node.query("SELECT k_timestamp FROM strings_table") == "1999-02-28 12:46:34\n"
)
assert node.query("SELECT k_string FROM strings_table") == "ClickHouse\n"
assert json.dumps(json.loads(node.query("SELECT k_document FROM strings_table"))) == json.dumps(data["k_document"])
assert json.dumps(json.loads(node.query("SELECT k_array FROM strings_table"))) == json.dumps(data["k_array"])
assert json.dumps(
json.loads(node.query("SELECT k_document FROM strings_table"))
) == json.dumps(data["k_document"])
assert json.dumps(
json.loads(node.query("SELECT k_array FROM strings_table"))
) == json.dumps(data["k_array"])
node.query("DROP TABLE strings_table")
string_mongo_table.drop()
@ -586,7 +683,10 @@ def test_dates_casting(started_cluster):
)
assert node.query("SELECT COUNT() FROM dates_table") == "1\n"
assert node.query("SELECT * FROM dates_table") == "1999-02-28 11:23:16\t1999-02-28 11:23:16.000\t1999-02-28\t1999-02-28\n"
assert (
node.query("SELECT * FROM dates_table")
== "1999-02-28 11:23:16\t1999-02-28 11:23:16.000\t1999-02-28\t1999-02-28\n"
)
node.query("DROP TABLE dates_table")
dates_mongo_table.drop()
@ -622,10 +722,26 @@ def test_order_by(started_cluster):
assert node.query("SELECT COUNT() FROM sort_table") == "900\n"
assert node.query("SELECT keyInt FROM sort_table ORDER BY keyInt LIMIT 1") == "1\n"
assert node.query("SELECT keyInt FROM sort_table ORDER BY keyInt DESC LIMIT 1") == "30\n"
assert node.query("SELECT keyInt, keyFloat FROM sort_table ORDER BY keyInt, keyFloat DESC LIMIT 1") == "1\t1.03\n"
assert node.query("SELECT keyDateTime FROM sort_table ORDER BY keyDateTime DESC LIMIT 1") == "1999-12-30 11:23:16\n"
assert node.query("SELECT keyDate FROM sort_table ORDER BY keyDate DESC LIMIT 1") == "1999-12-30\n"
assert (
node.query("SELECT keyInt FROM sort_table ORDER BY keyInt DESC LIMIT 1")
== "30\n"
)
assert (
node.query(
"SELECT keyInt, keyFloat FROM sort_table ORDER BY keyInt, keyFloat DESC LIMIT 1"
)
== "1\t1.03\n"
)
assert (
node.query(
"SELECT keyDateTime FROM sort_table ORDER BY keyDateTime DESC LIMIT 1"
)
== "1999-12-30 11:23:16\n"
)
assert (
node.query("SELECT keyDate FROM sort_table ORDER BY keyDate DESC LIMIT 1")
== "1999-12-30\n"
)
with pytest.raises(QueryRuntimeException):
node.query("SELECT * FROM sort_table ORDER BY keyInt WITH FILL")
@ -677,41 +793,116 @@ def test_where(started_cluster):
assert node.query("SELECT COUNT() FROM where_table") == "4\n"
assert node.query("SELECT keyString FROM where_table WHERE id = '11'") == "1string\n"
assert node.query("SELECT keyString FROM where_table WHERE id != '11' ORDER BY keyFloat") == "2string\n1string\n2string\n"
assert node.query("SELECT keyString FROM where_table WHERE id = '11' AND keyString = '1string'") == "1string\n"
assert node.query("SELECT id FROM where_table WHERE keyInt = 1 AND keyFloat = 1.001") == "11\n"
assert node.query("SELECT id FROM where_table WHERE keyInt = 0 OR keyFloat = 1.001") == "11\n"
assert (
node.query("SELECT keyString FROM where_table WHERE id = '11'") == "1string\n"
)
assert (
node.query(
"SELECT keyString FROM where_table WHERE id != '11' ORDER BY keyFloat"
)
== "2string\n1string\n2string\n"
)
assert (
node.query(
"SELECT keyString FROM where_table WHERE id = '11' AND keyString = '1string'"
)
== "1string\n"
)
assert (
node.query("SELECT id FROM where_table WHERE keyInt = 1 AND keyFloat = 1.001")
== "11\n"
)
assert (
node.query("SELECT id FROM where_table WHERE keyInt = 0 OR keyFloat = 1.001")
== "11\n"
)
assert node.query("SELECT id FROM where_table WHERE keyInt BETWEEN 1 AND 2") == "11\n12\n21\n22\n"
assert (
node.query("SELECT id FROM where_table WHERE keyInt BETWEEN 1 AND 2")
== "11\n12\n21\n22\n"
)
assert node.query("SELECT id FROM where_table WHERE keyInt > 10") == ""
assert node.query("SELECT id FROM where_table WHERE keyInt < 10.1 ORDER BY keyFloat") == "11\n12\n21\n22\n"
assert (
node.query("SELECT id FROM where_table WHERE keyInt < 10.1 ORDER BY keyFloat")
== "11\n12\n21\n22\n"
)
assert node.query("SELECT id FROM where_table WHERE id IN ('11')") == "11\n"
assert node.query("SELECT id FROM where_table WHERE id IN ['11']") == "11\n"
assert node.query("SELECT id FROM where_table WHERE id IN ('11', 100)") == "11\n"
assert node.query("SELECT id FROM where_table WHERE id IN ('11', '22') ORDER BY keyFloat") == "11\n22\n"
assert node.query("SELECT id FROM where_table WHERE id IN ['11', '22'] ORDER BY keyFloat") == "11\n22\n"
assert (
node.query(
"SELECT id FROM where_table WHERE id IN ('11', '22') ORDER BY keyFloat"
)
== "11\n22\n"
)
assert (
node.query(
"SELECT id FROM where_table WHERE id IN ['11', '22'] ORDER BY keyFloat"
)
== "11\n22\n"
)
assert node.query("SELECT id FROM where_table WHERE id NOT IN ('11') ORDER BY keyFloat") == "12\n21\n22\n"
assert node.query("SELECT id FROM where_table WHERE id NOT IN ['11'] ORDER BY keyFloat") == "12\n21\n22\n"
assert node.query("SELECT id FROM where_table WHERE id NOT IN ('11', 100) ORDER BY keyFloat") == "12\n21\n22\n"
assert node.query("SELECT id FROM where_table WHERE id NOT IN ('11') AND id IN ('12')") == "12\n"
assert node.query("SELECT id FROM where_table WHERE id NOT IN ['11'] AND id IN ('12')") == "12\n"
assert (
node.query(
"SELECT id FROM where_table WHERE id NOT IN ('11') ORDER BY keyFloat"
)
== "12\n21\n22\n"
)
assert (
node.query(
"SELECT id FROM where_table WHERE id NOT IN ['11'] ORDER BY keyFloat"
)
== "12\n21\n22\n"
)
assert (
node.query(
"SELECT id FROM where_table WHERE id NOT IN ('11', 100) ORDER BY keyFloat"
)
== "12\n21\n22\n"
)
assert (
node.query("SELECT id FROM where_table WHERE id NOT IN ('11') AND id IN ('12')")
== "12\n"
)
assert (
node.query("SELECT id FROM where_table WHERE id NOT IN ['11'] AND id IN ('12')")
== "12\n"
)
with pytest.raises(QueryRuntimeException):
assert node.query("SELECT id FROM where_table WHERE id NOT IN ['11', 100] ORDER BY keyFloat") == "12\n21\n22\n"
assert (
node.query(
"SELECT id FROM where_table WHERE id NOT IN ['11', 100] ORDER BY keyFloat"
)
== "12\n21\n22\n"
)
assert node.query("SELECT id FROM where_table WHERE keyDateTime > now()") == ""
assert node.query("SELECT keyInt FROM where_table WHERE keyDateTime < now() AND keyString = '1string' AND keyInt IS NOT NULL ORDER BY keyInt") == "1\n2\n"
assert (
node.query(
"SELECT keyInt FROM where_table WHERE keyDateTime < now() AND keyString = '1string' AND keyInt IS NOT NULL ORDER BY keyInt"
)
== "1\n2\n"
)
assert node.query("SELECT count() FROM where_table WHERE isNotNull(id)") == "4\n"
assert node.query("SELECT count() FROM where_table WHERE isNotNull(keyNull)") == "0\n"
assert (
node.query("SELECT count() FROM where_table WHERE isNotNull(keyNull)") == "0\n"
)
assert node.query("SELECT count() FROM where_table WHERE isNull(keyNull)") == "4\n"
assert node.query("SELECT count() FROM where_table WHERE isNotNull(keyNotExists)") == "0\n"
assert node.query("SELECT count() FROM where_table WHERE isNull(keyNotExists)") == "4\n"
assert (
node.query("SELECT count() FROM where_table WHERE isNotNull(keyNotExists)")
== "0\n"
)
assert (
node.query("SELECT count() FROM where_table WHERE isNull(keyNotExists)")
== "4\n"
)
assert node.query("SELECT count() FROM where_table WHERE keyNotExists = 0") == "0\n"
assert node.query("SELECT count() FROM where_table WHERE keyNotExists != 0") == "0\n"
assert (
node.query("SELECT count() FROM where_table WHERE keyNotExists != 0") == "0\n"
)
with pytest.raises(QueryRuntimeException):
node.query("SELECT * FROM where_table WHERE keyInt = keyFloat")
@ -854,13 +1045,32 @@ def test_oid(started_cluster):
assert node.query("SELECT COUNT() FROM oid_table") == "5\n"
assert node.query(f"SELECT key FROM oid_table WHERE _id = '{oid[0]}'") == "a\n"
assert node.query(f"SELECT * FROM oid_table WHERE _id = '{oid[2]}'") == f"{oid[2]}\tc\n"
assert (
node.query(f"SELECT * FROM oid_table WHERE _id = '{oid[2]}'")
== f"{oid[2]}\tc\n"
)
assert node.query(f"SELECT COUNT() FROM oid_table WHERE _id != '{oid[0]}'") == "4\n"
assert node.query(f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}', '{oid[1]}') ORDER BY key") == "a\nb\n"
assert node.query(f"SELECT key FROM oid_table WHERE _id in ['{oid[0]}', '{oid[1]}'] ORDER BY key") == "a\nb\n"
assert node.query(f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}') ORDER BY key") == "a\n"
assert node.query(f"SELECT key FROM oid_table WHERE _id in ['{oid[1]}'] ORDER BY key") == "b\n"
assert (
node.query(
f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}', '{oid[1]}') ORDER BY key"
)
== "a\nb\n"
)
assert (
node.query(
f"SELECT key FROM oid_table WHERE _id in ['{oid[0]}', '{oid[1]}'] ORDER BY key"
)
== "a\nb\n"
)
assert (
node.query(f"SELECT key FROM oid_table WHERE _id in ('{oid[0]}') ORDER BY key")
== "a\n"
)
assert (
node.query(f"SELECT key FROM oid_table WHERE _id in ['{oid[1]}'] ORDER BY key")
== "b\n"
)
with pytest.raises(QueryRuntimeException):
node.query("SELECT * FROM oid_table WHERE _id = 'invalidOID'")
@ -895,7 +1105,10 @@ def test_uuid(started_cluster):
"""
)
assert node.query(f"SELECT kUUID FROM uuid_table WHERE isValid = 1") == "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
assert (
node.query(f"SELECT kUUID FROM uuid_table WHERE isValid = 1")
== "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
)
with pytest.raises(QueryRuntimeException):
node.query("SELECT * FROM uuid_table WHERE isValid = 0")
@ -922,10 +1135,18 @@ def test_no_fail_on_unsupported_clauses(started_cluster):
"""
)
node.query(f"SELECT * FROM unsupported_clauses WHERE a > rand() SETTINGS mongodb_fail_on_query_build_error = 0")
node.query(f"SELECT * FROM unsupported_clauses WHERE a / 1000 > 0 SETTINGS mongodb_fail_on_query_build_error = 0")
node.query(f"SELECT * FROM unsupported_clauses WHERE toFloat64(a) < 6.66 > rand() SETTINGS mongodb_fail_on_query_build_error = 0")
node.query(f"SELECT * FROM unsupported_clauses ORDER BY a, b LIMIT 2 BY a SETTINGS mongodb_fail_on_query_build_error = 0")
node.query(
f"SELECT * FROM unsupported_clauses WHERE a > rand() SETTINGS mongodb_fail_on_query_build_error = 0"
)
node.query(
f"SELECT * FROM unsupported_clauses WHERE a / 1000 > 0 SETTINGS mongodb_fail_on_query_build_error = 0"
)
node.query(
f"SELECT * FROM unsupported_clauses WHERE toFloat64(a) < 6.66 > rand() SETTINGS mongodb_fail_on_query_build_error = 0"
)
node.query(
f"SELECT * FROM unsupported_clauses ORDER BY a, b LIMIT 2 BY a SETTINGS mongodb_fail_on_query_build_error = 0"
)
node.query("DROP TABLE unsupported_clauses")
unsupported_clauses_table.drop()

View File

@ -32,9 +32,13 @@ def get_mongo_connection(started_cluster, secure=False, with_credentials=True):
)
)
if with_credentials:
return pymongo.MongoClient("mongodb://root:clickhouse@localhost:{}".format(started_cluster.mongo_port))
return pymongo.MongoClient(
"mongodb://root:clickhouse@localhost:{}".format(started_cluster.mongo_port)
)
return pymongo.MongoClient("mongodb://localhost:{}".format(started_cluster.mongo_no_cred_port))
return pymongo.MongoClient(
"mongodb://localhost:{}".format(started_cluster.mongo_no_cred_port)
)
def test_simple_select(started_cluster):