thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 00:52:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix dates

Browse Source
This commit is contained in:
Kirill Nikiforov 2024-07-29 23:39:19 +03:00
parent 2c0d18c4f7
commit 9425be31b3
No known key found for this signature in database
4 changed files with 97 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/en/engines/table-engines/integrations/mongodb.md
View File

@ -55,6 +55,18 @@ Only constant literals are allowed.
PREWHERE and HAVING are not supported.
#### Note:
It's always better to explicitly set type of literal because Mongo requires strict typed filters.\
For example you want to filter by `Date`:
```sql
SELECT * FROM mongo_table WHERE date = '2024-01-01'
```
This will not work because Mongo will not cast string to `Date`, so you need to cast it manually:
```sql
SELECT * FROM mongo_table WHERE date = '2024-01-01'::Date OR date = toDate('2024-01-01')
```
This applied for `Date`, `Date32`, `DateTime`, `Bool`, `UUID`.
### LIMIT and OFFSET
Only `LIMIT` is supported.

34
src/Common/BSONCXXHelper.h
View File

@ -28,40 +28,38 @@ static bsoncxx::types::bson_value::value fieldAsBSONValue(const Field & field, c
switch (type->getTypeId())
{
case TypeIndex::String:
return field.safeGet<String>();
return bsoncxx::types::b_string{field.safeGet<String>()};
case TypeIndex::UInt8: {
if (isBool(type))
return field.safeGet<UInt8>() != 0;
return static_cast<Int32>(field.safeGet<UInt8>());
return bsoncxx::types::b_bool{field.safeGet<UInt8>() != 0};
return bsoncxx::types::b_int32{static_cast<Int32>(field.safeGet<UInt8 &>())};
}
case TypeIndex::UInt16:
return static_cast<Int32>(field.safeGet<UInt16>());
return bsoncxx::types::b_int32{static_cast<Int32>(field.safeGet<UInt16 &>())};
case TypeIndex::UInt32:
return static_cast<Int32>(field.safeGet<UInt32>());
return bsoncxx::types::b_int64{static_cast<Int64>(field.safeGet<UInt32 &>())};
case TypeIndex::UInt64:
return static_cast<Int64>(field.safeGet<UInt64>());
return bsoncxx::types::b_double{static_cast<Float64>(field.safeGet<UInt64 &>())};
case TypeIndex::Int8:
return field.safeGet<Int8 &>();
return bsoncxx::types::b_int32{static_cast<Int32>(field.safeGet<Int8 &>())};
case TypeIndex::Int16:
return field.safeGet<Int16>();
return bsoncxx::types::b_int32{static_cast<Int32>(field.safeGet<Int16 &>())};
case TypeIndex::Int32:
return field.safeGet<Int32>();
return bsoncxx::types::b_int32{static_cast<Int32>(field.safeGet<Int32 &>())};
case TypeIndex::Int64:
return field.safeGet<Int64>();
return bsoncxx::types::b_int64{field.safeGet<Int64 &>()};
case TypeIndex::Float32:
return field.safeGet<Float32>();
return bsoncxx::types::b_double{field.safeGet<Float32 &>()};
case TypeIndex::Float64:
return field.safeGet<Float64>();
return bsoncxx::types::b_double{field.safeGet<Float64 &>()};
case TypeIndex::Date:
return std::chrono::milliseconds(field.safeGet<UInt16>() * 1000);
return bsoncxx::types::b_date{std::chrono::seconds{field.safeGet<UInt16 &>() * 86400}};
case TypeIndex::Date32:
return std::chrono::milliseconds(field.safeGet<Int32>() * 1000);
return bsoncxx::types::b_date{std::chrono::seconds{field.safeGet<Int32 &>() * 86400}};
case TypeIndex::DateTime:
return std::chrono::milliseconds(field.safeGet<UInt32>() * 1000);
case TypeIndex::DateTime64:
return std::chrono::milliseconds(field.safeGet<Decimal64>().getValue());
return bsoncxx::types::b_date{std::chrono::seconds{field.safeGet<UInt32 &>()}};
case TypeIndex::UUID:
return static_cast<String>(formatUUID(field.safeGet<UUID>()));
return bsoncxx::types::b_string{static_cast<String>(formatUUID(field.safeGet<UUID &>()))};
case TypeIndex::Tuple: {
auto arr = array();
for (const auto & elem : field.safeGet<Tuple &>())

117
src/Storages/StorageMongoDB.cpp
View File

@ -8,6 +8,7 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/SortNode.h>
#include <Analyzer/TableNode.h>
#include <Formats/BSONTypes.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTIdentifier.h>
@ -178,61 +179,79 @@ std::string mongoFuncName(const std::string & func)
throw Exception(ErrorCodes::FAILED_TO_BUILD_MONGODB_QUERY, "Function '{}' is not supported. You can disable this error with 'SET mongodb_fail_on_query_build_error=0', but this may cause poor performance, and is highly not recommended.", func);
}
bsoncxx::document::value StorageMongoDB::visitWhereFunction(const ContextPtr & context, const FunctionNode * func)
std::optional<bsoncxx::document::value> StorageMongoDB::visitWhereFunction(const ContextPtr & context, const FunctionNode * func)
{
if (!func->getArguments().getNodes().empty())
if (func->getArguments().getNodes().empty())
return {};
std::cout << func->dumpTree() << std::endl;
if (const auto & column = func->getArguments().getNodes().at(0)->as<ColumnNode>())
{
if (const auto & column = func->getArguments().getNodes().at(0)->as<ColumnNode>())
std::cout << column->dumpTree() << std::endl;
// Skip unknows columns, which don't belong to the table.
const auto & table = column->getColumnSource()->as<TableNode>();
if (!table)
return {};
std::cout << table->getStorage()->getStorageID().getFullTableName() << std::endl;
std::cout << this->getStorageID().getFullTableName() << std::endl;
// Skip columns from other tables in JOIN queries.
if (table->getStorage()->getStorageID().getFullTableName() != this->getStorageID().getFullTableName())
return {};
// Only these function can have exactly one argument and be passed to MongoDB.
if (func->getFunctionName() == "isNull")
return make_document(kvp(column->getColumnName(), make_document(kvp("$eq", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "isNotNull")
return make_document(kvp(column->getColumnName(), make_document(kvp("$ne", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "empty")
return make_document(kvp(column->getColumnName(), make_document(kvp("$in", make_array(bsoncxx::types::b_null{}, "")))));
if (func->getFunctionName() == "notEmpty")
return make_document(kvp(column->getColumnName(), make_document(kvp("$nin", make_array(bsoncxx::types::b_null{}, "")))));
auto func_name = mongoFuncName(func->getFunctionName());
if (func->getArguments().getNodes().size() == 2)
{
if (func->getFunctionName() == "isNull")
return make_document(kvp(column->getColumnName(), make_document(kvp("$eq", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "isNotNull")
return make_document(kvp(column->getColumnName(), make_document(kvp("$ne", bsoncxx::types::b_null{}))));
if (func->getFunctionName() == "empty")
return make_document(kvp(column->getColumnName(), make_document(kvp("$in", make_array(bsoncxx::types::b_null{}, "")))));
if (func->getFunctionName() == "notEmpty")
return make_document(kvp(column->getColumnName(), make_document(kvp("$nin", make_array(bsoncxx::types::b_null{}, "")))));
const auto & value = func->getArguments().getNodes().at(1);
if (func->getArguments().getNodes().size() == 2)
if (const auto & const_value = value->as<ConstantNode>())
{
auto func_name = mongoFuncName(func->getFunctionName());
std::optional<bsoncxx::types::bson_value::value> func_value{};
if (column->getColumnName() == "_id")
func_value = fieldAsOID(const_value->getValue());
else
func_value = fieldAsBSONValue(const_value->getValue(), const_value->getResultType());
const auto & value = func->getArguments().getNodes().at(1);
if (const auto & const_value = value->as<ConstantNode>())
{
std::optional<bsoncxx::types::bson_value::value> func_value{};
if (column->getColumnName() == "_id")
func_value = fieldAsOID(const_value->getValue());
else
func_value = fieldAsBSONValue(const_value->getValue(), const_value->getResultType());
if (func_name == "$in" && func_value->view().type() != bsoncxx::v_noabi::type::k_array)
func_name = "$eq";
if (func_name == "$nin" && func_value->view().type() != bsoncxx::v_noabi::type::k_array)
func_name = "$ne";
if (func_name == "$in" && func_value->view().type() != bsoncxx::v_noabi::type::k_array)
func_name = "$eq";
if (func_name == "$nin" && func_value->view().type() != bsoncxx::v_noabi::type::k_array)
func_name = "$ne";
return make_document(kvp(column->getColumnName(), make_document(kvp(func_name, std::move(*func_value)))));
}
if (const auto & func_value = value->as<FunctionNode>())
return make_document(
kvp(column->getColumnName(), make_document(kvp(func_name, visitWhereFunction(context, func_value)))));
return make_document(kvp(column->getColumnName(), make_document(kvp(func_name, std::move(*func_value)))));
}
}
else
{
auto arr = bsoncxx::builder::basic::array{};
for (const auto & elem : func->getArguments().getNodes())
{
if (const auto & elem_func = elem->as<FunctionNode>())
arr.append(visitWhereFunction(context, elem_func));
}
if (!arr.view().empty())
return make_document(kvp(mongoFuncName(func->getFunctionName()), arr));
if (const auto & func_value = value->as<FunctionNode>())
if (const auto & res_value = visitWhereFunction(context, func_value); res_value.has_value())
return make_document(kvp(column->getColumnName(), make_document(kvp(func_name, *res_value))));
}
}
else
{
auto arr = bsoncxx::builder::basic::array{};
for (const auto & elem : func->getArguments().getNodes())
{
if (const auto & elem_func = elem->as<FunctionNode>())
if (const auto & res_value = visitWhereFunction(context, elem_func); res_value.has_value())
arr.append(*res_value);
}
if (!arr.view().empty())
return make_document(kvp(mongoFuncName(func->getFunctionName()), arr));
}
throw Exception(ErrorCodes::FAILED_TO_BUILD_MONGODB_QUERY, "Only constant expressions are supported in WHERE section. You can disable this error with 'SET mongodb_fail_on_query_build_error=0', but this may cause poor performance, and is highly not recommended.");
throw Exception(
ErrorCodes::FAILED_TO_BUILD_MONGODB_QUERY,
"Only constant expressions are supported in WHERE section. You can disable this error with 'SET "
"mongodb_fail_on_query_build_error=0', but this may cause poor performance, and is highly not recommended.");
}
bsoncxx::document::value StorageMongoDB::buildMongoDBQuery(const ContextPtr & context, mongocxx::options::find & options, const SelectQueryInfo & query, const Block & sample_block)
@ -326,11 +345,11 @@ bsoncxx::document::value StorageMongoDB::buildMongoDBQuery(const ContextPtr & co
}
}
if (!filter.has_value())
throw Exception(ErrorCodes::FAILED_TO_BUILD_MONGODB_QUERY, "Only constant expressions are supported in WHERE section. You can disable this error with 'SET mongodb_fail_on_query_build_error=0', but this may cause poor performance, and is highly not recommended.");
LOG_DEBUG(log, "MongoDB query has built: '{}'.", bsoncxx::to_json(*filter));
return std::move(*filter);
if (filter.has_value())
{
LOG_DEBUG(log, "MongoDB query has built: '{}'.", bsoncxx::to_json(*filter));
return std::move(*filter);
}
}
catch (Exception & e)
{

2
src/Storages/StorageMongoDB.h
View File

@ -60,7 +60,7 @@ public:
size_t num_streams) override;
private:
static bsoncxx::document::value visitWhereFunction(const ContextPtr & context, const FunctionNode * func);
std::optional<bsoncxx::document::value> visitWhereFunction(const ContextPtr & context, const FunctionNode * func);
bsoncxx::document::value buildMongoDBQuery(const ContextPtr & context, mongocxx::options::find & options, const SelectQueryInfo & query, const Block & sample_block);
const MongoDBConfiguration configuration;