thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:yandex/ClickHouse

Browse Source
This commit is contained in:
Ivan Blinkov 2018-10-10 19:42:55 +03:00
commit 61ccc2479f
128 changed files with 2315 additions and 796 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dbms/cmake/version.cmake
View File

@ -2,10 +2,10 @@
set(VERSION_REVISION 54409 CACHE STRING "")
set(VERSION_MAJOR 18 CACHE STRING "")
set(VERSION_MINOR 14 CACHE STRING "")
set(VERSION_PATCH 2 CACHE STRING "")
set(VERSION_GITHASH 147a2a13c256c72b7155e864e6a95024ca9ba31a CACHE STRING "")
set(VERSION_DESCRIBE v18.14.2-testing CACHE STRING "")
set(VERSION_STRING 18.14.2 CACHE STRING "")
set(VERSION_PATCH 5 CACHE STRING "")
set(VERSION_GITHASH ab198e77d1c0ee73c711f24f8638a6d6c226eec9 CACHE STRING "")
set(VERSION_DESCRIBE v18.14.5-testing CACHE STRING "")
set(VERSION_STRING 18.14.5 CACHE STRING "")
# end of autochange
set(VERSION_EXTRA "" CACHE STRING "")

2
dbms/programs/client/Client.cpp
View File

@ -904,7 +904,7 @@ private:
ASTPtr parseQuery(const char * & pos, const char * end, bool allow_multi_statements)
{
ParserQuery parser(end);
ParserQuery parser(end, true);
ASTPtr res;
const auto ignore_error = config().getBool("ignore-error", false);

2
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -2,8 +2,10 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
PingHandler.cpp
MainHandler.cpp
ColumnInfoHandler.cpp
IdentifierQuoteHandler.cpp
HandlerFactory.cpp
ODBCBridge.cpp
getIdentifierQuote.cpp
validateODBCConnectionString.cpp
)

20
dbms/programs/odbc-bridge/ColumnInfoHandler.cpp
View File

@ -1,4 +1,5 @@
#include "ColumnInfoHandler.h"
#include "getIdentifierQuote.h"
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#if USE_POCO_SQLODBC
@ -58,6 +59,11 @@ namespace
}
}
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response)
{
Poco::Net::HTMLForm params(request, request.stream());
@ -113,10 +119,22 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques
IAST::FormatSettings settings(ss, true);
settings.always_quote_identifiers = true;
settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes;
auto identifier_quote = getIdentifierQuote(hdbc);
if (identifier_quote.length() == 0)
settings.identifier_quoting_style = IdentifierQuotingStyle::None;
else if(identifier_quote[0] == '`')
settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks;
else if(identifier_quote[0] == '"')
settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes;
else
throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
select->format(settings);
std::string query = ss.str();
LOG_TRACE(log, "Inferring structure with query '" << query << "'");
if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLPrepare(hstmt, reinterpret_cast<SQLCHAR *>(query.data()), query.size())))
throw POCO_SQL_ODBC_CLASS::DescriptorException(session.dbc());

6
dbms/programs/odbc-bridge/HandlerFactory.cpp
View File

@ -24,6 +24,12 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco:
return new ODBCColumnsInfoHandler(keep_alive_timeout, context);
#else
return nullptr;
#endif
else if(uri.getPath() == "/identifier_quote")
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
return new IdentifierQuoteHandler(keep_alive_timeout, context);
#else
return nullptr;
#endif
else
return new ODBCHandler(pool_map, keep_alive_timeout, context);

3
dbms/programs/odbc-bridge/HandlerFactory.h
View File

@ -5,6 +5,7 @@
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include "MainHandler.h"
#include "ColumnInfoHandler.h"
#include "IdentifierQuoteHandler.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
@ -14,7 +15,7 @@
namespace DB
{
/** Factory for '/ping', '/' and '/columns_info' handlers.
/** Factory for '/ping', '/', '/columns_info', '/identifier_quote' handlers.
* Also stores Session pools for ODBC connections
*/
class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory

69
dbms/programs/odbc-bridge/IdentifierQuoteHandler.cpp Normal file
View File

@ -0,0 +1,69 @@
#include "IdentifierQuoteHandler.h"
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#if USE_POCO_SQLODBC
#include <Poco/SQL/ODBC/ODBCException.h>
#include <Poco/SQL/ODBC/SessionImpl.h>
#include <Poco/SQL/ODBC/Utility.h>
#define POCO_SQL_ODBC_CLASS Poco::SQL::ODBC
#endif
#if USE_POCO_DATAODBC
#include <Poco/Data/ODBC/ODBCException.h>
#include <Poco/Data/ODBC/SessionImpl.h>
#include <Poco/Data/ODBC/Utility.h>
#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC
#endif
#include <DataTypes/DataTypeFactory.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteHelpers.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/parseQuery.h>
#include <Poco/Net/HTMLForm.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include "getIdentifierQuote.h"
#include "validateODBCConnectionString.h"
namespace DB
{
void IdentifierQuoteHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response)
{
Poco::Net::HTMLForm params(request, request.stream());
LOG_TRACE(log, "Request URI: " + request.getURI());
auto process_error = [&response, this](const std::string & message)
{
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
response.send() << message << std::endl;
LOG_WARNING(log, message);
};
if (!params.has("connection_string"))
{
process_error("No 'connection_string' in request URL");
return;
}
try
{
std::string connection_string = params.get("connection_string");
POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC);
SQLHDBC hdbc = session.dbc().handle();
auto identifier = getIdentifierQuote(hdbc);
WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout);
writeStringBinary(identifier, out);
}
catch (...)
{
process_error("Error getting identifier quote style from ODBC '" + getCurrentExceptionMessage(false) + "'");
tryLogCurrentException(log);
}
}
}
#endif

28
dbms/programs/odbc-bridge/IdentifierQuoteHandler.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequestHandler.h>
#include <Common/config.h>
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
/** This handler establish connection to database, and retrieve quote style identifier
*/
namespace DB
{
class IdentifierQuoteHandler : public Poco::Net::HTTPRequestHandler
{
public:
IdentifierQuoteHandler(size_t keep_alive_timeout_, std::shared_ptr<Context> context_)
: log(&Poco::Logger::get("IdentifierQuoteHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_)
{
}
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
private:
Poco::Logger * log;
size_t keep_alive_timeout;
std::shared_ptr<Context> context;
};
}
#endif

44
dbms/programs/odbc-bridge/getIdentifierQuote.cpp Normal file
View File

@ -0,0 +1,44 @@
#include "getIdentifierQuote.h"
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#if USE_POCO_SQLODBC
#include <Poco/SQL/ODBC/ODBCException.h>
#include <Poco/SQL/ODBC/SessionImpl.h>
#include <Poco/SQL/ODBC/Utility.h>
#define POCO_SQL_ODBC_CLASS Poco::SQL::ODBC
#endif
#if USE_POCO_DATAODBC
#include <Poco/Data/ODBC/ODBCException.h>
#include <Poco/Data/ODBC/SessionImpl.h>
#include <Poco/Data/ODBC/Utility.h>
#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC
#endif
namespace DB
{
std::string getIdentifierQuote(SQLHDBC hdbc)
{
std::string identifier;
SQLSMALLINT t;
SQLRETURN r = POCO_SQL_ODBC_CLASS::SQLGetInfo(hdbc, SQL_IDENTIFIER_QUOTE_CHAR, nullptr, 0, &t);
if (POCO_SQL_ODBC_CLASS::Utility::isError(r))
throw POCO_SQL_ODBC_CLASS::ConnectionException(hdbc);
if (t > 0)
{
// I have no idea, why to add '2' here, got from: contrib/poco/Data/ODBC/src/ODBCStatementImpl.cpp:60 (SQL_DRIVER_NAME)
identifier.resize(static_cast<std::size_t>(t) + 2);
if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLGetInfo(
hdbc, SQL_IDENTIFIER_QUOTE_CHAR, &identifier[0], SQLSMALLINT((identifier.length() - 1) * sizeof(identifier[0])), &t)))
throw POCO_SQL_ODBC_CLASS::ConnectionException(hdbc);
identifier.resize(static_cast<std::size_t>(t));
}
return identifier;
}
}
#endif

22
dbms/programs/odbc-bridge/getIdentifierQuote.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include <Common/config.h>
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequestHandler.h>
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#if USE_POCO_SQLODBC
#include <Poco/SQL/ODBC/Utility.h>
#endif
#if USE_POCO_DATAODBC
#include <Poco/Data/ODBC/Utility.h>
#endif
namespace DB
{
std::string getIdentifierQuote(SQLHDBC hdbc);
}
#endif

11
dbms/src/Columns/ColumnDecimal.cpp
View File

@ -1,5 +1,5 @@
//#include <cstring>
#include <cmath>
#include <ext/bit_cast.h>
#include <Common/Exception.h>
#include <Common/Arena.h>
@ -20,6 +20,7 @@ namespace ErrorCodes
{
extern const int PARAMETER_OUT_OF_BOUND;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int NOT_IMPLEMENTED;
}
template <typename T>
@ -47,6 +48,14 @@ const char * ColumnDecimal<T>::deserializeAndInsertFromArena(const char * pos)
return pos + sizeof(T);
}
template <typename T>
UInt64 ColumnDecimal<T>::get64(size_t n) const
{
if constexpr (sizeof(T) > sizeof(UInt64))
throw Exception(String("Method get64 is not supported for ") + getFamilyName(), ErrorCodes::NOT_IMPLEMENTED);
return ext::bit_cast<UInt64>(data[n]);
}
template <typename T>
void ColumnDecimal<T>::updateHashWithValue(size_t n, SipHash & hash) const
{

1
dbms/src/Columns/ColumnDecimal.h
View File

@ -111,6 +111,7 @@ public:
void get(size_t n, Field & res) const override { res = (*this)[n]; }
bool getBool(size_t n) const override { return bool(data[n]); }
Int64 getInt(size_t n) const override { return Int64(data[n] * scale); }
UInt64 get64(size_t n) const override;
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;

151
dbms/src/Common/ODBCBridgeHelper.cpp
View File

@ -1,151 +0,0 @@
#include <Common/ODBCBridgeHelper.h>
#include <sstream>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/ShellCommand.h>
#include <Common/config.h>
#include <common/logger_useful.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING;
}
ODBCBridgeHelper::ODBCBridgeHelper(
const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: config(config_), http_timeout(http_timeout_), connection_string(connection_string_)
{
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
ping_url.setHost(bridge_host);
ping_url.setPort(bridge_port);
ping_url.setScheme("http");
ping_url.setPath(PING_HANDLER);
}
void ODBCBridgeHelper::startODBCBridge() const
{
Poco::Path path{config.getString("application.dir", "")};
path.setFileName(
#if CLICKHOUSE_SPLIT_BINARY
"clickhouse-odbc-bridge"
#else
"clickhouse"
#endif
);
if (!Poco::File(path).exists())
throw Exception("clickhouse binary (" + path.toString() + ") is not found", ErrorCodes::EXTERNAL_EXECUTABLE_NOT_FOUND);
std::stringstream command;
command << path.toString() <<
#if CLICKHOUSE_SPLIT_BINARY
" "
#else
" odbc-bridge "
#endif
;
command << "--http-port " << config.getUInt("odbc_bridge.port", DEFAULT_PORT) << ' ';
command << "--listen-host " << config.getString("odbc_bridge.listen_host", DEFAULT_HOST) << ' ';
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
if (config.has("logger.odbc_bridge_log"))
command << "--log-path " << config.getString("logger.odbc_bridge_log") << ' ';
if (config.has("logger.odbc_bridge_errlog"))
command << "--err-log-path " << config.getString("logger.odbc_bridge_errlog") << ' ';
if (config.has("logger.odbc_bridge_level"))
command << "--log-level " << config.getString("logger.odbc_bridge_level") << ' ';
command << "&"; /// we don't want to wait this process
auto command_str = command.str();
LOG_TRACE(log, "Starting clickhouse-odbc-bridge with command: " << command_str);
auto cmd = ShellCommand::execute(command_str);
cmd->wait();
}
std::vector<std::pair<std::string, std::string>> ODBCBridgeHelper::getURLParams(const std::string & cols, size_t max_block_size) const
{
std::vector<std::pair<std::string, std::string>> result;
result.emplace_back("connection_string", connection_string); /// already validated
result.emplace_back("columns", cols);
result.emplace_back("max_block_size", std::to_string(max_block_size));
return result;
}
bool ODBCBridgeHelper::checkODBCBridgeIsRunning() const
{
try
{
ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr);
return checkString(ODBCBridgeHelper::PING_OK_ANSWER, buf);
}
catch (...)
{
return false;
}
}
void ODBCBridgeHelper::startODBCBridgeSync() const
{
if (!checkODBCBridgeIsRunning())
{
LOG_TRACE(log, "clickhouse-odbc-bridge is not running, will try to start it");
startODBCBridge();
bool started = false;
for (size_t counter : ext::range(1, 20))
{
LOG_TRACE(log, "Checking clickhouse-odbc-bridge is running, try " << counter);
if (checkODBCBridgeIsRunning())
{
started = true;
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
if (!started)
throw Exception("ODBCBridgeHelper: clickhouse-odbc-bridge is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
}
Poco::URI ODBCBridgeHelper::getMainURI() const
{
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
Poco::URI main_uri;
main_uri.setHost(bridge_host);
main_uri.setPort(bridge_port);
main_uri.setScheme("http");
main_uri.setPath(MAIN_HANDLER);
return main_uri;
}
Poco::URI ODBCBridgeHelper::getColumnsInfoURI() const
{
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
Poco::URI columns_info_uri;
columns_info_uri.setHost(bridge_host);
columns_info_uri.setPort(bridge_port);
columns_info_uri.setScheme("http");
columns_info_uri.setPath(COL_INFO_HANDLER);
return columns_info_uri;
}
}

52
dbms/src/Common/ODBCBridgeHelper.h
View File

@ -1,52 +0,0 @@
#pragma once
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Poco/URI.h>
#include <Poco/Util/AbstractConfiguration.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_EXECUTABLE_NOT_FOUND;
}
/** Helper for odbc-bridge, provide utility methods, not main request
*/
class ODBCBridgeHelper
{
private:
using Configuration = Poco::Util::AbstractConfiguration;
const Configuration & config;
Poco::Timespan http_timeout;
std::string connection_string;
Poco::URI ping_url;
Poco::Logger * log = &Poco::Logger::get("ODBCBridgeHelper");
public:
static constexpr inline size_t DEFAULT_PORT = 9018;
static constexpr inline auto DEFAULT_HOST = "localhost";
static constexpr inline auto DEFAULT_FORMAT = "RowBinary";
static constexpr inline auto PING_HANDLER = "/ping";
static constexpr inline auto MAIN_HANDLER = "/";
static constexpr inline auto COL_INFO_HANDLER = "/columns_info";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
ODBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_);
std::vector<std::pair<std::string, std::string>> getURLParams(const std::string & cols, size_t max_block_size) const;
bool checkODBCBridgeIsRunning() const;
void startODBCBridge() const;
void startODBCBridgeSync() const;
Poco::URI getMainURI() const;
Poco::URI getColumnsInfoURI() const;
};
}

302
dbms/src/Common/XDBCBridgeHelper.h Normal file
View File

@ -0,0 +1,302 @@
#pragma once
#include <sstream>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Interpreters/Context.h>
#include <Poco/File.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Poco/URI.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/ShellCommand.h>
#include <Common/config.h>
#include <common/logger_useful.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_EXECUTABLE_NOT_FOUND;
extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
/**
* Class for Helpers for Xdbc-bridges, provide utility methods, not main request
*/
class IXDBCBridgeHelper
{
public:
static constexpr inline auto DEFAULT_FORMAT = "RowBinary";
virtual std::vector<std::pair<std::string, std::string>> getURLParams(const std::string & cols, size_t max_block_size) const = 0;
virtual void startBridgeSync() const = 0;
virtual Poco::URI getMainURI() const = 0;
virtual Poco::URI getColumnsInfoURI() const = 0;
virtual IdentifierQuotingStyle getIdentifierQuotingStyle() = 0;
virtual String getName() const = 0;
virtual ~IXDBCBridgeHelper() = default;
};
using BridgeHelperPtr = std::shared_ptr<IXDBCBridgeHelper>;
template <typename BridgeHelperMixin>
class XDBCBridgeHelper : public IXDBCBridgeHelper
{
private:
Poco::Timespan http_timeout;
std::string connection_string;
Poco::URI ping_url;
Poco::Logger * log = &Poco::Logger::get(BridgeHelperMixin::getName() + "BridgeHelper");
std::optional<IdentifierQuotingStyle> quote_style;
protected:
auto getConnectionString() const
{
return connection_string;
}
public:
using Configuration = Poco::Util::AbstractConfiguration;
const Configuration & config;
static constexpr inline auto DEFAULT_HOST = "localhost";
static constexpr inline auto DEFAULT_PORT = BridgeHelperMixin::DEFAULT_PORT;
static constexpr inline auto PING_HANDLER = "/ping";
static constexpr inline auto MAIN_HANDLER = "/";
static constexpr inline auto COL_INFO_HANDLER = "/columns_info";
static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
{
size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
ping_url.setHost(bridge_host);
ping_url.setPort(bridge_port);
ping_url.setScheme("http");
ping_url.setPath(PING_HANDLER);
}
String getName() const override
{
return BridgeHelperMixin::getName();
}
IdentifierQuotingStyle getIdentifierQuotingStyle() override
{
if (!quote_style.has_value())
{
startBridgeSync();
auto uri = createBaseURI();
uri.setPath(IDENTIFIER_QUOTE_HANDLER);
uri.addQueryParameter("connection_string", getConnectionString());
ReadWriteBufferFromHTTP buf(uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr);
std::string character;
readStringBinary(character, buf);
if (character.length() > 1)
throw Exception("Failed to parse quoting style from '" + character + "' for service " + BridgeHelperMixin::serviceAlias(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
else if (character.length() == 0)
quote_style = IdentifierQuotingStyle::None;
else if (character[0] == '`')
quote_style = IdentifierQuotingStyle::Backticks;
else if (character[0] == '"')
quote_style = IdentifierQuotingStyle::DoubleQuotes;
else
throw Exception("Can not map quote identifier '" + character + "' to enum value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return *quote_style;
}
/**
* @todo leaky abstraction - used by external API's
*/
std::vector<std::pair<std::string, std::string>> getURLParams(const std::string & cols, size_t max_block_size) const override
{
std::vector<std::pair<std::string, std::string>> result;
result.emplace_back("connection_string", connection_string); /// already validated
result.emplace_back("columns", cols);
result.emplace_back("max_block_size", std::to_string(max_block_size));
return result;
}
/**
* Performs spawn of external daemon
*/
void startBridgeSync() const override
{
if (!checkBridgeIsRunning())
{
LOG_TRACE(log, BridgeHelperMixin::serviceAlias() + " is not running, will try to start it");
startBridge();
bool started = false;
for (size_t counter : ext::range(1, 20))
{
LOG_TRACE(log, "Checking " + BridgeHelperMixin::serviceAlias() + " is running, try " << counter);
if (checkBridgeIsRunning())
{
started = true;
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
if (!started)
throw Exception(BridgeHelperMixin::getName() + "BridgeHelper: " + BridgeHelperMixin::serviceAlias() + " is not responding",
ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
}
/**
* URI to fetch the data from external service
*/
Poco::URI getMainURI() const override
{
auto uri = createBaseURI();
uri.setPath(MAIN_HANDLER);
return uri;
}
/**
* URI to retrieve column description from external service
*/
Poco::URI getColumnsInfoURI() const override
{
auto uri = createBaseURI();
uri.setPath(COL_INFO_HANDLER);
return uri;
}
protected:
Poco::URI createBaseURI() const
{
Poco::URI uri;
uri.setHost(ping_url.getHost());
uri.setPort(ping_url.getPort());
uri.setScheme("http");
return uri;
}
private:
bool checkBridgeIsRunning() const
{
try
{
ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr);
return checkString(XDBCBridgeHelper::PING_OK_ANSWER, buf);
}
catch (...)
{
return false;
}
}
/* Contains logic for instantiation of the bridge instance */
void startBridge() const
{
BridgeHelperMixin::startBridge(config, log, http_timeout);
}
};
struct JDBCBridgeMixin
{
static constexpr inline auto DEFAULT_PORT = 9019;
static const String configPrefix()
{
return "jdbc_bridge";
}
static const String serviceAlias()
{
return "clickhouse-jdbc-bridge";
}
static const String getName()
{
return "JDBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
{
throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
};
struct ODBCBridgeMixin
{
static constexpr inline auto DEFAULT_PORT = 9018;
static const String configPrefix()
{
return "odbc_bridge";
}
static const String serviceAlias()
{
return "clickhouse-odbc-bridge";
}
static const String getName()
{
return "ODBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
{
Poco::Path path{config.getString("application.dir", "")};
path.setFileName(
#if CLICKHOUSE_SPLIT_BINARY
"clickhouse-odbc-bridge"
#else
"clickhouse"
#endif
);
if (!Poco::File(path).exists())
throw Exception("clickhouse binary (" + path.toString() + ") is not found", ErrorCodes::EXTERNAL_EXECUTABLE_NOT_FOUND);
std::stringstream command;
command << path.toString() <<
#if CLICKHOUSE_SPLIT_BINARY
" "
#else
" odbc-bridge "
#endif
;
command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
<< ' ';
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
if (config.has("logger." + configPrefix() + "_log"))
command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
if (config.has("logger." + configPrefix() + "_errlog"))
command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
if (config.has("logger." + configPrefix() + "_level"))
command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
command << "&"; /// we don't want to wait this process
auto command_str = command.str();
std::cerr << command_str << std::endl;
LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
auto cmd = ShellCommand::execute(command_str);
cmd->wait();
}
};
}

54
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -2,7 +2,6 @@
#include <sstream>
#include <memory>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnString.h>
#include <Common/BitHelpers.h>
#include <Common/randomSeed.h>
@ -209,7 +208,7 @@ void CacheDictionary::isInConstantVector(
#define DECLARE(TYPE)\
void CacheDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const\
void CacheDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const\
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -230,6 +229,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
@ -246,7 +248,7 @@ void CacheDictionary::getString(const std::string & attribute_name, const Padded
#define DECLARE(TYPE)\
void CacheDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -265,6 +267,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void CacheDictionary::getString(
@ -280,7 +285,7 @@ void CacheDictionary::getString(
#define DECLARE(TYPE)\
void CacheDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, PaddedPODArray<TYPE> & out) const\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const\
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -299,6 +304,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void CacheDictionary::getString(
@ -491,6 +499,21 @@ CacheDictionary::Attribute CacheDictionary::createAttributeWithType(const Attrib
std::get<ContainerPtrType<Int64>>(attr.arrays) = std::make_unique<ContainerType<Int64>>(size);
bytes_allocated += size * sizeof(Int64);
break;
case AttributeUnderlyingType::Decimal32:
std::get<Decimal32>(attr.null_values) = null_value.get<Decimal32>();
std::get<ContainerPtrType<Decimal32>>(attr.arrays) = std::make_unique<ContainerType<Decimal32>>(size);
bytes_allocated += size * sizeof(Decimal32);
break;
case AttributeUnderlyingType::Decimal64:
std::get<Decimal64>(attr.null_values) = null_value.get<Decimal64>();
std::get<ContainerPtrType<Decimal64>>(attr.arrays) = std::make_unique<ContainerType<Decimal64>>(size);
bytes_allocated += size * sizeof(Decimal64);
break;
case AttributeUnderlyingType::Decimal128:
std::get<Decimal128>(attr.null_values) = null_value.get<Decimal128>();
std::get<ContainerPtrType<Decimal128>>(attr.arrays) = std::make_unique<ContainerType<Decimal128>>(size);
bytes_allocated += size * sizeof(Decimal128);
break;
case AttributeUnderlyingType::Float32:
std::get<Float32>(attr.null_values) = null_value.get<Float64>();
std::get<ContainerPtrType<Float32>>(attr.arrays) = std::make_unique<ContainerType<Float32>>(size);
@ -518,7 +541,7 @@ template <typename OutputType, typename DefaultGetter>
void CacheDictionary::getItemsNumber(
Attribute & attribute,
const PaddedPODArray<Key> & ids,
PaddedPODArray<OutputType> & out,
ResultArrayType<OutputType> & out,
DefaultGetter && get_default) const
{
if (false) {}
@ -536,6 +559,9 @@ void CacheDictionary::getItemsNumber(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -545,7 +571,7 @@ template <typename AttributeType, typename OutputType, typename DefaultGetter>
void CacheDictionary::getItemsNumberImpl(
Attribute & attribute,
const PaddedPODArray<Key> & ids,
PaddedPODArray<OutputType> & out,
ResultArrayType<OutputType> & out,
DefaultGetter && get_default) const
{
/// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
@ -893,6 +919,17 @@ void CacheDictionary::setDefaultAttributeValue(Attribute & attribute, const Key
case AttributeUnderlyingType::Int64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = std::get<Int64>(attribute.null_values); break;
case AttributeUnderlyingType::Float32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = std::get<Float32>(attribute.null_values); break;
case AttributeUnderlyingType::Float64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = std::get<Float64>(attribute.null_values); break;
case AttributeUnderlyingType::Decimal32:
std::get<ContainerPtrType<Decimal32>>(attribute.arrays)[idx] = std::get<Decimal32>(attribute.null_values);
break;
case AttributeUnderlyingType::Decimal64:
std::get<ContainerPtrType<Decimal64>>(attribute.arrays)[idx] = std::get<Decimal64>(attribute.null_values);
break;
case AttributeUnderlyingType::Decimal128:
std::get<ContainerPtrType<Decimal128>>(attribute.arrays)[idx] = std::get<Decimal128>(attribute.null_values);
break;
case AttributeUnderlyingType::String:
{
const auto & null_value_ref = std::get<String>(attribute.null_values);
@ -926,6 +963,11 @@ void CacheDictionary::setAttributeValue(Attribute & attribute, const Key idx, co
case AttributeUnderlyingType::Int64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::Float32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::Float64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::Decimal32: std::get<ContainerPtrType<Decimal32>>(attribute.arrays)[idx] = value.get<Decimal32>(); break;
case AttributeUnderlyingType::Decimal64: std::get<ContainerPtrType<Decimal64>>(attribute.arrays)[idx] = value.get<Decimal64>(); break;
case AttributeUnderlyingType::Decimal128: std::get<ContainerPtrType<Decimal128>>(attribute.arrays)[idx] = value.get<Decimal128>(); break;
case AttributeUnderlyingType::String:
{
const auto & string = value.get<String>();

26
dbms/src/Dictionaries/CacheDictionary.h
View File

@ -5,6 +5,7 @@
#include <Dictionaries/DictionaryStructure.h>
#include <Common/ArenaWithFreeLists.h>
#include <Common/CurrentMetrics.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <ext/bit_cast.h>
#include <cmath>
@ -80,8 +81,11 @@ public:
void isInVectorConstant(const PaddedPODArray<Key> & child_ids, const Key ancestor_id, PaddedPODArray<UInt8> & out) const override;
void isInConstantVector(const Key child_id, const PaddedPODArray<Key> & ancestor_ids, PaddedPODArray<UInt8> & out) const override;
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE(TYPE)\
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const;
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -93,6 +97,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const;
@ -100,7 +107,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -112,6 +119,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -119,8 +129,7 @@ public:
ColumnString * const out) const;
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, PaddedPODArray<TYPE> & out) const;
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -132,6 +141,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -174,12 +186,14 @@ private:
UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
String> null_values;
std::tuple<
ContainerPtrType<UInt8>, ContainerPtrType<UInt16>, ContainerPtrType<UInt32>, ContainerPtrType<UInt64>,
ContainerPtrType<UInt128>,
ContainerPtrType<Int8>, ContainerPtrType<Int16>, ContainerPtrType<Int32>, ContainerPtrType<Int64>,
ContainerPtrType<Decimal32>, ContainerPtrType<Decimal64>, ContainerPtrType<Decimal128>,
ContainerPtrType<Float32>, ContainerPtrType<Float64>,
ContainerPtrType<StringRef>> arrays;
};
@ -193,14 +207,14 @@ private:
void getItemsNumber(
Attribute & attribute,
const PaddedPODArray<Key> & ids,
PaddedPODArray<OutputType> & out,
ResultArrayType<OutputType> & out,
DefaultGetter && get_default) const;
template <typename AttributeType, typename OutputType, typename DefaultGetter>
void getItemsNumberImpl(
Attribute & attribute,
const PaddedPODArray<Key> & ids,
PaddedPODArray<OutputType> & out,
ResultArrayType<OutputType> & out,
DefaultGetter && get_default) const;
template <typename DefaultGetter>

29
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -6,6 +6,7 @@
#include <tuple>
#include <vector>
#include <shared_mutex>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Common/ArenaWithFreeLists.h>
#include <Common/HashTable/HashMap.h>
@ -128,11 +129,14 @@ public:
return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective;
}
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
/// In all functions below, key_columns must be full (non-constant) columns.
/// See the requirement in IDataType.h for text-serialization functions.
#define DECLARE(TYPE) \
void get##TYPE( \
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, PaddedPODArray<TYPE> & out) const;
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -144,6 +148,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ColumnString * out) const;
@ -153,7 +160,7 @@ public:
const Columns & key_columns, \
const DataTypes & key_types, \
const PaddedPODArray<TYPE> & def, \
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -165,6 +172,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name,
@ -178,7 +188,7 @@ public:
const Columns & key_columns, \
const DataTypes & key_types, \
const TYPE def, \
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -190,6 +200,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name,
@ -247,7 +260,9 @@ private:
struct Attribute final
{
AttributeUnderlyingType type;
std::tuple<UInt8, UInt16, UInt32, UInt64, UInt128, Int8, Int16, Int32, Int64, Float32, Float64, String> null_values;
std::tuple<UInt8, UInt16, UInt32, UInt64, UInt128, Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64, String> null_values;
std::tuple<ContainerPtrType<UInt8>,
ContainerPtrType<UInt16>,
ContainerPtrType<UInt32>,
@ -257,6 +272,9 @@ private:
ContainerPtrType<Int16>,
ContainerPtrType<Int32>,
ContainerPtrType<Int64>,
ContainerPtrType<Decimal32>,
ContainerPtrType<Decimal64>,
ContainerPtrType<Decimal128>,
ContainerPtrType<Float32>,
ContainerPtrType<Float64>,
ContainerPtrType<StringRef>>
@ -288,6 +306,9 @@ private:
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
}

15
dbms/src/Dictionaries/ComplexKeyCacheDictionary_createAttributeWithType.cpp
View File

@ -64,6 +64,21 @@ ComplexKeyCacheDictionary::Attribute ComplexKeyCacheDictionary::createAttributeW
std::get<ContainerPtrType<Float64>>(attr.arrays) = std::make_unique<ContainerType<Float64>>(size);
bytes_allocated += size * sizeof(Float64);
break;
case AttributeUnderlyingType::Decimal32:
std::get<Decimal32>(attr.null_values) = null_value.get<Decimal32>();
std::get<ContainerPtrType<Decimal32>>(attr.arrays) = std::make_unique<ContainerType<Decimal32>>(size);
bytes_allocated += size * sizeof(Decimal32);
break;
case AttributeUnderlyingType::Decimal64:
std::get<Decimal64>(attr.null_values) = null_value.get<Decimal64>();
std::get<ContainerPtrType<Decimal64>>(attr.arrays) = std::make_unique<ContainerType<Decimal64>>(size);
bytes_allocated += size * sizeof(Decimal64);
break;
case AttributeUnderlyingType::Decimal128:
std::get<Decimal128>(attr.null_values) = null_value.get<Decimal128>();
std::get<ContainerPtrType<Decimal128>>(attr.arrays) = std::make_unique<ContainerType<Decimal128>>(size);
bytes_allocated += size * sizeof(Decimal128);
break;
case AttributeUnderlyingType::String:
std::get<String>(attr.null_values) = null_value.get<String>();
std::get<ContainerPtrType<StringRef>>(attr.arrays) = std::make_unique<ContainerType<StringRef>>(size);

5
dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp
View File

@ -9,7 +9,7 @@ namespace ErrorCodes
#define DECLARE(TYPE) \
void ComplexKeyCacheDictionary::get##TYPE( \
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, PaddedPODArray<TYPE> & out) const \
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const \
{ \
dict_struct.validateKeyTypes(key_types); \
\
@ -33,5 +33,8 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
}

5
dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp
View File

@ -12,7 +12,7 @@ namespace ErrorCodes
const Columns & key_columns, \
const DataTypes & key_types, \
const PaddedPODArray<TYPE> & def, \
PaddedPODArray<TYPE> & out) const \
ResultArrayType<TYPE> & out) const \
{ \
dict_struct.validateKeyTypes(key_types); \
\
@ -34,5 +34,8 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
}

5
dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp
View File

@ -12,7 +12,7 @@ namespace ErrorCodes
const Columns & key_columns, \
const DataTypes & key_types, \
const TYPE def, \
PaddedPODArray<TYPE> & out) const \
ResultArrayType<TYPE> & out) const \
{ \
dict_struct.validateKeyTypes(key_types); \
\
@ -34,5 +34,8 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
}

5
dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp
View File

@ -18,6 +18,11 @@ void ComplexKeyCacheDictionary::setAttributeValue(Attribute & attribute, const s
case AttributeUnderlyingType::Int64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::Float32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::Float64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::Decimal32: std::get<ContainerPtrType<Decimal32>>(attribute.arrays)[idx] = value.get<Decimal32>(); break;
case AttributeUnderlyingType::Decimal64: std::get<ContainerPtrType<Decimal64>>(attribute.arrays)[idx] = value.get<Decimal64>(); break;
case AttributeUnderlyingType::Decimal128: std::get<ContainerPtrType<Decimal128>>(attribute.arrays)[idx] = value.get<Decimal128>(); break;
case AttributeUnderlyingType::String:
{
const auto & string = value.get<String>();

11
dbms/src/Dictionaries/ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
View File

@ -18,6 +18,17 @@ void ComplexKeyCacheDictionary::setDefaultAttributeValue(Attribute & attribute,
case AttributeUnderlyingType::Int64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = std::get<Int64>(attribute.null_values); break;
case AttributeUnderlyingType::Float32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = std::get<Float32>(attribute.null_values); break;
case AttributeUnderlyingType::Float64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = std::get<Float64>(attribute.null_values); break;
case AttributeUnderlyingType::Decimal32:
std::get<ContainerPtrType<Decimal32>>(attribute.arrays)[idx] = std::get<Decimal32>(attribute.null_values);
break;
case AttributeUnderlyingType::Decimal64:
std::get<ContainerPtrType<Decimal64>>(attribute.arrays)[idx] = std::get<Decimal64>(attribute.null_values);
break;
case AttributeUnderlyingType::Decimal128:
std::get<ContainerPtrType<Decimal128>>(attribute.arrays)[idx] = std::get<Decimal128>(attribute.null_values);
break;
case AttributeUnderlyingType::String:
{
const auto & null_value_ref = std::get<String>(attribute.null_values);

41
dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
View File

@ -45,7 +45,7 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(const ComplexKeyHashedDic
#define DECLARE(TYPE)\
void ComplexKeyHashedDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
dict_struct.validateKeyTypes(key_types);\
\
@ -70,6 +70,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void ComplexKeyHashedDictionary::getString(
@ -92,7 +95,7 @@ void ComplexKeyHashedDictionary::getString(
#define DECLARE(TYPE)\
void ComplexKeyHashedDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const PaddedPODArray<TYPE> & def, PaddedPODArray<TYPE> & out) const\
const PaddedPODArray<TYPE> & def, ResultArrayType<TYPE> & out) const\
{\
dict_struct.validateKeyTypes(key_types);\
\
@ -115,6 +118,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void ComplexKeyHashedDictionary::getString(
@ -135,7 +141,7 @@ void ComplexKeyHashedDictionary::getString(
#define DECLARE(TYPE)\
void ComplexKeyHashedDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const TYPE def, PaddedPODArray<TYPE> & out) const\
const TYPE def, ResultArrayType<TYPE> & out) const\
{\
dict_struct.validateKeyTypes(key_types);\
\
@ -158,6 +164,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void ComplexKeyHashedDictionary::getString(
@ -195,6 +204,10 @@ void ComplexKeyHashedDictionary::has(const Columns & key_columns, const DataType
case AttributeUnderlyingType::Float32: has<Float32>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Float64: has<Float64>(attribute, key_columns, out); break;
case AttributeUnderlyingType::String: has<StringRef>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal32: has<Decimal32>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal64: has<Decimal64>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal128: has<Decimal128>(attribute, key_columns, out); break;
}
}
@ -387,6 +400,11 @@ void ComplexKeyHashedDictionary::calculateBytesAllocated()
case AttributeUnderlyingType::Int64: addAttributeSize<Int64>(attribute); break;
case AttributeUnderlyingType::Float32: addAttributeSize<Float32>(attribute); break;
case AttributeUnderlyingType::Float64: addAttributeSize<Float64>(attribute); break;
case AttributeUnderlyingType::Decimal32: addAttributeSize<Decimal32>(attribute); break;
case AttributeUnderlyingType::Decimal64: addAttributeSize<Decimal64>(attribute); break;
case AttributeUnderlyingType::Decimal128: addAttributeSize<Decimal128>(attribute); break;
case AttributeUnderlyingType::String:
{
addAttributeSize<StringRef>(attribute);
@ -424,6 +442,11 @@ ComplexKeyHashedDictionary::Attribute ComplexKeyHashedDictionary::createAttribut
case AttributeUnderlyingType::Int64: createAttributeImpl<Int64>(attr, null_value); break;
case AttributeUnderlyingType::Float32: createAttributeImpl<Float32>(attr, null_value); break;
case AttributeUnderlyingType::Float64: createAttributeImpl<Float64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal32: createAttributeImpl<Decimal32>(attr, null_value); break;
case AttributeUnderlyingType::Decimal64: createAttributeImpl<Decimal64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal128: createAttributeImpl<Decimal128>(attr, null_value); break;
case AttributeUnderlyingType::String:
{
std::get<String>(attr.null_values) = null_value.get<String>();
@ -459,6 +482,9 @@ void ComplexKeyHashedDictionary::getItemsNumber(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -517,6 +543,11 @@ bool ComplexKeyHashedDictionary::setAttributeValue(Attribute & attribute, const
case AttributeUnderlyingType::Int64: return setAttributeValueImpl<Int64>(attribute, key, value.get<Int64>());
case AttributeUnderlyingType::Float32: return setAttributeValueImpl<Float32>(attribute, key, value.get<Float64>());
case AttributeUnderlyingType::Float64: return setAttributeValueImpl<Float64>(attribute, key, value.get<Float64>());
case AttributeUnderlyingType::Decimal32: return setAttributeValueImpl<Decimal32>(attribute, key, value.get<Decimal32>());
case AttributeUnderlyingType::Decimal64: return setAttributeValueImpl<Decimal64>(attribute, key, value.get<Decimal64>());
case AttributeUnderlyingType::Decimal128: return setAttributeValueImpl<Decimal128>(attribute, key, value.get<Decimal128>());
case AttributeUnderlyingType::String:
{
auto & map = *std::get<ContainerPtrType<StringRef>>(attribute.maps);
@ -604,6 +635,10 @@ std::vector<StringRef> ComplexKeyHashedDictionary::getKeys() const
case AttributeUnderlyingType::Float32: return getKeys<Float32>(attribute);
case AttributeUnderlyingType::Float64: return getKeys<Float64>(attribute);
case AttributeUnderlyingType::String: return getKeys<StringRef>(attribute);
case AttributeUnderlyingType::Decimal32: return getKeys<Decimal32>(attribute);
case AttributeUnderlyingType::Decimal64: return getKeys<Decimal64>(attribute);
case AttributeUnderlyingType::Decimal128: return getKeys<Decimal128>(attribute);
}
return {};
}

21
dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
View File

@ -5,6 +5,7 @@
#include <Dictionaries/DictionaryStructure.h>
#include <common/StringRef.h>
#include <Common/HashTable/HashMap.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Common/Arena.h>
#include <ext/range.h>
@ -65,10 +66,13 @@ public:
return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective;
}
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -80,6 +84,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -89,7 +96,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const PaddedPODArray<TYPE> & def, PaddedPODArray<TYPE> & out) const;
const PaddedPODArray<TYPE> & def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -101,6 +108,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -110,7 +120,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const TYPE def, PaddedPODArray<TYPE> & out) const;
const TYPE def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -122,6 +132,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -143,12 +156,14 @@ private:
UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
String> null_values;
std::tuple<
ContainerPtrType<UInt8>, ContainerPtrType<UInt16>, ContainerPtrType<UInt32>, ContainerPtrType<UInt64>,
ContainerPtrType<UInt128>,
ContainerPtrType<Int8>, ContainerPtrType<Int16>, ContainerPtrType<Int32>, ContainerPtrType<Int64>,
ContainerPtrType<Decimal32>, ContainerPtrType<Decimal64>, ContainerPtrType<Decimal128>,
ContainerPtrType<Float32>, ContainerPtrType<Float64>,
ContainerPtrType<StringRef>> maps;
std::unique_ptr<Arena> string_arena;

94
dbms/src/Dictionaries/DictionaryBlockInputStream.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Columns/ColumnVector.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Columns/IColumn.h>
#include <DataStreams/IProfilingBlockInputStream.h>
@ -63,12 +64,20 @@ private:
template <typename Type>
using DictionaryGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &, PaddedPODArray<Type> &) const;
template <typename Type>
using DictionaryDecimalGetter =
void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &, DecimalPaddedPODArray<Type> &) const;
using DictionaryStringGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &, ColumnString *) const;
// for complex complex key dictionaries
template <typename Type>
using GetterByKey = void (DictionaryType::*)(const std::string &, const Columns &, const DataTypes &, PaddedPODArray<Type> & out) const;
template <typename Type>
using DecimalGetterByKey =
void (DictionaryType::*)(const std::string &, const Columns &, const DataTypes &, DecimalPaddedPODArray<Type> & out) const;
using StringGetterByKey = void (DictionaryType::*)(const std::string &, const Columns &, const DataTypes &, ColumnString * out) const;
// call getXXX
@ -78,6 +87,11 @@ private:
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
template <typename Type, typename Container>
void callGetter(DictionaryDecimalGetter<Type> getter, const PaddedPODArray<Key> & ids_to_fill,
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
template <typename Container>
void callGetter(DictionaryStringGetter getter, const PaddedPODArray<Key> & ids_to_fill,
const Columns & keys, const DataTypes & data_types,
@ -89,12 +103,17 @@ private:
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
template <typename Type, typename Container>
void callGetter(DecimalGetterByKey<Type> getter, const PaddedPODArray<Key> & ids_to_fill,
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
template <typename Container>
void callGetter(StringGetterByKey getter, const PaddedPODArray<Key> & ids_to_fill,
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
template <template <typename> class Getter, typename StringGetter>
template <template <typename> class Getter, template <typename> class DecimalGetter, typename StringGetter>
Block fillBlock(const PaddedPODArray<Key> & ids_to_fill, const Columns & keys,
const DataTypes & types, ColumnsWithTypeAndName && view) const;
@ -147,7 +166,8 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)),
column_names(column_names), ids(std::move(ids)),
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<DictionaryGetter, DictionaryStringGetter>),
fill_block_function(
&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<DictionaryGetter, DictionaryDecimalGetter, DictionaryStringGetter>),
key_type(DictionaryKeyType::Id)
{
}
@ -159,7 +179,7 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
: DictionaryBlockInputStreamBase(keys.size(), max_block_size),
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)), column_names(column_names),
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, DecimalGetterByKey, StringGetterByKey>),
key_type(DictionaryKeyType::ComplexKey)
{
const DictionaryStructure & dictionaty_structure = dictionary->getStructure();
@ -175,7 +195,7 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
: DictionaryBlockInputStreamBase(data_columns.front()->size(), max_block_size),
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)), column_names(column_names),
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, DecimalGetterByKey, StringGetterByKey>),
data_columns(data_columns),
get_key_columns_function(get_key_columns_function), get_view_columns_function(get_view_columns_function),
key_type(DictionaryKeyType::Callback)
@ -243,6 +263,16 @@ void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
(dict.*getter)(attribute.name, ids_to_fill, container);
}
template <typename DictionaryType, typename Key>
template <typename Type, typename Container>
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
DictionaryDecimalGetter<Type> getter, const PaddedPODArray<Key> & ids_to_fill,
const Columns & /*keys*/, const DataTypes & /*data_types*/,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
{
(dict.*getter)(attribute.name, ids_to_fill, container);
}
template <typename DictionaryType, typename Key>
template <typename Container>
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
@ -263,6 +293,16 @@ void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
(dict.*getter)(attribute.name, keys, data_types, container);
}
template <typename DictionaryType, typename Key>
template <typename Type, typename Container>
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
DecimalGetterByKey<Type> getter, const PaddedPODArray<Key> & /*ids_to_fill*/,
const Columns & keys, const DataTypes & data_types,
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dict) const
{
(dict.*getter)(attribute.name, keys, data_types, container);
}
template <typename DictionaryType, typename Key>
template <typename Container>
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
@ -275,7 +315,7 @@ void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
template <typename DictionaryType, typename Key>
template <template <typename> class Getter, typename StringGetter>
template <template <typename> class Getter, template <typename> class DecimalGetter, typename StringGetter>
Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
const PaddedPODArray<Key> & ids_to_fill, const Columns & keys, const DataTypes & types, ColumnsWithTypeAndName && view) const
{
@ -343,6 +383,24 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
case AttributeUnderlyingType::Float64:
GET_COLUMN_FORM_ATTRIBUTE(Float64);
break;
case AttributeUnderlyingType::Decimal32:
{
column = getColumnFromAttribute<Decimal32, DecimalGetter<Decimal32>>(
&DictionaryType::getDecimal32, ids_to_fill, keys, data_types, attribute, *dictionary);
break;
}
case AttributeUnderlyingType::Decimal64:
{
column = getColumnFromAttribute<Decimal64, DecimalGetter<Decimal64>>(
&DictionaryType::getDecimal64, ids_to_fill, keys, data_types, attribute, *dictionary);
break;
}
case AttributeUnderlyingType::Decimal128:
{
column = getColumnFromAttribute<Decimal128, DecimalGetter<Decimal128>>(
&DictionaryType::getDecimal128, ids_to_fill, keys, data_types, attribute, *dictionary);
break;
}
case AttributeUnderlyingType::String:
{
column = getColumnFromStringAttribute<StringGetter>(
@ -350,7 +408,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
break;
}
}
#undef GET_COLUMN_FORM_ATTRIBUTE
block_columns.emplace_back(column, attribute.type, attribute.name);
}
}
@ -365,12 +423,24 @@ ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromAttribut
const Columns & keys, const DataTypes & data_types,
const DictionaryAttribute & attribute, const DictionaryType & dict) const
{
auto size = ids_to_fill.size();
if (!keys.empty())
size = keys.front()->size();
auto column_vector = ColumnVector<AttributeType>::create(size);
callGetter(getter, ids_to_fill, keys, data_types, column_vector->getData(), attribute, dict);
return column_vector;
if constexpr (IsDecimalNumber<AttributeType>)
{
auto size = ids_to_fill.size();
if (!keys.empty())
size = keys.front()->size();
auto column = ColumnDecimal<AttributeType>::create(size, 0); /// NOTE: There's wrong scale here, but it's unused.
callGetter(getter, ids_to_fill, keys, data_types, column->getData(), attribute, dict);
return column;
}
else
{
auto size = ids_to_fill.size();
if (!keys.empty())
size = keys.front()->size();
auto column_vector = ColumnVector<AttributeType>::create(size);
callGetter(getter, ids_to_fill, keys, data_types, column_vector->getData(), attribute, dict);
return column_vector;
}
}

13
dbms/src/Dictionaries/DictionarySourceFactory.cpp
View File

@ -7,10 +7,12 @@
#include <Dictionaries/ExecutableDictionarySource.h>
#include <Dictionaries/HTTPDictionarySource.h>
#include <Dictionaries/LibraryDictionarySource.h>
#include <Dictionaries/XDBCDictionarySource.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeNullable.h>
#include <Common/FieldVisitors.h>
#include <Common/XDBCBridgeHelper.h>
#include <Columns/ColumnsNumber.h>
#include <IO/HTTPCommon.h>
#include <memory>
@ -22,7 +24,6 @@
#endif
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#include <Poco/Data/ODBC/Connector.h>
#include <Dictionaries/ODBCDictionarySource.h>
#endif
#if USE_MYSQL
#include <Dictionaries/MySQLDictionarySource.h>
@ -154,11 +155,19 @@ DictionarySourcePtr DictionarySourceFactory::create(
else if ("odbc" == source_type)
{
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
return std::make_unique<ODBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context);
BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".odbc.connection_string"));
return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge);
#else
throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
}
else if ("jdbc" == source_type)
{
throw Exception{"Dictionary source of type `jdbc` is disabled until consistent support for nullable fields.",
ErrorCodes::SUPPORT_IS_DISABLED};
// BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<JDBCBridgeMixin>>(config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".connection_string"));
// return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".jdbc", sample_block, context, bridge);
}
else if ("executable" == source_type)
{

27
dbms/src/Dictionaries/DictionaryStructure.cpp
View File

@ -104,6 +104,17 @@ AttributeUnderlyingType getAttributeUnderlyingType(const std::string & type)
if (it != std::end(dictionary))
return it->second;
if (type.find("Decimal") == 0)
{
size_t start = strlen("Decimal");
if (type.find("32", start) == start)
return AttributeUnderlyingType::Decimal32;
if (type.find("64", start) == start)
return AttributeUnderlyingType::Decimal64;
if (type.find("128", start) == start)
return AttributeUnderlyingType::Decimal128;
}
throw Exception{"Unknown type " + type, ErrorCodes::UNKNOWN_TYPE};
}
@ -123,6 +134,9 @@ std::string toString(const AttributeUnderlyingType type)
case AttributeUnderlyingType::Int64: return "Int64";
case AttributeUnderlyingType::Float32: return "Float32";
case AttributeUnderlyingType::Float64: return "Float64";
case AttributeUnderlyingType::Decimal32: return "Decimal32";
case AttributeUnderlyingType::Decimal64: return "Decimal64";
case AttributeUnderlyingType::Decimal128: return "Decimal128";
case AttributeUnderlyingType::String: return "String";
}
@ -316,10 +330,15 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
const auto null_value_string = config.getString(prefix + "null_value");
try
{
ReadBufferFromString null_value_buffer{null_value_string};
auto column_with_null_value = type->createColumn();
type->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
null_value = (*column_with_null_value)[0];
if (null_value_string.empty())
null_value = type->getDefault();
else
{
ReadBufferFromString null_value_buffer{null_value_string};
auto column_with_null_value = type->createColumn();
type->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
null_value = (*column_with_null_value)[0];
}
}
catch (Exception & e)
{

3
dbms/src/Dictionaries/DictionaryStructure.h
View File

@ -27,6 +27,9 @@ enum class AttributeUnderlyingType
Int64,
Float32,
Float64,
Decimal32,
Decimal64,
Decimal128,
String
};

36
dbms/src/Dictionaries/FlatDictionary.cpp
View File

@ -115,7 +115,7 @@ void FlatDictionary::isInConstantVector(
#define DECLARE(TYPE)\
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const\
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -138,6 +138,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
@ -156,7 +159,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
#define DECLARE(TYPE)\
void FlatDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -177,6 +180,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void FlatDictionary::getString(
@ -194,8 +200,7 @@ void FlatDictionary::getString(
#define DECLARE(TYPE)\
void FlatDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def,\
PaddedPODArray<TYPE> & out) const\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -216,6 +221,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void FlatDictionary::getString(
@ -250,6 +258,10 @@ void FlatDictionary::has(const PaddedPODArray<Key> & ids, PaddedPODArray<UInt8>
case AttributeUnderlyingType::Float32: has<Float32>(attribute, ids, out); break;
case AttributeUnderlyingType::Float64: has<Float64>(attribute, ids, out); break;
case AttributeUnderlyingType::String: has<String>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal32: has<Decimal32>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal64: has<Decimal64>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal128: has<Decimal128>(attribute, ids, out); break;
}
}
@ -408,6 +420,11 @@ void FlatDictionary::calculateBytesAllocated()
case AttributeUnderlyingType::Int64: addAttributeSize<Int64>(attribute); break;
case AttributeUnderlyingType::Float32: addAttributeSize<Float32>(attribute); break;
case AttributeUnderlyingType::Float64: addAttributeSize<Float64>(attribute); break;
case AttributeUnderlyingType::Decimal32: addAttributeSize<Decimal32>(attribute); break;
case AttributeUnderlyingType::Decimal64: addAttributeSize<Decimal64>(attribute); break;
case AttributeUnderlyingType::Decimal128: addAttributeSize<Decimal128>(attribute); break;
case AttributeUnderlyingType::String:
{
addAttributeSize<StringRef>(attribute);
@ -460,6 +477,10 @@ FlatDictionary::Attribute FlatDictionary::createAttributeWithType(const Attribut
case AttributeUnderlyingType::Float32: createAttributeImpl<Float32>(attr, null_value); break;
case AttributeUnderlyingType::Float64: createAttributeImpl<Float64>(attr, null_value); break;
case AttributeUnderlyingType::String: createAttributeImpl<String>(attr, null_value); break;
case AttributeUnderlyingType::Decimal32: createAttributeImpl<Decimal32>(attr, null_value); break;
case AttributeUnderlyingType::Decimal64: createAttributeImpl<Decimal64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal128: createAttributeImpl<Decimal128>(attr, null_value); break;
}
return attr;
@ -488,6 +509,9 @@ void FlatDictionary::getItemsNumber(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -563,6 +587,10 @@ void FlatDictionary::setAttributeValue(Attribute & attribute, const Key id, cons
case AttributeUnderlyingType::Float32: setAttributeValueImpl<Float32>(attribute, id, value.get<Float64>()); break;
case AttributeUnderlyingType::Float64: setAttributeValueImpl<Float64>(attribute, id, value.get<Float64>()); break;
case AttributeUnderlyingType::String: setAttributeValueImpl<String>(attribute, id, value.get<String>()); break;
case AttributeUnderlyingType::Decimal32: setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal128>()); break;
case AttributeUnderlyingType::Decimal64: setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal128>()); break;
case AttributeUnderlyingType::Decimal128: setAttributeValueImpl<Decimal128>(attribute, id, value.get<Decimal128>()); break;
}
}

21
dbms/src/Dictionaries/FlatDictionary.h
View File

@ -3,6 +3,7 @@
#include <Dictionaries/IDictionary.h>
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Common/Arena.h>
#include <ext/range.h>
@ -69,8 +70,11 @@ public:
void isInVectorConstant(const PaddedPODArray<Key> & child_ids, const Key ancestor_id, PaddedPODArray<UInt8> & out) const override;
void isInConstantVector(const Key child_id, const PaddedPODArray<Key> & ancestor_ids, PaddedPODArray<UInt8> & out) const override;
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE(TYPE)\
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const;
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -82,6 +86,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const;
@ -89,7 +96,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -101,6 +108,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -110,7 +120,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -122,6 +132,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -143,12 +156,14 @@ private:
UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
StringRef> null_values;
std::tuple<
ContainerPtrType<UInt8>, ContainerPtrType<UInt16>, ContainerPtrType<UInt32>, ContainerPtrType<UInt64>,
ContainerPtrType<UInt128>,
ContainerPtrType<Int8>, ContainerPtrType<Int16>, ContainerPtrType<Int32>, ContainerPtrType<Int64>,
ContainerPtrType<Decimal32>, ContainerPtrType<Decimal64>, ContainerPtrType<Decimal128>,
ContainerPtrType<Float32>, ContainerPtrType<Float64>,
ContainerPtrType<StringRef>> arrays;
std::unique_ptr<Arena> string_arena;

41
dbms/src/Dictionaries/HashedDictionary.cpp
View File

@ -110,7 +110,7 @@ void HashedDictionary::isInConstantVector(
#define DECLARE(TYPE)\
void HashedDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const\
void HashedDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -133,6 +133,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
@ -151,7 +154,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
#define DECLARE(TYPE)\
void HashedDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -172,6 +175,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void HashedDictionary::getString(
@ -189,7 +195,7 @@ void HashedDictionary::getString(
#define DECLARE(TYPE)\
void HashedDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, PaddedPODArray<TYPE> & out) const\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
@ -210,6 +216,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void HashedDictionary::getString(
@ -243,6 +252,10 @@ void HashedDictionary::has(const PaddedPODArray<Key> & ids, PaddedPODArray<UInt8
case AttributeUnderlyingType::Float32: has<Float32>(attribute, ids, out); break;
case AttributeUnderlyingType::Float64: has<Float64>(attribute, ids, out); break;
case AttributeUnderlyingType::String: has<StringRef>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal32: has<Decimal32>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal64: has<Decimal64>(attribute, ids, out); break;
case AttributeUnderlyingType::Decimal128: has<Decimal128>(attribute, ids, out); break;
}
}
@ -398,6 +411,11 @@ void HashedDictionary::calculateBytesAllocated()
case AttributeUnderlyingType::Int64: addAttributeSize<Int64>(attribute); break;
case AttributeUnderlyingType::Float32: addAttributeSize<Float32>(attribute); break;
case AttributeUnderlyingType::Float64: addAttributeSize<Float64>(attribute); break;
case AttributeUnderlyingType::Decimal32: addAttributeSize<Decimal32>(attribute); break;
case AttributeUnderlyingType::Decimal64: addAttributeSize<Decimal64>(attribute); break;
case AttributeUnderlyingType::Decimal128: addAttributeSize<Decimal128>(attribute); break;
case AttributeUnderlyingType::String:
{
addAttributeSize<StringRef>(attribute);
@ -433,6 +451,11 @@ HashedDictionary::Attribute HashedDictionary::createAttributeWithType(const Attr
case AttributeUnderlyingType::Int64: createAttributeImpl<Int64>(attr, null_value); break;
case AttributeUnderlyingType::Float32: createAttributeImpl<Float32>(attr, null_value); break;
case AttributeUnderlyingType::Float64: createAttributeImpl<Float64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal32: createAttributeImpl<Decimal32>(attr, null_value); break;
case AttributeUnderlyingType::Decimal64: createAttributeImpl<Decimal64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal128: createAttributeImpl<Decimal128>(attr, null_value); break;
case AttributeUnderlyingType::String:
{
std::get<String>(attr.null_values) = null_value.get<String>();
@ -468,6 +491,9 @@ void HashedDictionary::getItemsNumber(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -515,6 +541,11 @@ void HashedDictionary::setAttributeValue(Attribute & attribute, const Key id, co
case AttributeUnderlyingType::Int64: setAttributeValueImpl<Int64>(attribute, id, value.get<Int64>()); break;
case AttributeUnderlyingType::Float32: setAttributeValueImpl<Float32>(attribute, id, value.get<Float64>()); break;
case AttributeUnderlyingType::Float64: setAttributeValueImpl<Float64>(attribute, id, value.get<Float64>()); break;
case AttributeUnderlyingType::Decimal32: setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal32>()); break;
case AttributeUnderlyingType::Decimal64: setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal64>()); break;
case AttributeUnderlyingType::Decimal128: setAttributeValueImpl<Decimal128>(attribute, id, value.get<Decimal128>()); break;
case AttributeUnderlyingType::String:
{
auto & map = *std::get<CollectionPtrType<StringRef>>(attribute.maps);
@ -578,6 +609,10 @@ PaddedPODArray<HashedDictionary::Key> HashedDictionary::getIds() const
case AttributeUnderlyingType::Float32: return getIds<Float32>(attribute);
case AttributeUnderlyingType::Float64: return getIds<Float64>(attribute);
case AttributeUnderlyingType::String: return getIds<StringRef>(attribute);
case AttributeUnderlyingType::Decimal32: return getIds<Decimal32>(attribute);
case AttributeUnderlyingType::Decimal64: return getIds<Decimal64>(attribute);
case AttributeUnderlyingType::Decimal128: return getIds<Decimal128>(attribute);
}
return PaddedPODArray<Key>();
}

21
dbms/src/Dictionaries/HashedDictionary.h
View File

@ -4,6 +4,7 @@
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Common/HashTable/HashMap.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <ext/range.h>
#include <atomic>
@ -64,8 +65,11 @@ public:
void toParent(const PaddedPODArray<Key> & ids, PaddedPODArray<Key> & out) const override;
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE(TYPE)\
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, PaddedPODArray<TYPE> & out) const;
void get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -77,6 +81,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const;
@ -84,7 +91,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -96,6 +103,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -104,7 +114,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, PaddedPODArray<TYPE> & out) const;
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -116,6 +126,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -141,12 +154,14 @@ private:
UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
String> null_values;
std::tuple<
CollectionPtrType<UInt8>, CollectionPtrType<UInt16>, CollectionPtrType<UInt32>, CollectionPtrType<UInt64>,
CollectionPtrType<UInt128>,
CollectionPtrType<Int8>, CollectionPtrType<Int16>, CollectionPtrType<Int32>, CollectionPtrType<Int64>,
CollectionPtrType<Decimal32>, CollectionPtrType<Decimal64>, CollectionPtrType<Decimal128>,
CollectionPtrType<Float32>, CollectionPtrType<Float64>,
CollectionPtrType<StringRef>> maps;
std::unique_ptr<Arena> string_arena;

3
dbms/src/Dictionaries/MongoDBDictionarySource.cpp
View File

@ -250,6 +250,9 @@ BlockInputStreamPtr MongoDBDictionarySource::loadKeys(
case AttributeUnderlyingType::Int16:
case AttributeUnderlyingType::Int32:
case AttributeUnderlyingType::Int64:
case AttributeUnderlyingType::Decimal32:
case AttributeUnderlyingType::Decimal64:
case AttributeUnderlyingType::Decimal128:
key.add(attr.second.name, Int32(key_columns[attr.first]->get64(row_idx)));
break;

38
dbms/src/Dictionaries/RangeDictionaryBlockInputStream.h
View File

@ -41,8 +41,12 @@ private:
using DictionaryGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &,
const PaddedPODArray<Int64> &, PaddedPODArray<Type> &) const;
template <typename AttributeType>
ColumnPtr getColumnFromAttribute(DictionaryGetter<AttributeType> getter,
template <typename Type>
using DictionaryDecimalGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &,
const PaddedPODArray<Int64> &, DecimalPaddedPODArray<Type> &) const;
template <typename AttributeType, typename Getter>
ColumnPtr getColumnFromAttribute(Getter getter,
const PaddedPODArray<Key> & ids_to_fill, const PaddedPODArray<Int64> & dates,
const DictionaryAttribute & attribute, const DictionaryType & concrete_dictionary) const;
ColumnPtr getColumnFromAttributeString(const PaddedPODArray<Key> & ids_to_fill, const PaddedPODArray<Int64> & dates,
@ -101,14 +105,23 @@ Block RangeDictionaryBlockInputStream<DictionaryType, RangeType, Key>::getBlock(
}
template <typename DictionaryType, typename RangeType, typename Key>
template <typename AttributeType>
template <typename AttributeType, typename Getter>
ColumnPtr RangeDictionaryBlockInputStream<DictionaryType, RangeType, Key>::getColumnFromAttribute(
DictionaryGetter<AttributeType> getter, const PaddedPODArray<Key> & ids_to_fill,
Getter getter, const PaddedPODArray<Key> & ids_to_fill,
const PaddedPODArray<Int64> & dates, const DictionaryAttribute & attribute, const DictionaryType & concrete_dictionary) const
{
auto column_vector = ColumnVector<AttributeType>::create(ids_to_fill.size());
(concrete_dictionary.*getter)(attribute.name, ids_to_fill, dates, column_vector->getData());
return column_vector;
if constexpr (IsDecimalNumber<AttributeType>)
{
auto column = ColumnDecimal<AttributeType>::create(ids_to_fill.size(), 0); /// NOTE: There's wrong scale here, but it's unused.
(concrete_dictionary.*getter)(attribute.name, ids_to_fill, dates, column->getData());
return column;
}
else
{
auto column_vector = ColumnVector<AttributeType>::create(ids_to_fill.size());
(concrete_dictionary.*getter)(attribute.name, ids_to_fill, dates, column_vector->getData());
return column_vector;
}
}
template <typename DictionaryType, typename RangeType, typename Key>
@ -224,11 +237,20 @@ Block RangeDictionaryBlockInputStream<DictionaryType, RangeType, Key>::fillBlock
case AttributeUnderlyingType::Float64:
GET_COLUMN_FORM_ATTRIBUTE(Float64);
break;
case AttributeUnderlyingType::Decimal32:
GET_COLUMN_FORM_ATTRIBUTE(Decimal32);
break;
case AttributeUnderlyingType::Decimal64:
GET_COLUMN_FORM_ATTRIBUTE(Decimal64);
break;
case AttributeUnderlyingType::Decimal128:
GET_COLUMN_FORM_ATTRIBUTE(Decimal128);
break;
case AttributeUnderlyingType::String:
column = getColumnFromAttributeString(ids_to_fill, date_key, attribute, *dictionary);
break;
}
#undef GET_COLUMN_FORM_ATTRIBUTE
columns.emplace_back(column, attribute.type, attribute.name);
}
}

35
dbms/src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -99,7 +99,7 @@ RangeHashedDictionary::RangeHashedDictionary(const RangeHashedDictionary & other
#define DECLARE_MULTIPLE_GETTER(TYPE)\
void RangeHashedDictionary::get##TYPE(\
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<RangeStorageType> & dates,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
const auto & attribute = getAttributeWithType(attribute_name, AttributeUnderlyingType::TYPE);\
getItems<TYPE>(attribute, ids, dates, out);\
@ -115,6 +115,9 @@ DECLARE_MULTIPLE_GETTER(Int32)
DECLARE_MULTIPLE_GETTER(Int64)
DECLARE_MULTIPLE_GETTER(Float32)
DECLARE_MULTIPLE_GETTER(Float64)
DECLARE_MULTIPLE_GETTER(Decimal32)
DECLARE_MULTIPLE_GETTER(Decimal64)
DECLARE_MULTIPLE_GETTER(Decimal128)
#undef DECLARE_MULTIPLE_GETTER
void RangeHashedDictionary::getString(
@ -239,6 +242,11 @@ void RangeHashedDictionary::calculateBytesAllocated()
case AttributeUnderlyingType::Int64: addAttributeSize<Int64>(attribute); break;
case AttributeUnderlyingType::Float32: addAttributeSize<Float32>(attribute); break;
case AttributeUnderlyingType::Float64: addAttributeSize<Float64>(attribute); break;
case AttributeUnderlyingType::Decimal32: addAttributeSize<Decimal32>(attribute); break;
case AttributeUnderlyingType::Decimal64: addAttributeSize<Decimal64>(attribute); break;
case AttributeUnderlyingType::Decimal128: addAttributeSize<Decimal128>(attribute); break;
case AttributeUnderlyingType::String:
{
addAttributeSize<StringRef>(attribute);
@ -274,6 +282,11 @@ RangeHashedDictionary::Attribute RangeHashedDictionary::createAttributeWithType(
case AttributeUnderlyingType::Int64: createAttributeImpl<Int64>(attr, null_value); break;
case AttributeUnderlyingType::Float32: createAttributeImpl<Float32>(attr, null_value); break;
case AttributeUnderlyingType::Float64: createAttributeImpl<Float64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal32: createAttributeImpl<Decimal32>(attr, null_value); break;
case AttributeUnderlyingType::Decimal64: createAttributeImpl<Decimal64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal128: createAttributeImpl<Decimal128>(attr, null_value); break;
case AttributeUnderlyingType::String:
{
std::get<String>(attr.null_values) = null_value.get<String>();
@ -309,6 +322,9 @@ void RangeHashedDictionary::getItems(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -381,6 +397,17 @@ void RangeHashedDictionary::setAttributeValue(Attribute & attribute, const Key i
case AttributeUnderlyingType::Int64: setAttributeValueImpl<Int64>(attribute, id, range, value.get<Int64>()); break;
case AttributeUnderlyingType::Float32: setAttributeValueImpl<Float32>(attribute, id, range, value.get<Float64>()); break;
case AttributeUnderlyingType::Float64: setAttributeValueImpl<Float64>(attribute, id, range, value.get<Float64>()); break;
case AttributeUnderlyingType::Decimal32:
setAttributeValueImpl<Decimal32>(attribute, id, range, value.get<Decimal32>());
break;
case AttributeUnderlyingType::Decimal64:
setAttributeValueImpl<Decimal64>(attribute, id, range, value.get<Decimal64>());
break;
case AttributeUnderlyingType::Decimal128:
setAttributeValueImpl<Decimal128>(attribute, id, range, value.get<Decimal128>());
break;
case AttributeUnderlyingType::String:
{
auto & map = *std::get<Ptr<StringRef>>(attribute.maps);
@ -449,6 +476,10 @@ void RangeHashedDictionary::getIdsAndDates(PaddedPODArray<Key> & ids,
case AttributeUnderlyingType::Float32: getIdsAndDates<Float32>(attribute, ids, start_dates, end_dates); break;
case AttributeUnderlyingType::Float64: getIdsAndDates<Float64>(attribute, ids, start_dates, end_dates); break;
case AttributeUnderlyingType::String: getIdsAndDates<StringRef>(attribute, ids, start_dates, end_dates); break;
case AttributeUnderlyingType::Decimal32: getIdsAndDates<Decimal32>(attribute, ids, start_dates, end_dates); break;
case AttributeUnderlyingType::Decimal64: getIdsAndDates<Decimal64>(attribute, ids, start_dates, end_dates); break;
case AttributeUnderlyingType::Decimal128: getIdsAndDates<Decimal128>(attribute, ids, start_dates, end_dates); break;
}
}
@ -512,7 +543,7 @@ struct RangeHashedDIctionaryCallGetBlockInputStreamImpl
BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & column_names, size_t max_block_size) const
{
using ListType = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using ListType = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Int128, Float32, Float64>;
RangeHashedDIctionaryCallGetBlockInputStreamImpl callable;
callable.dict = this;

11
dbms/src/Dictionaries/RangeHashedDictionary.h
View File

@ -4,6 +4,7 @@
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Common/HashTable/HashMap.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <atomic>
@ -61,12 +62,15 @@ public:
typedef Int64 RangeStorageType;
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE_MULTIPLE_GETTER(TYPE)\
void get##TYPE(\
const std::string & attribute_name,\
const PaddedPODArray<Key> & ids,\
const PaddedPODArray<RangeStorageType> & dates,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE_MULTIPLE_GETTER(UInt8)
DECLARE_MULTIPLE_GETTER(UInt16)
DECLARE_MULTIPLE_GETTER(UInt32)
@ -78,6 +82,9 @@ public:
DECLARE_MULTIPLE_GETTER(Int64)
DECLARE_MULTIPLE_GETTER(Float32)
DECLARE_MULTIPLE_GETTER(Float64)
DECLARE_MULTIPLE_GETTER(Decimal32)
DECLARE_MULTIPLE_GETTER(Decimal64)
DECLARE_MULTIPLE_GETTER(Decimal128)
#undef DECLARE_MULTIPLE_GETTER
void getString(
@ -114,11 +121,13 @@ private:
std::tuple<UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
String> null_values;
std::tuple<Ptr<UInt8>, Ptr<UInt16>, Ptr<UInt32>, Ptr<UInt64>,
Ptr<UInt128>,
Ptr<Int8>, Ptr<Int16>, Ptr<Int32>, Ptr<Int64>,
Ptr<Decimal32>, Ptr<Decimal64>, Ptr<Decimal128>,
Ptr<Float32>, Ptr<Float64>, Ptr<StringRef>> maps;
std::unique_ptr<Arena> string_arena;
};

37
dbms/src/Dictionaries/TrieDictionary.cpp
View File

@ -62,7 +62,7 @@ TrieDictionary::~TrieDictionary()
#define DECLARE(TYPE)\
void TrieDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
PaddedPODArray<TYPE> & out) const\
ResultArrayType<TYPE> & out) const\
{\
validateKeyTypes(key_types);\
\
@ -87,6 +87,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void TrieDictionary::getString(
@ -109,7 +112,7 @@ void TrieDictionary::getString(
#define DECLARE(TYPE)\
void TrieDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const PaddedPODArray<TYPE> & def, PaddedPODArray<TYPE> & out) const\
const PaddedPODArray<TYPE> & def, ResultArrayType<TYPE> & out) const\
{\
validateKeyTypes(key_types);\
\
@ -132,6 +135,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void TrieDictionary::getString(
@ -152,7 +158,7 @@ void TrieDictionary::getString(
#define DECLARE(TYPE)\
void TrieDictionary::get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const TYPE def, PaddedPODArray<TYPE> & out) const\
const TYPE def, ResultArrayType<TYPE> & out) const\
{\
validateKeyTypes(key_types);\
\
@ -175,6 +181,9 @@ DECLARE(Int32)
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void TrieDictionary::getString(
@ -212,6 +221,10 @@ void TrieDictionary::has(const Columns & key_columns, const DataTypes & key_type
case AttributeUnderlyingType::Float32: has<Float32>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Float64: has<Float64>(attribute, key_columns, out); break;
case AttributeUnderlyingType::String: has<StringRef>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal32: has<Decimal32>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal64: has<Decimal64>(attribute, key_columns, out); break;
case AttributeUnderlyingType::Decimal128: has<Decimal128>(attribute, key_columns, out); break;
}
}
@ -306,6 +319,11 @@ void TrieDictionary::calculateBytesAllocated()
case AttributeUnderlyingType::Int64: addAttributeSize<Int64>(attribute); break;
case AttributeUnderlyingType::Float32: addAttributeSize<Float32>(attribute); break;
case AttributeUnderlyingType::Float64: addAttributeSize<Float64>(attribute); break;
case AttributeUnderlyingType::Decimal32: addAttributeSize<Decimal32>(attribute); break;
case AttributeUnderlyingType::Decimal64: addAttributeSize<Decimal64>(attribute); break;
case AttributeUnderlyingType::Decimal128: addAttributeSize<Decimal128>(attribute); break;
case AttributeUnderlyingType::String:
{
addAttributeSize<StringRef>(attribute);
@ -355,6 +373,11 @@ TrieDictionary::Attribute TrieDictionary::createAttributeWithType(const Attribut
case AttributeUnderlyingType::Int64: createAttributeImpl<Int64>(attr, null_value); break;
case AttributeUnderlyingType::Float32: createAttributeImpl<Float32>(attr, null_value); break;
case AttributeUnderlyingType::Float64: createAttributeImpl<Float64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal32: createAttributeImpl<Decimal32>(attr, null_value); break;
case AttributeUnderlyingType::Decimal64: createAttributeImpl<Decimal64>(attr, null_value); break;
case AttributeUnderlyingType::Decimal128: createAttributeImpl<Decimal128>(attr, null_value); break;
case AttributeUnderlyingType::String:
{
std::get<String>(attr.null_values) = null_value.get<String>();
@ -390,6 +413,9 @@ void TrieDictionary::getItemsNumber(
DISPATCH(Int64)
DISPATCH(Float32)
DISPATCH(Float64)
DISPATCH(Decimal32)
DISPATCH(Decimal64)
DISPATCH(Decimal128)
#undef DISPATCH
else
throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
@ -490,6 +516,11 @@ bool TrieDictionary::setAttributeValue(Attribute & attribute, const StringRef ke
case AttributeUnderlyingType::Int64: return setAttributeValueImpl<Int64>(attribute, key, value.get<Int64>());
case AttributeUnderlyingType::Float32: return setAttributeValueImpl<Float32>(attribute, key, value.get<Float64>());
case AttributeUnderlyingType::Float64: return setAttributeValueImpl<Float64>(attribute, key, value.get<Float64>());
case AttributeUnderlyingType::Decimal32: return setAttributeValueImpl<Decimal32>(attribute, key, value.get<Decimal32>());
case AttributeUnderlyingType::Decimal64: return setAttributeValueImpl<Decimal64>(attribute, key, value.get<Decimal64>());
case AttributeUnderlyingType::Decimal128: return setAttributeValueImpl<Decimal128>(attribute, key, value.get<Decimal128>());
case AttributeUnderlyingType::String:
{
const auto & string = value.get<String>();

21
dbms/src/Dictionaries/TrieDictionary.h
View File

@ -5,6 +5,7 @@
#include <Dictionaries/DictionaryStructure.h>
#include <common/StringRef.h>
#include <Common/HashTable/HashMap.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Common/Arena.h>
#include <ext/range.h>
@ -68,10 +69,13 @@ public:
return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective;
}
template <typename T>
using ResultArrayType = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<T>, PaddedPODArray<T>>;
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
PaddedPODArray<TYPE> & out) const;
ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -83,6 +87,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -92,7 +99,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const PaddedPODArray<TYPE> & def, PaddedPODArray<TYPE> & out) const;
const PaddedPODArray<TYPE> & def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -104,6 +111,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -113,7 +123,7 @@ public:
#define DECLARE(TYPE)\
void get##TYPE(\
const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types,\
const TYPE def, PaddedPODArray<TYPE> & out) const;
const TYPE def, ResultArrayType<TYPE> & out) const;
DECLARE(UInt8)
DECLARE(UInt16)
DECLARE(UInt32)
@ -125,6 +135,9 @@ public:
DECLARE(Int64)
DECLARE(Float32)
DECLARE(Float64)
DECLARE(Decimal32)
DECLARE(Decimal64)
DECLARE(Decimal128)
#undef DECLARE
void getString(
@ -146,12 +159,14 @@ private:
UInt8, UInt16, UInt32, UInt64,
UInt128,
Int8, Int16, Int32, Int64,
Decimal32, Decimal64, Decimal128,
Float32, Float64,
String> null_values;
std::tuple<
ContainerPtrType<UInt8>, ContainerPtrType<UInt16>, ContainerPtrType<UInt32>, ContainerPtrType<UInt64>,
ContainerPtrType<UInt128>,
ContainerPtrType<Int8>, ContainerPtrType<Int16>, ContainerPtrType<Int32>, ContainerPtrType<Int64>,
ContainerPtrType<Decimal32>, ContainerPtrType<Decimal64>, ContainerPtrType<Decimal128>,
ContainerPtrType<Float32>, ContainerPtrType<Float64>,
ContainerPtrType<StringRef>> maps;
std::unique_ptr<Arena> string_arena;

98
dbms/src/Dictionaries/ODBCDictionarySource.cpp → dbms/src/Dictionaries/XDBCDictionarySource.cpp
View File

@ -1,4 +1,4 @@
#include <Dictionaries/ODBCDictionarySource.h>
#include <Dictionaries/XDBCDictionarySource.h>
#include <common/logger_useful.h>
#include <common/LocalDateTime.h>
#include <Poco/Ext/SessionPoolHelpers.h>
@ -12,6 +12,7 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Formats/FormatFactory.h>
#include <Common/XDBCBridgeHelper.h>
namespace DB
@ -19,18 +20,18 @@ namespace DB
namespace
{
class ODBCBridgeBlockInputStream : public IProfilingBlockInputStream
class XDBCBridgeBlockInputStream : public IProfilingBlockInputStream
{
public:
ODBCBridgeBlockInputStream(const Poco::URI & uri,
XDBCBridgeBlockInputStream(const Poco::URI & uri,
std::function<void(std::ostream &)> callback,
const Block & sample_block,
const Context & context,
size_t max_block_size,
const ConnectionTimeouts & timeouts)
const ConnectionTimeouts & timeouts, const String name) : name(name)
{
read_buf = std::make_unique<ReadWriteBufferFromHTTP>(uri, Poco::Net::HTTPRequest::HTTP_POST, callback, timeouts);
reader = FormatFactory::instance().getInput(ODBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size);
reader = FormatFactory::instance().getInput(IXDBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size);
}
Block getHeader() const override
@ -40,7 +41,7 @@ namespace
String getName() const override
{
return "ODBCBridgeBlockInputStream";
return name;
}
private:
@ -49,6 +50,7 @@ namespace
return reader->read();
}
String name;
std::unique_ptr<ReadWriteBufferFromHTTP> read_buf;
BlockInputStreamPtr reader;
};
@ -57,34 +59,34 @@ namespace
static const size_t max_block_size = 8192;
ODBCDictionarySource::ODBCDictionarySource(const DictionaryStructure & dict_struct_,
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
const Block & sample_block, const Context & context)
: log(&Logger::get("ODBCDictionarySource")),
XDBCDictionarySource::XDBCDictionarySource(const DictionaryStructure & dict_struct_,
const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix_,
const Block & sample_block_, const Context & context_, const BridgeHelperPtr bridge_)
: log(&Logger::get(bridge_->getName() + "DictionarySource")),
update_time{std::chrono::system_clock::from_time_t(0)},
dict_struct{dict_struct_},
db{config.getString(config_prefix + ".db", "")},
table{config.getString(config_prefix + ".table")},
where{config.getString(config_prefix + ".where", "")},
update_field{config.getString(config_prefix + ".update_field", "")},
sample_block{sample_block},
query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::None}, /// NOTE Better to obtain quoting style via ODBC interface.
db{config_.getString(config_prefix_ + ".db", "")},
table{config_.getString(config_prefix_ + ".table")},
where{config_.getString(config_prefix_ + ".where", "")},
update_field{config_.getString(config_prefix_ + ".update_field", "")},
sample_block{sample_block_},
query_builder{dict_struct, db, table, where, bridge_->getIdentifierQuotingStyle()},
load_all_query{query_builder.composeLoadAllQuery()},
invalidate_query{config.getString(config_prefix + ".invalidate_query", "")},
odbc_bridge_helper{context.getConfigRef(), context.getSettingsRef().http_receive_timeout.value, config.getString(config_prefix + ".connection_string")},
timeouts{ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef())},
global_context(context)
invalidate_query{config_.getString(config_prefix_ + ".invalidate_query", "")},
bridge_helper{bridge_},
timeouts{ConnectionTimeouts::getHTTPTimeouts(context_.getSettingsRef())},
global_context(context_)
{
bridge_url = odbc_bridge_helper.getMainURI();
bridge_url = bridge_helper->getMainURI();
auto url_params = odbc_bridge_helper.getURLParams(sample_block.getNamesAndTypesList().toString(), max_block_size);
auto url_params = bridge_helper->getURLParams(sample_block_.getNamesAndTypesList().toString(), max_block_size);
for (const auto & [name, value] : url_params)
bridge_url.addQueryParameter(name, value);
}
/// copy-constructor is provided in order to support cloneability
ODBCDictionarySource::ODBCDictionarySource(const ODBCDictionarySource & other)
: log(&Logger::get("ODBCDictionarySource")),
XDBCDictionarySource::XDBCDictionarySource(const XDBCDictionarySource & other)
: log(&Logger::get(other.bridge_helper->getName() + "DictionarySource")),
update_time{other.update_time},
dict_struct{other.dict_struct},
db{other.db},
@ -92,11 +94,11 @@ ODBCDictionarySource::ODBCDictionarySource(const ODBCDictionarySource & other)
where{other.where},
update_field{other.update_field},
sample_block{other.sample_block},
query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::None},
query_builder{dict_struct, db, table, where, other.bridge_helper->getIdentifierQuotingStyle()},
load_all_query{other.load_all_query},
invalidate_query{other.invalidate_query},
invalidate_query_response{other.invalidate_query_response},
odbc_bridge_helper{other.odbc_bridge_helper},
bridge_helper{other.bridge_helper},
bridge_url{other.bridge_url},
timeouts{other.timeouts},
global_context{other.global_context}
@ -104,7 +106,7 @@ ODBCDictionarySource::ODBCDictionarySource(const ODBCDictionarySource & other)
}
std::string ODBCDictionarySource::getUpdateFieldAndDate()
std::string XDBCDictionarySource::getUpdateFieldAndDate()
{
if (update_time != std::chrono::system_clock::from_time_t(0))
{
@ -122,13 +124,13 @@ std::string ODBCDictionarySource::getUpdateFieldAndDate()
}
}
BlockInputStreamPtr ODBCDictionarySource::loadAll()
BlockInputStreamPtr XDBCDictionarySource::loadAll()
{
LOG_TRACE(log, load_all_query);
return loadBase(load_all_query);
}
BlockInputStreamPtr ODBCDictionarySource::loadUpdatedAll()
BlockInputStreamPtr XDBCDictionarySource::loadUpdatedAll()
{
std::string load_query_update = getUpdateFieldAndDate();
@ -136,40 +138,40 @@ BlockInputStreamPtr ODBCDictionarySource::loadUpdatedAll()
return loadBase(load_query_update);
}
BlockInputStreamPtr ODBCDictionarySource::loadIds(const std::vector<UInt64> & ids)
BlockInputStreamPtr XDBCDictionarySource::loadIds(const std::vector<UInt64> & ids)
{
const auto query = query_builder.composeLoadIdsQuery(ids);
return loadBase(query);
}
BlockInputStreamPtr ODBCDictionarySource::loadKeys(
BlockInputStreamPtr XDBCDictionarySource::loadKeys(
const Columns & key_columns, const std::vector<size_t> & requested_rows)
{
const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN);
return loadBase(query);
}
bool ODBCDictionarySource::supportsSelectiveLoad() const
bool XDBCDictionarySource::supportsSelectiveLoad() const
{
return true;
}
bool ODBCDictionarySource::hasUpdateField() const
bool XDBCDictionarySource::hasUpdateField() const
{
return !update_field.empty();
}
DictionarySourcePtr ODBCDictionarySource::clone() const
DictionarySourcePtr XDBCDictionarySource::clone() const
{
return std::make_unique<ODBCDictionarySource>(*this);
return std::make_unique<XDBCDictionarySource>(*this);
}
std::string ODBCDictionarySource::toString() const
std::string XDBCDictionarySource::toString() const
{
return "ODBC: " + db + '.' + table + (where.empty() ? "" : ", where: " + where);
return bridge_helper->getName() + ": " + db + '.' + table + (where.empty() ? "" : ", where: " + where);
}
bool ODBCDictionarySource::isModified() const
bool XDBCDictionarySource::isModified() const
{
if (!invalidate_query.empty())
{
@ -182,39 +184,39 @@ bool ODBCDictionarySource::isModified() const
}
std::string ODBCDictionarySource::doInvalidateQuery(const std::string & request) const
std::string XDBCDictionarySource::doInvalidateQuery(const std::string & request) const
{
Block invalidate_sample_block;
ColumnPtr column(ColumnString::create());
invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
odbc_bridge_helper.startODBCBridgeSync();
bridge_helper->startBridgeSync();
auto invalidate_url = odbc_bridge_helper.getMainURI();
auto url_params = odbc_bridge_helper.getURLParams(invalidate_sample_block.getNamesAndTypesList().toString(), max_block_size);
auto invalidate_url = bridge_helper->getMainURI();
auto url_params = bridge_helper->getURLParams(invalidate_sample_block.getNamesAndTypesList().toString(), max_block_size);
for (const auto & [name, value] : url_params)
invalidate_url.addQueryParameter(name, value);
ODBCBridgeBlockInputStream stream(
XDBCBridgeBlockInputStream stream(
invalidate_url,
[request](std::ostream & os) { os << "query=" << request; },
invalidate_sample_block,
global_context,
max_block_size,
timeouts);
timeouts, bridge_helper->getName() + "BlockInputStream");
return readInvalidateQuery(stream);
}
BlockInputStreamPtr ODBCDictionarySource::loadBase(const std::string & query) const
BlockInputStreamPtr XDBCDictionarySource::loadBase(const std::string & query) const
{
odbc_bridge_helper.startODBCBridgeSync();
return std::make_shared<ODBCBridgeBlockInputStream>(bridge_url,
bridge_helper->startBridgeSync();
return std::make_shared<XDBCBridgeBlockInputStream>(bridge_url,
[query](std::ostream & os) { os << "query=" << query; },
sample_block,
global_context,
max_block_size,
timeouts);
timeouts, bridge_helper->getName() + "BlockInputStream");
}
}

42
dbms/src/Dictionaries/ODBCDictionarySource.h → dbms/src/Dictionaries/XDBCDictionarySource.h
View File

@ -3,40 +3,40 @@
#include <Poco/Data/SessionPool.h>
#include <Poco/URI.h>
#include <Common/ODBCBridgeHelper.h>
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/ExternalQueryBuilder.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/ExternalQueryBuilder.h>
#include <Dictionaries/IDictionarySource.h>
#include <IO/ConnectionTimeouts.h>
#include <Common/XDBCBridgeHelper.h>
namespace Poco
{
namespace Util
{
class AbstractConfiguration;
}
namespace Util
{
class AbstractConfiguration;
}
class Logger;
class Logger;
}
namespace DB
{
/// Allows loading dictionaries from a ODBC source
class ODBCDictionarySource final : public IDictionarySource
/// Allows loading dictionaries from a XDBC source via bridges
class XDBCDictionarySource final : public IDictionarySource
{
public:
ODBCDictionarySource(const DictionaryStructure & dict_struct_,
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
const Block & sample_block, const Context & context);
XDBCDictionarySource(const DictionaryStructure & dict_struct_,
const Poco::Util::AbstractConfiguration & config_,
const std::string & config_prefix_,
const Block & sample_block_,
const Context & context_,
BridgeHelperPtr bridge);
/// copy-constructor is provided in order to support cloneability
ODBCDictionarySource(const ODBCDictionarySource & other);
XDBCDictionarySource(const XDBCDictionarySource & other);
BlockInputStreamPtr loadAll() override;
@ -44,8 +44,7 @@ public:
BlockInputStreamPtr loadIds(const std::vector<UInt64> & ids) override;
BlockInputStreamPtr loadKeys(
const Columns & key_columns, const std::vector<size_t> & requested_rows) override;
BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows) override;
bool isModified() const override;
@ -79,12 +78,9 @@ private:
std::string invalidate_query;
mutable std::string invalidate_query_response;
ODBCBridgeHelper odbc_bridge_helper;
BridgeHelperPtr bridge_helper;
Poco::URI bridge_url;
ConnectionTimeouts timeouts;
const Context & global_context;
};
}

2
dbms/src/Dictionaries/readInvalidateQuery.h
View File

@ -6,7 +6,7 @@ class IProfilingBlockInputStream;
namespace DB
{
// Using in MySQLDictionarySource and ODBCDictionarySource after processing invalidate_query
// Using in MySQLDictionarySource and XDBCDictionarySource after processing invalidate_query
std::string readInvalidateQuery(IProfilingBlockInputStream & block_input_stream);

6
dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp
View File

@ -17,6 +17,8 @@ namespace ErrorCodes
extern const int CANNOT_PARSE_NUMBER;
extern const int CANNOT_PARSE_UUID;
extern const int TOO_LARGE_STRING_SIZE;
extern const int CANNOT_READ_ALL_DATA;
extern const int INCORRECT_DATA;
}
@ -40,7 +42,9 @@ static bool isParseError(int code)
|| code == ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT
|| code == ErrorCodes::CANNOT_PARSE_NUMBER
|| code == ErrorCodes::CANNOT_PARSE_UUID
|| code == ErrorCodes::TOO_LARGE_STRING_SIZE;
|| code == ErrorCodes::TOO_LARGE_STRING_SIZE
|| code == ErrorCodes::CANNOT_READ_ALL_DATA
|| code == ErrorCodes::INCORRECT_DATA;
}

75
dbms/src/Formats/JSONEachRowRowInputStream.cpp
View File

@ -12,6 +12,7 @@ namespace ErrorCodes
{
extern const int INCORRECT_DATA;
extern const int CANNOT_READ_ALL_DATA;
extern const int LOGICAL_ERROR;
}
namespace
@ -23,7 +24,7 @@ enum
NESTED_FIELD = size_t(-2)
};
} // unnamed namespace
}
JSONEachRowRowInputStream::JSONEachRowRowInputStream(ReadBuffer & istr_, const Block & header_, const FormatSettings & format_settings)
@ -35,7 +36,7 @@ JSONEachRowRowInputStream::JSONEachRowRowInputStream(ReadBuffer & istr_, const B
size_t num_columns = header.columns();
for (size_t i = 0; i < num_columns; ++i)
{
const String& colname = columnName(i);
const String & colname = columnName(i);
name_map[colname] = i; /// NOTE You could place names more cache-locally.
if (format_settings.import_nested_json)
{
@ -47,39 +48,60 @@ JSONEachRowRowInputStream::JSONEachRowRowInputStream(ReadBuffer & istr_, const B
}
}
}
prev_positions.assign(num_columns, name_map.end());
}
const String& JSONEachRowRowInputStream::columnName(size_t i) const
const String & JSONEachRowRowInputStream::columnName(size_t i) const
{
return header.safeGetByPosition(i).name;
return header.getByPosition(i).name;
}
size_t JSONEachRowRowInputStream::columnIndex(const StringRef& name) const
inline size_t JSONEachRowRowInputStream::columnIndex(const StringRef & name, size_t key_index)
{
/// NOTE Optimization is possible by caching the order of fields (which is almost always the same)
/// Optimization by caching the order of fields (which is almost always the same)
/// and a quick check to match the next expected field, instead of searching the hash table.
const auto it = name_map.find(name);
return name_map.end() == it ? UNKNOWN_FIELD : it->second;
if (prev_positions.size() > key_index
&& prev_positions[key_index] != name_map.end()
&& name == prev_positions[key_index]->first)
{
return prev_positions[key_index]->second;
}
else
{
const auto it = name_map.find(name);
if (name_map.end() != it)
{
if (key_index < prev_positions.size())
prev_positions[key_index] = it;
return it->second;
}
else
return UNKNOWN_FIELD;
}
}
/** Read the field name and convert it to column name
* (taking into account the current nested name prefix)
* Resulting StringRef is valid only before next read from buf.
*/
StringRef JSONEachRowRowInputStream::readColumnName(ReadBuffer & buf)
{
// This is just an optimization: try to avoid copying the name into current_column_name
if (nested_prefix_length == 0 && buf.position() + 1 < buf.buffer().end())
{
const char * next_pos = find_first_symbols<'\\', '"'>(buf.position() + 1, buf.buffer().end());
char * next_pos = find_first_symbols<'\\', '"'>(buf.position() + 1, buf.buffer().end());
if (next_pos != buf.buffer().end() && *next_pos != '\\')
{
/// The most likely option is that there is no escape sequence in the key name, and the entire name is placed in the buffer.
assertChar('"', buf);
StringRef res(buf.position(), next_pos - buf.position());
buf.position() += next_pos - buf.position();
assertChar('"', buf);
buf.position() = next_pos + 1;
return res;
}
}
@ -90,7 +112,7 @@ StringRef JSONEachRowRowInputStream::readColumnName(ReadBuffer & buf)
}
static void skipColonDelimeter(ReadBuffer & istr)
static inline void skipColonDelimeter(ReadBuffer & istr)
{
skipWhitespaceIfAny(istr);
assertChar(':', istr);
@ -123,7 +145,7 @@ void JSONEachRowRowInputStream::readField(size_t index, MutableColumns & columns
read_columns[index] = true;
}
bool JSONEachRowRowInputStream::advanceToNextKey(size_t key_index)
inline bool JSONEachRowRowInputStream::advanceToNextKey(size_t key_index)
{
skipWhitespaceIfAny(istr);
@ -150,16 +172,31 @@ void JSONEachRowRowInputStream::readJSONObject(MutableColumns & columns)
for (size_t key_index = 0; advanceToNextKey(key_index); ++key_index)
{
StringRef name_ref = readColumnName(istr);
const size_t column_index = columnIndex(name_ref, key_index);
skipColonDelimeter(istr);
if (unlikely(ssize_t(column_index) < 0))
{
/// name_ref may point directly to the input buffer
/// and input buffer may be filled with new data on next read
/// If we want to use name_ref after another reads from buffer, we must copy it to temporary string.
const size_t column_index = columnIndex(name_ref);
if (column_index == UNKNOWN_FIELD)
skipUnknownField(name_ref);
else if (column_index == NESTED_FIELD)
readNestedData(name_ref.toString(), columns);
current_column_name.assign(name_ref.data, name_ref.size);
name_ref = StringRef(current_column_name);
skipColonDelimeter(istr);
if (column_index == UNKNOWN_FIELD)
skipUnknownField(name_ref);
else if (column_index == NESTED_FIELD)
readNestedData(name_ref.toString(), columns);
else
throw Exception("Logical error: illegal value of column_index", ErrorCodes::LOGICAL_ERROR);
}
else
{
skipColonDelimeter(istr);
readField(column_index, columns);
}
}
}

5
dbms/src/Formats/JSONEachRowRowInputStream.h
View File

@ -28,7 +28,7 @@ public:
private:
const String & columnName(size_t i) const;
size_t columnIndex(const StringRef & name) const;
size_t columnIndex(const StringRef & name, size_t key_index);
bool advanceToNextKey(size_t key_index);
void skipUnknownField(const StringRef & name_ref);
StringRef readColumnName(ReadBuffer & buf);
@ -60,6 +60,9 @@ private:
/// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map.
using NameMap = HashMap<StringRef, size_t, StringRefHash>;
NameMap name_map;
/// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
std::vector<NameMap::iterator> prev_positions;
};
}

13
dbms/src/Functions/GeoUtils.h
View File

@ -622,17 +622,22 @@ float calcLinestringRotation(const Linestring & points)
auto sqrLength = [](const Point & point) { return point.x() * point.x() + point.y() * point.y(); };
auto vecProduct = [](const Point & from, const Point & to) { return from.x() * to.y() - from.y() * to.x(); };
auto scalarProduct = [](const Point & from, const Point & to) { return from.x() * to.x() + from.y() * to.y(); };
auto getVector = [](const Point & from, const Point & to) -> Point
{
return Point(to.x() - from.x(), to.y() - from.y());
};
for (auto it = points.begin(); std::next(it) != points.end(); ++it)
for (auto it = points.begin(); it != points.end(); ++it)
{
if (it != points.begin())
{
auto prev = std::prev(it);
auto next = std::next(it);
if (next == points.end())
next = std::next(points.begin());
Point from = getVector(*prev, *it);
Point to = getVector(*it, *next);
float sqr_from_len = sqrLength(from);
@ -641,9 +646,13 @@ float calcLinestringRotation(const Linestring & points)
if (std::isfinite(sqr_len_product))
{
float vec_prod = vecProduct(from, to);
float scalar_prod = scalarProduct(from, to);
float sin_ang = vec_prod * std::fabs(vec_prod) / sqr_len_product;
float cos_ang = scalar_prod * std::fabs(scalar_prod) / sqr_len_product;
sin_ang = std::max(-1.f, std::min(1.f, sin_ang));
rotation += std::asin(sin_ang);
cos_ang = std::max(-1.f, std::min(1.f, cos_ang));
float ang = std::atan2(sin_ang, cos_ang);
rotation += ang;
}
}
}

11
dbms/src/IO/ReadHelpers.cpp
View File

@ -71,7 +71,7 @@ UInt128 stringToUUID(const String & str)
return parseFromString<UUID>(str);
}
static void __attribute__((__noinline__)) throwAtAssertionFailed(const char * s, ReadBuffer & buf)
void NO_INLINE throwAtAssertionFailed(const char * s, ReadBuffer & buf)
{
WriteBufferFromOwnString out;
out << "Cannot parse input: expected " << escape << s;
@ -120,15 +120,6 @@ void assertString(const char * s, ReadBuffer & buf)
throwAtAssertionFailed(s, buf);
}
void assertChar(char symbol, ReadBuffer & buf)
{
if (buf.eof() || *buf.position() != symbol)
{
char err[2] = {symbol, '\0'};
throwAtAssertionFailed(err, buf);
}
++buf.position();
}
void assertEOF(ReadBuffer & buf)
{

13
dbms/src/IO/ReadHelpers.h
View File

@ -162,7 +162,18 @@ void readVectorBinary(std::vector<T> & v, ReadBuffer & buf, size_t MAX_VECTOR_SI
void assertString(const char * s, ReadBuffer & buf);
void assertEOF(ReadBuffer & buf);
void assertChar(char symbol, ReadBuffer & buf);
void throwAtAssertionFailed(const char * s, ReadBuffer & buf);
inline void assertChar(char symbol, ReadBuffer & buf)
{
if (buf.eof() || *buf.position() != symbol)
{
char err[2] = {symbol, '\0'};
throwAtAssertionFailed(err, buf);
}
++buf.position();
}
inline void assertString(const String & s, ReadBuffer & buf)
{

129
dbms/src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -14,6 +14,7 @@
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/formatAST.h>
#include <Parsers/DumpASTNode.h>
#include <DataTypes/DataTypeSet.h>
#include <DataTypes/DataTypeNullable.h>
@ -35,6 +36,7 @@
#include <Interpreters/Join.h>
#include <Interpreters/ProjectionManipulation.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Interpreters/TranslateQualifiedNamesVisitor.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
@ -68,13 +70,16 @@
#include <Parsers/queryToString.h>
#include <Interpreters/evaluateQualified.h>
#include <Interpreters/QueryNormalizer.h>
#include <Interpreters/getQueryAliases.h>
#include <Interpreters/QueryAliasesVisitor.h>
#include <DataTypes/DataTypeLowCardinality.h>
namespace DB
{
using LogAST = DebugASTLog<false>; /// set to true to enable logs
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
@ -229,7 +234,11 @@ ExpressionAnalyzer::ExpressionAnalyzer(
LogicalExpressionsOptimizer(select_query, settings).perform();
/// Creates a dictionary `aliases`: alias -> ASTPtr
getQueryAliases(query, aliases);
{
LogAST log;
QueryAliasesVisitor query_aliases_visitor(log.stream());
query_aliases_visitor.visit(query, aliases);
}
/// Common subexpression elimination. Rewrite rules.
normalizeTree();
@ -309,100 +318,29 @@ void ExpressionAnalyzer::translateQualifiedNames()
std::vector<DatabaseAndTableWithAlias> tables;
std::vector<ASTTableExpression> tables_expression = getTableExpressions(query);
LogAST log;
for (const auto & table_expression : tables_expression)
tables.emplace_back(getTableNameWithAliasFromTableExpression(table_expression, context));
translateQualifiedNamesImpl(query, tables);
}
void ExpressionAnalyzer::translateQualifiedNamesImpl(ASTPtr & ast, const std::vector<DatabaseAndTableWithAlias> & tables)
{
if (auto * identifier = typeid_cast<ASTIdentifier *>(ast.get()))
{
if (identifier->general())
{
/// Select first table name with max number of qualifiers which can be stripped.
size_t max_num_qualifiers_to_strip = 0;
size_t best_table_pos = 0;
auto table = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase());
for (size_t table_pos = 0; table_pos < tables.size(); ++table_pos)
{
const auto & table = tables[table_pos];
auto num_qualifiers_to_strip = getNumComponentsToStripInOrderToTranslateQualifiedName(*identifier, table);
if (num_qualifiers_to_strip > max_num_qualifiers_to_strip)
{
max_num_qualifiers_to_strip = num_qualifiers_to_strip;
best_table_pos = table_pos;
}
}
stripIdentifier(ast, max_num_qualifiers_to_strip);
/// In case if column from the joined table are in source columns, change it's name to qualified.
if (best_table_pos && source_columns.contains(ast->getColumnName()))
tables[best_table_pos].makeQualifiedName(ast);
}
}
else if (typeid_cast<ASTQualifiedAsterisk *>(ast.get()))
{
if (ast->children.size() != 1)
throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
ASTIdentifier * ident = typeid_cast<ASTIdentifier *>(ast->children[0].get());
if (!ident)
throw Exception("Logical error: qualified asterisk must have identifier as its child", ErrorCodes::LOGICAL_ERROR);
size_t num_components = ident->children.size();
if (num_components > 2)
throw Exception("Qualified asterisk cannot have more than two qualifiers", ErrorCodes::UNKNOWN_ELEMENT_IN_AST);
for (const auto & table_names : tables)
{
/// database.table.*, table.* or alias.*
if ((num_components == 2
&& !table_names.database.empty()
&& static_cast<const ASTIdentifier &>(*ident->children[0]).name == table_names.database
&& static_cast<const ASTIdentifier &>(*ident->children[1]).name == table_names.table)
|| (num_components == 0
&& ((!table_names.table.empty() && ident->name == table_names.table)
|| (!table_names.alias.empty() && ident->name == table_names.alias))))
{
return;
}
{ /// debug print
size_t depth = 0;
DumpASTNode dump(table_expression, log.stream(), depth, "getTableNames");
if (table_expression.database_and_table_name)
DumpASTNode(*table_expression.database_and_table_name, log.stream(), depth);
if (table_expression.table_function)
DumpASTNode(*table_expression.table_function, log.stream(), depth);
if (table_expression.subquery)
DumpASTNode(*table_expression.subquery, log.stream(), depth);
dump.print("getTableNameWithAlias", table.database + '.' + table.table + ' ' + table.alias);
}
throw Exception("Unknown qualified identifier: " + ident->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
tables.emplace_back(table);
}
else if (auto * join = typeid_cast<ASTTableJoin *>(ast.get()))
{
/// Don't translate on_expression here in order to resolve equation parts later.
if (join->using_expression_list)
translateQualifiedNamesImpl(join->using_expression_list, tables);
}
else
{
/// If the WHERE clause or HAVING consists of a single quailified column, the reference must be translated not only in children, but also in where_expression and having_expression.
if (ASTSelectQuery * select = typeid_cast<ASTSelectQuery *>(ast.get()))
{
if (select->prewhere_expression)
translateQualifiedNamesImpl(select->prewhere_expression, tables);
if (select->where_expression)
translateQualifiedNamesImpl(select->where_expression, tables);
if (select->having_expression)
translateQualifiedNamesImpl(select->having_expression, tables);
}
for (auto & child : ast->children)
{
/// Do not go to FROM, JOIN, subqueries.
if (!typeid_cast<const ASTTableExpression *>(child.get())
&& !typeid_cast<const ASTSelectWithUnionQuery *>(child.get()))
{
translateQualifiedNamesImpl(child, tables);
}
}
}
TranslateQualifiedNamesVisitor visitor(source_columns, tables, log.stream());
visitor.visit(query);
}
void ExpressionAnalyzer::optimizeIfWithConstantCondition()
@ -924,7 +862,7 @@ void ExpressionAnalyzer::normalizeTree()
bool first = true;
for (const auto & table_expression : tables_expression)
{
const auto table_name = getTableNameWithAliasFromTableExpression(table_expression, context);
const auto table_name = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase());
NamesAndTypesList names_and_types = getNamesAndTypeListFromTableExpression(table_expression, context);
if (!first)
@ -2347,7 +2285,7 @@ const ExpressionAnalyzer::AnalyzedJoin::JoinedColumnsList & ExpressionAnalyzer::
if (const ASTTablesInSelectQueryElement * node = select_query_with_join->join())
{
const auto & table_expression = static_cast<const ASTTableExpression &>(*node->table_expression);
auto table_name_with_alias = getTableNameWithAliasFromTableExpression(table_expression, context);
auto table_name_with_alias = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase());
auto columns = getNamesAndTypeListFromTableExpression(table_expression, context);
@ -2943,8 +2881,8 @@ void ExpressionAnalyzer::collectJoinedColumnsFromJoinOnExpr()
const auto & left_table_expression = static_cast<const ASTTableExpression &>(*left_tables_element->table_expression);
const auto & right_table_expression = static_cast<const ASTTableExpression &>(*right_tables_element->table_expression);
auto left_source_names = getTableNameWithAliasFromTableExpression(left_table_expression, context);
auto right_source_names = getTableNameWithAliasFromTableExpression(right_table_expression, context);
auto left_source_names = getTableNameWithAliasFromTableExpression(left_table_expression, context.getCurrentDatabase());
auto right_source_names = getTableNameWithAliasFromTableExpression(right_table_expression, context.getCurrentDatabase());
/// Stores examples of columns which are only from one table.
struct TableBelonging
@ -2997,8 +2935,7 @@ void ExpressionAnalyzer::collectJoinedColumnsFromJoinOnExpr()
std::function<void(ASTPtr &, const DatabaseAndTableWithAlias &, bool)> translate_qualified_names;
translate_qualified_names = [&](ASTPtr & ast, const DatabaseAndTableWithAlias & source_names, bool right_table)
{
auto * identifier = typeid_cast<const ASTIdentifier *>(ast.get());
if (identifier)
if (auto * identifier = typeid_cast<const ASTIdentifier *>(ast.get()))
{
if (identifier->general())
{
@ -3098,7 +3035,7 @@ void ExpressionAnalyzer::collectJoinedColumns(NameSet & joined_columns)
const auto & table_join = static_cast<const ASTTableJoin &>(*node->table_join);
const auto & table_expression = static_cast<const ASTTableExpression &>(*node->table_expression);
auto joined_table_name = getTableNameWithAliasFromTableExpression(table_expression, context);
auto joined_table_name = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase());
auto add_name_to_join_keys = [&](Names & join_keys, ASTs & join_asts, const ASTPtr & ast, bool right_table)
{

2
dbms/src/Interpreters/ExpressionAnalyzer.h
View File

@ -4,7 +4,6 @@
#include <Interpreters/Settings.h>
#include <Core/Block.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/evaluateQualified.h>
#include <Interpreters/ProjectionManipulation.h>
#include <Parsers/StringRange.h>
#include <Parsers/ASTTablesInSelectQuery.h>
@ -420,7 +419,6 @@ private:
* only one ("main") table is supported. Ambiguity is not detected or resolved.
*/
void translateQualifiedNames();
void translateQualifiedNamesImpl(ASTPtr & node, const std::vector<DatabaseAndTableWithAlias> & tables);
/** Sometimes we have to calculate more columns in SELECT clause than will be returned from query.
* This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result.

14
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -57,6 +57,7 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
extern const int DATABASE_ALREADY_EXISTS;
extern const int QUERY_IS_PROHIBITED;
extern const int THERE_IS_NO_DEFAULT_VALUE;
}
@ -224,6 +225,10 @@ static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast
const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, columns}.getActions(true);
const auto block = actions->getSampleBlock();
for (auto action : actions->getActions())
if (action.type == ExpressionAction::Type::JOIN || action.type == ExpressionAction::Type::ARRAY_JOIN)
throw Exception("Cannot CREATE table. Unsupported default value that requires ARRAY JOIN or JOIN action", ErrorCodes::THERE_IS_NO_DEFAULT_VALUE);
for (auto & column : defaulted_columns)
{
const auto name_and_type_ptr = column.first;
@ -364,9 +369,8 @@ void InterpreterCreateQuery::checkSupportedTypes(const ColumnsDescription & colu
{
const auto & settings = context.getSettingsRef();
bool allow_low_cardinality = settings.allow_experimental_low_cardinality_type != 0;
bool allow_decimal = settings.allow_experimental_decimal_type;
if (allow_low_cardinality && allow_decimal)
if (allow_low_cardinality)
return;
auto check_types = [&](const NamesAndTypesList & list)
@ -380,12 +384,6 @@ void InterpreterCreateQuery::checkSupportedTypes(const ColumnsDescription & colu
+ "Set setting allow_experimental_low_cardinality_type = 1 in order to allow it.";
throw Exception(message, ErrorCodes::ILLEGAL_COLUMN);
}
if (!allow_decimal && column.type && isDecimal(column.type))
{
String message = "Cannot create table with column '" + column.name + "' which type is '" + column.type->getName()
+ "'. Set setting allow_experimental_decimal_type = 1 in order to allow it.";
throw Exception(message, ErrorCodes::ILLEGAL_COLUMN);
}
}
};

52
dbms/src/Interpreters/InterpreterExplainQuery.cpp Normal file
View File

@ -0,0 +1,52 @@
#include <sstream>
#include <DataStreams/OneBlockInputStream.h>
#include <DataStreams/BlockIO.h>
#include <DataTypes/DataTypeString.h>
#include <Parsers/queryToString.h>
#include <Common/typeid_cast.h>
#include <Interpreters/InterpreterExplainQuery.h>
#include <Parsers/ASTExplainQuery.h>
#include <Parsers/DumpASTNode.h>
namespace DB
{
BlockIO InterpreterExplainQuery::execute()
{
BlockIO res;
res.in = executeImpl();
return res;
}
Block InterpreterExplainQuery::getSampleBlock()
{
Block block;
ColumnWithTypeAndName col;
col.name = "ast";
col.type = std::make_shared<DataTypeString>();
col.column = col.type->createColumn();
block.insert(col);
return block;
}
BlockInputStreamPtr InterpreterExplainQuery::executeImpl()
{
const ASTExplainQuery & ast = typeid_cast<const ASTExplainQuery &>(*query);
std::stringstream ss;
dumpAST(ast, ss);
Block sample_block = getSampleBlock();
MutableColumns res_columns = sample_block.cloneEmptyColumns();
res_columns[0]->insert(ss.str());
return std::make_shared<OneBlockInputStream>(sample_block.cloneWithColumns(std::move(res_columns)));
}
}

32
dbms/src/Interpreters/InterpreterExplainQuery.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#include <Interpreters/IInterpreter.h>
namespace DB
{
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
/// Returns single row with explain results
class InterpreterExplainQuery : public IInterpreter
{
public:
InterpreterExplainQuery(const ASTPtr & query_, const Context & )
: query(query_)
{}
BlockIO execute() override;
static Block getSampleBlock();
private:
ASTPtr query;
BlockInputStreamPtr executeImpl();
};
}

6
dbms/src/Interpreters/InterpreterFactory.cpp
View File

@ -12,12 +12,14 @@
#include <Parsers/ASTShowProcesslistQuery.h>
#include <Parsers/ASTShowTablesQuery.h>
#include <Parsers/ASTUseQuery.h>
#include <Parsers/ASTExplainQuery.h>
#include <Parsers/TablePropertiesQueriesASTs.h>
#include <Interpreters/InterpreterAlterQuery.h>
#include <Interpreters/InterpreterCheckQuery.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/InterpreterDescribeQuery.h>
#include <Interpreters/InterpreterExplainQuery.h>
#include <Interpreters/InterpreterDropQuery.h>
#include <Interpreters/InterpreterExistsQuery.h>
#include <Interpreters/InterpreterFactory.h>
@ -145,6 +147,10 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, Context &
{
return std::make_unique<InterpreterDescribeQuery>(query, context);
}
else if (typeid_cast<ASTExplainQuery *>(query.get()))
{
return std::make_unique<InterpreterExplainQuery>(query, context);
}
else if (typeid_cast<ASTShowProcesslistQuery *>(query.get()))
{
return std::make_unique<InterpreterShowProcesslistQuery>(query, context);

1
dbms/src/Interpreters/InterpreterSelectQuery.h
View File

@ -16,7 +16,6 @@ namespace Poco { class Logger; }
namespace DB
{
class ExpressionAnalyzer;
class ASTSelectQuery;
struct SubqueryForSet;
class InterpreterSelectWithUnionQuery;

8
dbms/src/Interpreters/LogicalExpressionsOptimizer.cpp
View File

@ -228,10 +228,17 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain
/// Construct a list of literals `x1, ..., xN` from the string `expr = x1 OR ... OR expr = xN`
ASTPtr value_list = std::make_shared<ASTExpressionList>();
const char * min_range_first = nullptr;
const char * max_range_second = nullptr;
for (const auto function : equality_functions)
{
const auto & operands = getFunctionOperands(function);
value_list->children.push_back(operands[1]);
/// Get range min/max from all literals x1...xN, which will be used as tuple_functions' range
if (min_range_first == nullptr || min_range_first > operands[1]->range.first)
min_range_first = operands[1]->range.first;
if (max_range_second == nullptr || max_range_second < operands[1]->range.second)
max_range_second = operands[1]->range.second;
}
/// Sort the literals so that they are specified in the same order in the IN expression.
@ -253,6 +260,7 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain
auto tuple_function = std::make_shared<ASTFunction>();
tuple_function->name = "tuple";
tuple_function->range = StringRange(min_range_first, max_range_second);
tuple_function->arguments = value_list;
tuple_function->children.push_back(tuple_function->arguments);

14
dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp
View File

@ -8,7 +8,7 @@
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/queryToString.h>
#include <Interpreters/QueryNormalizer.h>
#include <Interpreters/getQueryAliases.h>
#include <Interpreters/QueryAliasesVisitor.h>
namespace DB
{
@ -47,7 +47,8 @@ bool PredicateExpressionsOptimizer::optimizeImpl(
std::vector<ASTTableExpression *> tables_expression = getSelectTablesExpression(ast_select);
std::vector<DatabaseAndTableWithAlias> database_and_table_with_aliases;
for (const auto & table_expression : tables_expression)
database_and_table_with_aliases.emplace_back(getTableNameWithAliasFromTableExpression(*table_expression, context));
database_and_table_with_aliases.emplace_back(
getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase()));
bool is_rewrite_subquery = false;
for (const auto & outer_predicate : outer_predicate_expressions)
@ -267,7 +268,8 @@ void PredicateExpressionsOptimizer::getAllSubqueryProjectionColumns(SubqueriesPr
if (table_expression->subquery)
{
/// Use qualifiers to translate the columns of subqueries
const auto database_and_table_with_alias = getTableNameWithAliasFromTableExpression(*table_expression, context);
const auto database_and_table_with_alias =
getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase());
String qualified_name_prefix = database_and_table_with_alias.getQualifiedNamePrefix();
getSubqueryProjectionColumns(all_subquery_projection_columns, qualified_name_prefix,
static_cast<const ASTSubquery *>(table_expression->subquery.get())->children[0]);
@ -304,7 +306,8 @@ ASTs PredicateExpressionsOptimizer::getSelectQueryProjectionColumns(ASTPtr & ast
{
/// first should normalize query tree.
std::unordered_map<String, ASTPtr> aliases;
getQueryAliases(ast, aliases, 0);
QueryAliasesVisitor query_aliases_visitor;
query_aliases_visitor.visit(ast, aliases, 0);
QueryNormalizer(ast, aliases, settings, {}, {}).perform();
ASTs projection_columns;
@ -351,7 +354,8 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que
for (auto it = tables_expression.begin(); it != tables_expression.end(); ++it)
{
const ASTTableExpression * table_expression = *it;
const auto database_and_table_with_alias = getTableNameWithAliasFromTableExpression(*table_expression, context);
const auto database_and_table_with_alias =
getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase());
/// database.table.*
if (num_components == 2 && !database_and_table_with_alias.database.empty()
&& static_cast<const ASTIdentifier &>(*ident->children[0]).name == database_and_table_with_alias.database

1
dbms/src/Interpreters/ProjectionManipulation.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <string>
#include <Interpreters/ExpressionAnalyzer.h>
namespace DB
{

15
dbms/src/Interpreters/getQueryAliases.cpp → dbms/src/Interpreters/QueryAliasesVisitor.cpp
View File

@ -1,11 +1,12 @@
#include <ostream>
#include <sstream>
#include <Interpreters/getQueryAliases.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Common/typeid_cast.h>
#include <Interpreters/QueryAliasesVisitor.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/DumpASTNode.h>
#include <IO/WriteHelpers.h>
namespace DB
@ -18,8 +19,9 @@ namespace ErrorCodes
/// ignore_levels - aliases in how many upper levels of the subtree should be ignored.
/// For example, with ignore_levels=1 ast can not be put in the dictionary, but its children can.
void getQueryAliases(ASTPtr & ast, Aliases & aliases, int ignore_levels)
void QueryAliasesVisitor::getQueryAliases(const ASTPtr & ast, Aliases & aliases, int ignore_levels) const
{
DumpASTNode dump(*ast, ostr, visit_depth, "getQueryAliases");
/// Bottom-up traversal. We do not go into subqueries.
for (auto & child : ast->children)
@ -40,6 +42,11 @@ void getQueryAliases(ASTPtr & ast, Aliases & aliases, int ignore_levels)
if (ignore_levels > 0)
return;
getNodeAlias(ast, aliases, dump);
}
void QueryAliasesVisitor::getNodeAlias(const ASTPtr & ast, Aliases & aliases, const DumpASTNode & dump) const
{
String alias = ast->tryGetAlias();
if (!alias.empty())
{
@ -56,6 +63,7 @@ void getQueryAliases(ASTPtr & ast, Aliases & aliases, int ignore_levels)
}
aliases[alias] = ast;
dump.print(visit_action, alias);
}
else if (auto subquery = typeid_cast<ASTSubquery *>(ast.get()))
{
@ -76,6 +84,7 @@ void getQueryAliases(ASTPtr & ast, Aliases & aliases, int ignore_levels)
subquery->setAlias(alias);
subquery->prefer_alias_to_column_name = true;
aliases[alias] = ast;
dump.print(visit_action, alias);
}
}
}

34
dbms/src/Interpreters/QueryAliasesVisitor.h Normal file
View File

@ -0,0 +1,34 @@
#pragma once
#include <Parsers/DumpASTNode.h>
#include <unordered_map>
namespace DB
{
using Aliases = std::unordered_map<String, ASTPtr>;
/// Visits AST nodes and collect their aliases in one map (with links to source nodes).
class QueryAliasesVisitor
{
public:
QueryAliasesVisitor(std::ostream * ostr_ = nullptr)
: visit_depth(0),
ostr(ostr_)
{}
void visit(const ASTPtr & ast, Aliases & aliases, int ignore_levels = 0) const
{
getQueryAliases(ast, aliases, ignore_levels);
}
private:
static constexpr const char * visit_action = "addAlias";
mutable size_t visit_depth;
std::ostream * ostr;
void getQueryAliases(const ASTPtr & ast, Aliases & aliases, int ignore_levels) const;
void getNodeAlias(const ASTPtr & ast, Aliases & aliases, const DumpASTNode & dump) const;
};
}

2
dbms/src/Interpreters/Settings.h
View File

@ -282,7 +282,6 @@ struct Settings
M(SettingUInt64, low_cardinality_max_dictionary_size, 8192, "Maximum size (in rows) of shared global dictionary for LowCardinality type.") \
M(SettingBool, low_cardinality_use_single_dictionary_for_part, false, "LowCardinality type serialization setting. If is true, than will use additional keys when global dictionary overflows. Otherwise, will create several shared dictionaries.") \
M(SettingBool, allow_experimental_low_cardinality_type, false, "Allows to create table with LowCardinality types.") \
M(SettingBool, allow_experimental_decimal_type, false, "Enables Decimal data type.") \
M(SettingBool, decimal_check_overflow, true, "Check overflow of decimal arithmetic/comparison operations") \
\
M(SettingBool, prefer_localhost_replica, 1, "1 - always send query to local replica, if it exists. 0 - choose replica to send query between local and remote ones according to load_balancing") \
@ -292,6 +291,7 @@ struct Settings
M(SettingBool, calculate_text_stack_trace, 1, "Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when huge amount of wrong queries are executed. In normal cases you should not disable this option.") \
M(SettingBool, allow_ddl, true, "If it is set to true, then a user is allowed to executed DDL queries.") \
M(SettingBool, parallel_view_processing, false, "Enables pushing to attached views concurrently instead of sequentially.") \
M(SettingBool, enable_debug_queries, false, "Enables debug queries such as AST.") \
#define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \

120
dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp Normal file
View File

@ -0,0 +1,120 @@
#include <Interpreters/TranslateQualifiedNamesVisitor.h>
#include <Core/NamesAndTypes.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNKNOWN_IDENTIFIER;
}
void TranslateQualifiedNamesVisitor::visit(ASTIdentifier * identifier, ASTPtr & ast, const DumpASTNode & dump) const
{
if (identifier->general())
{
/// Select first table name with max number of qualifiers which can be stripped.
size_t max_num_qualifiers_to_strip = 0;
size_t best_table_pos = 0;
for (size_t table_pos = 0; table_pos < tables.size(); ++table_pos)
{
const auto & table = tables[table_pos];
auto num_qualifiers_to_strip = getNumComponentsToStripInOrderToTranslateQualifiedName(*identifier, table);
if (num_qualifiers_to_strip > max_num_qualifiers_to_strip)
{
max_num_qualifiers_to_strip = num_qualifiers_to_strip;
best_table_pos = table_pos;
}
}
if (max_num_qualifiers_to_strip)
{
dump.print(String("stripIdentifier ") + identifier->name, max_num_qualifiers_to_strip);
stripIdentifier(ast, max_num_qualifiers_to_strip);
}
/// In case if column from the joined table are in source columns, change it's name to qualified.
if (best_table_pos && source_columns.contains(ast->getColumnName()))
{
const DatabaseAndTableWithAlias & table = tables[best_table_pos];
table.makeQualifiedName(ast);
dump.print("makeQualifiedName", table.database + '.' + table.table + ' ' + ast->getColumnName());
}
}
}
void TranslateQualifiedNamesVisitor::visit(ASTQualifiedAsterisk *, ASTPtr & ast, const DumpASTNode &) const
{
if (ast->children.size() != 1)
throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
ASTIdentifier * ident = typeid_cast<ASTIdentifier *>(ast->children[0].get());
if (!ident)
throw Exception("Logical error: qualified asterisk must have identifier as its child", ErrorCodes::LOGICAL_ERROR);
size_t num_components = ident->children.size();
if (num_components > 2)
throw Exception("Qualified asterisk cannot have more than two qualifiers", ErrorCodes::UNKNOWN_ELEMENT_IN_AST);
for (const auto & table_names : tables)
{
/// database.table.*, table.* or alias.*
if ((num_components == 2
&& !table_names.database.empty()
&& static_cast<const ASTIdentifier &>(*ident->children[0]).name == table_names.database
&& static_cast<const ASTIdentifier &>(*ident->children[1]).name == table_names.table)
|| (num_components == 0
&& ((!table_names.table.empty() && ident->name == table_names.table)
|| (!table_names.alias.empty() && ident->name == table_names.alias))))
{
return;
}
}
throw Exception("Unknown qualified identifier: " + ident->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
}
void TranslateQualifiedNamesVisitor::visit(ASTTableJoin * join, ASTPtr &, const DumpASTNode &) const
{
/// Don't translate on_expression here in order to resolve equation parts later.
if (join->using_expression_list)
visit(join->using_expression_list);
}
void TranslateQualifiedNamesVisitor::visit(ASTSelectQuery * select, ASTPtr & ast, const DumpASTNode &) const
{
/// If the WHERE clause or HAVING consists of a single quailified column, the reference must be translated not only in children,
/// but also in where_expression and having_expression.
if (select->prewhere_expression)
visit(select->prewhere_expression);
if (select->where_expression)
visit(select->where_expression);
if (select->having_expression)
visit(select->having_expression);
visitChildren(ast);
}
void TranslateQualifiedNamesVisitor::visitChildren(ASTPtr & ast) const
{
for (auto & child : ast->children)
{
/// Do not go to FROM, JOIN, subqueries.
if (!typeid_cast<const ASTTableExpression *>(child.get())
&& !typeid_cast<const ASTSelectWithUnionQuery *>(child.get()))
{
visit(child);
}
}
}
}

69
dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h Normal file
View File

@ -0,0 +1,69 @@
#pragma once
#include <memory>
#include <vector>
#include <Common/typeid_cast.h>
#include <Parsers/DumpASTNode.h>
#include <Interpreters/evaluateQualified.h>
namespace DB
{
class ASTIdentifier;
class ASTQualifiedAsterisk;
class ASTSelectQuery;
struct ASTTableJoin;
class NamesAndTypesList;
/// It visits nodes, find identifiers and translate their names to needed form.
class TranslateQualifiedNamesVisitor
{
public:
TranslateQualifiedNamesVisitor(const NamesAndTypesList & source_columns_, const std::vector<DatabaseAndTableWithAlias> & tables_,
std::ostream * ostr_ = nullptr)
: source_columns(source_columns_),
tables(tables_),
visit_depth(0),
ostr(ostr_)
{}
void visit(ASTPtr & ast) const
{
DumpASTNode dump(*ast, ostr, visit_depth, "translateQualifiedNames");
if (!tryVisit<ASTIdentifier>(ast, dump) &&
!tryVisit<ASTQualifiedAsterisk>(ast, dump) &&
!tryVisit<ASTTableJoin>(ast, dump) &&
!tryVisit<ASTSelectQuery>(ast, dump))
visitChildren(ast); /// default: do nothing, visit children
}
private:
const NamesAndTypesList & source_columns;
const std::vector<DatabaseAndTableWithAlias> & tables;
mutable size_t visit_depth;
std::ostream * ostr;
void visit(ASTIdentifier * node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTQualifiedAsterisk * node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTTableJoin * node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTSelectQuery * ast, ASTPtr &, const DumpASTNode & dump) const;
void visitChildren(ASTPtr &) const;
template <typename T>
bool tryVisit(ASTPtr & ast, const DumpASTNode & dump) const
{
if (T * t = typeid_cast<T *>(ast.get()))
{
visit(t, ast, dump);
return true;
}
return false;
}
};
}

8
dbms/src/Interpreters/evaluateQualified.cpp
View File

@ -2,6 +2,10 @@
#include <Interpreters/Context.h>
#include <Common/typeid_cast.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTTablesInSelectQuery.h>
namespace DB
{
@ -44,7 +48,7 @@ void stripIdentifier(DB::ASTPtr & ast, size_t num_qualifiers_to_strip)
DatabaseAndTableWithAlias getTableNameWithAliasFromTableExpression(const ASTTableExpression & table_expression,
const Context & context)
const String & current_database)
{
DatabaseAndTableWithAlias database_and_table_with_alias;
@ -56,7 +60,7 @@ DatabaseAndTableWithAlias getTableNameWithAliasFromTableExpression(const ASTTabl
if (table_expression.database_and_table_name->children.empty())
{
database_and_table_with_alias.database = context.getCurrentDatabase();
database_and_table_with_alias.database = current_database;
database_and_table_with_alias.table = identifier.name;
}
else

17
dbms/src/Interpreters/evaluateQualified.h
View File

@ -1,13 +1,18 @@
#pragma once
#include <Parsers/IAST.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Interpreters/Context.h>
#include <memory>
#include <Core/Types.h>
namespace DB
{
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
class ASTIdentifier;
struct ASTTableExpression;
struct DatabaseAndTableWithAlias
{
String database;
@ -24,11 +29,11 @@ struct DatabaseAndTableWithAlias
void stripIdentifier(DB::ASTPtr & ast, size_t num_qualifiers_to_strip);
DatabaseAndTableWithAlias getTableNameWithAliasFromTableExpression(const ASTTableExpression & table_expression,
const Context & context);
const String & current_database);
size_t getNumComponentsToStripInOrderToTranslateQualifiedName(const ASTIdentifier & identifier,
const DatabaseAndTableWithAlias & names);
std::pair<String, String> getDatabaseAndTableNameFromIdentifier(const ASTIdentifier & identifier);
}
}

2
dbms/src/Interpreters/executeQuery.cpp
View File

@ -142,7 +142,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
const Settings & settings = context.getSettingsRef();
ParserQuery parser(end);
ParserQuery parser(end, settings.enable_debug_queries);
ASTPtr ast;
size_t query_size;

13
dbms/src/Interpreters/getQueryAliases.h
View File

@ -1,13 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <unordered_map>
namespace DB
{
using Aliases = std::unordered_map<String, ASTPtr>;
void getQueryAliases(ASTPtr & ast, Aliases & aliases, int ignore_levels = 0);
}

45
dbms/src/Parsers/ASTExplainQuery.h Normal file
View File

@ -0,0 +1,45 @@
#pragma once
#include <Parsers/IAST.h>
namespace DB
{
/// AST, EXPLAIN or other query with meaning of explanation query instead of execution
class ASTExplainQuery : public IAST
{
public:
enum ExplainKind
{
ParsedAST,
};
ASTExplainQuery(ExplainKind kind_ = ParsedAST)
: kind(kind_)
{}
String getID() const override { return "Explain_" + toString(kind); }
ASTPtr clone() const override { return std::make_shared<ASTExplainQuery>(*this); }
protected:
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << toString(kind) << (settings.hilite ? hilite_none : "");
}
private:
ExplainKind kind;
static String toString(ExplainKind kind)
{
switch (kind)
{
case ParsedAST: return "ParsedAST";
}
__builtin_unreachable();
}
};
}

121
dbms/src/Parsers/DumpASTNode.h Normal file
View File

@ -0,0 +1,121 @@
#pragma once
#include <sstream>
#include <common/logger_useful.h>
#include <Poco/Util/Application.h>
#include <Parsers/IAST.h>
namespace DB
{
/// If output stream set dumps node with indents and some additional info. Do nothing otherwise.
/// Allow to print kay-value pairs inside of tree dump.
class DumpASTNode
{
public:
DumpASTNode(const IAST & ast_, std::ostream * ostr_, size_t & depth, const char * label_ = nullptr)
: ast(ast_),
ostr(ostr_),
indent(depth),
visit_depth(depth),
label(label_)
{
if (!ostr)
return;
if (label && visit_depth == 0)
(*ostr) << "-- " << label << std::endl;
++visit_depth;
(*ostr) << String(indent, ' ');
printNode();
(*ostr) << std::endl;
}
~DumpASTNode()
{
if (!ostr)
return;
--visit_depth;
if (label && visit_depth == 0)
(*ostr) << "--" << std::endl;
}
template <typename T, typename U>
void print(const T & name, const U & value, const char * str_indent = nullptr) const
{
if (!ostr)
return;
(*ostr) << (str_indent ? String(str_indent) : String(indent, ' '));
(*ostr) << '(' << name << ' ' << value << ')';
if (!str_indent)
(*ostr) << std::endl;
}
size_t & getDepth() { return visit_depth; }
private:
const IAST & ast;
std::ostream * ostr;
size_t indent;
size_t & visit_depth; /// shared with children
const char * label;
String nodeId() const
{
String id = ast.getID();
std::replace(id.begin(), id.end(), '_', ' ');
return id;
}
void printNode() const
{
(*ostr) << nodeId();
String aslias = ast.tryGetAlias();
if (!aslias.empty())
print("alias", aslias, " ");
if (!ast.children.empty())
print("/", ast.children.size(), " "); /// slash is just a short name for 'children' here
}
};
inline void dumpAST(const IAST & ast, std::ostream & ostr, DumpASTNode * parent = nullptr)
{
size_t depth = 0;
DumpASTNode dump(ast, &ostr, (parent ? parent->getDepth() : depth));
for (const auto & child : ast.children)
dumpAST(*child, ostr, &dump);
}
/// String stream dumped in dtor
template <bool _enable>
class DebugASTLog
{
public:
DebugASTLog()
: log(nullptr)
{
if constexpr (_enable)
log = &Poco::Logger::get("AST");
}
~DebugASTLog()
{
if constexpr (_enable)
LOG_DEBUG(log, ss.str());
}
std::ostream * stream() { return (_enable ? &ss : nullptr); }
private:
Poco::Logger * log;
std::stringstream ss;
};
}

2
dbms/src/Parsers/IAST.h
View File

@ -3,6 +3,7 @@
#include <set>
#include <memory>
#include <ostream>
#include <algorithm>
#include <Core/Types.h>
#include <Common/Exception.h>
@ -218,5 +219,4 @@ private:
/// Surrounds an identifier by back quotes if it is necessary.
String backQuoteIfNeed(const String & x);
}

2
dbms/src/Parsers/ParserCase.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/IParserBase.h>
namespace DB
{

2
dbms/src/Parsers/ParserCheckQuery.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/IParserBase.h>
namespace DB
{

1
dbms/src/Parsers/ParserDescribeTableQuery.h
View File

@ -2,7 +2,6 @@
#include <Parsers/IParserBase.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/ExpressionElementParsers.h>

2
dbms/src/Parsers/ParserQuery.cpp
View File

@ -17,7 +17,7 @@ namespace DB
bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ParserQueryWithOutput query_with_output_p;
ParserQueryWithOutput query_with_output_p(enable_explain);
ParserInsertQuery insert_p(end);
ParserUseQuery use_p;
ParserSetQuery set_p;

7
dbms/src/Parsers/ParserQuery.h
View File

@ -10,11 +10,16 @@ class ParserQuery : public IParserBase
{
private:
const char * end;
bool enable_explain;
const char * getName() const override { return "Query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
public:
ParserQuery(const char * end) : end(end) {}
ParserQuery(const char * end, bool enable_explain_ = false)
: end(end),
enable_explain(enable_explain_)
{}
};
}

16
dbms/src/Parsers/ParserQueryWithOutput.cpp
View File

@ -11,6 +11,7 @@
#include <Parsers/ParserDropQuery.h>
#include <Parsers/ParserKillQueryQuery.h>
#include <Parsers/ParserOptimizeQuery.h>
#include <Parsers/ASTExplainQuery.h>
namespace DB
@ -33,6 +34,12 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
ASTPtr query;
ParserKeyword s_ast("AST");
bool explain_ast = false;
if (enable_explain && s_ast.ignore(pos, expected))
explain_ast = true;
bool parsed = select_p.parse(pos, query, expected)
|| show_tables_p.parse(pos, query, expected)
|| table_p.parse(pos, query, expected)
@ -74,7 +81,14 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
query_with_output.children.push_back(query_with_output.format);
}
node = query;
if (explain_ast)
{
node = std::make_shared<ASTExplainQuery>();
node->children.push_back(query);
}
else
node = query;
return true;
}

8
dbms/src/Parsers/ParserQueryWithOutput.h
View File

@ -10,10 +10,18 @@ namespace DB
/// Parse queries supporting [INTO OUTFILE 'file_name'] [FORMAT format_name] suffix.
class ParserQueryWithOutput : public IParserBase
{
public:
ParserQueryWithOutput(bool enable_explain_ = false)
: enable_explain(enable_explain_)
{}
protected:
const char * getName() const override { return "Query with output"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
private:
bool enable_explain;
};
}

1
dbms/src/Parsers/ParserShowProcesslistQuery.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <Parsers/IParserBase.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/CommonParsers.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/ASTShowProcesslistQuery.h>

2
dbms/src/Parsers/ParserShowTablesQuery.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/IParserBase.h>
namespace DB

1
dbms/src/Parsers/ParserTablePropertiesQuery.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <Parsers/IParserBase.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/ExpressionElementParsers.h>

2
dbms/src/Storages/MergeTree/MergeTreeRangeReader.h
View File

@ -68,7 +68,7 @@ public:
Stream() = default;
Stream(size_t from_mark, size_t to_mark, size_t index_granularity, MergeTreeReader * merge_tree_reader);
/// Returns the n
/// Returns the number of rows added to block.
size_t read(Block & block, size_t num_rows, bool skip_remaining_rows_in_current_granule);
size_t finalize(Block & block);
void skip(size_t num_rows);

115
dbms/src/Storages/StorageODBC.cpp
View File

@ -1,115 +0,0 @@
#include <Dictionaries/ODBCBlockInputStream.h>
#include <Interpreters/Context.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTLiteral.h>
#include <Storages/StorageFactory.h>
#include <Storages/StorageODBC.h>
#include <Storages/transformQueryForExternalDatabase.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <common/logger_useful.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Common/ShellCommand.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int EXTERNAL_EXECUTABLE_NOT_FOUND;
extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING;
}
StorageODBC::StorageODBC(const std::string & table_name_,
const std::string & connection_string,
const std::string & remote_database_name_,
const std::string & remote_table_name_,
const ColumnsDescription & columns_,
const Context & context_)
: IStorageURLBase(Poco::URI(), context_, table_name_, ODBCBridgeHelper::DEFAULT_FORMAT, columns_)
, odbc_bridge_helper(context_global.getConfigRef(), context_global.getSettingsRef().http_receive_timeout.value, connection_string)
, remote_database_name(remote_database_name_)
, remote_table_name(remote_table_name_)
, log(&Poco::Logger::get("StorageODBC"))
{
uri = odbc_bridge_helper.getMainURI();
}
std::string StorageODBC::getReadMethod() const
{
return Poco::Net::HTTPRequest::HTTP_POST;
}
std::vector<std::pair<std::string, std::string>> StorageODBC::getReadURIParams(const Names & column_names,
const SelectQueryInfo & /*query_info*/,
const Context & /*context*/,
QueryProcessingStage::Enum & /*processed_stage*/,
size_t max_block_size) const
{
NamesAndTypesList cols;
for (const String & name : column_names)
{
auto column_data = getColumn(name);
cols.emplace_back(column_data.name, column_data.type);
}
return odbc_bridge_helper.getURLParams(cols.toString(), max_block_size);
}
std::function<void(std::ostream &)> StorageODBC::getReadPOSTDataCallback(const Names & /*column_names*/,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & /*processed_stage*/,
size_t /*max_block_size*/) const
{
String query = transformQueryForExternalDatabase(
*query_info.query, getColumns().ordinary, IdentifierQuotingStyle::DoubleQuotes, remote_database_name, remote_table_name, context);
return [query](std::ostream & os) { os << "query=" << query; };
}
BlockInputStreams StorageODBC::read(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams)
{
check(column_names);
odbc_bridge_helper.startODBCBridgeSync();
return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams);
}
Block StorageODBC::getHeaderBlock(const Names & column_names) const
{
return getSampleBlockForColumns(column_names);
}
void registerStorageODBC(StorageFactory & factory)
{
factory.registerStorage("ODBC", [](const StorageFactory::Arguments & args)
{
ASTs & engine_args = args.engine_args;
if (engine_args.size() != 3)
throw Exception(
"Storage ODBC requires exactly 3 parameters: ODBC('DSN', database or schema, table)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (size_t i = 0; i < 3; ++i)
engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.local_context);
return StorageODBC::create(args.table_name,
static_cast<const ASTLiteral &>(*engine_args[0]).value.safeGet<String>(),
static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>(),
static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>(),
args.columns,
args.context);
});
}
}

61
dbms/src/Storages/StorageODBC.h
View File

@ -1,61 +0,0 @@
#pragma once
#include <Storages/StorageURL.h>
#include <Common/ODBCBridgeHelper.h>
#include <ext/shared_ptr_helper.h>
namespace DB
{
/** Implements storage in the ODBC database.
* Use ENGINE = odbc(connection_string, table_name)
* Example ENGINE = odbc('dsn=test', table)
* Read only.
*/
class StorageODBC : public ext::shared_ptr_helper<StorageODBC>, public IStorageURLBase
{
public:
std::string getName() const override
{
return "ODBC";
}
BlockInputStreams read(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
protected:
StorageODBC(const std::string & table_name_,
const std::string & connection_string,
const std::string & remote_database_name,
const std::string & remote_table_name,
const ColumnsDescription & columns_,
const Context & context_);
private:
ODBCBridgeHelper odbc_bridge_helper;
std::string remote_database_name;
std::string remote_table_name;
Poco::Logger * log;
std::string getReadMethod() const override;
std::vector<std::pair<std::string, std::string>> getReadURIParams(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
size_t max_block_size) const override;
std::function<void(std::ostream &)> getReadPOSTDataCallback(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
size_t max_block_size) const override;
Block getHeaderBlock(const Names & column_names) const override;
};
}

139
dbms/src/Storages/StorageXDBC.cpp Normal file
View File

@ -0,0 +1,139 @@
#include <Interpreters/Context.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTLiteral.h>
#include <Storages/StorageFactory.h>
#include <Storages/StorageXDBC.h>
#include <Storages/transformQueryForExternalDatabase.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <common/logger_useful.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Poco/File.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Common/ShellCommand.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
StorageXDBC::StorageXDBC(const std::string & table_name_,
const std::string & remote_database_name_,
const std::string & remote_table_name_,
const ColumnsDescription & columns_,
const Context & context_,
const BridgeHelperPtr bridge_helper_)
: IStorageURLBase(Poco::URI(), context_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_)
, bridge_helper(bridge_helper_)
, remote_database_name(remote_database_name_)
, remote_table_name(remote_table_name_)
{
log = &Poco::Logger::get("Storage" + bridge_helper->getName());
uri = bridge_helper->getMainURI();
}
std::string StorageXDBC::getReadMethod() const
{
return Poco::Net::HTTPRequest::HTTP_POST;
}
std::vector<std::pair<std::string, std::string>> StorageXDBC::getReadURIParams(const Names & column_names,
const SelectQueryInfo & /*query_info*/,
const Context & /*context*/,
QueryProcessingStage::Enum & /*processed_stage*/,
size_t max_block_size) const
{
NamesAndTypesList cols;
for (const String & name : column_names)
{
auto column_data = getColumn(name);
cols.emplace_back(column_data.name, column_data.type);
}
return bridge_helper->getURLParams(cols.toString(), max_block_size);
}
std::function<void(std::ostream &)> StorageXDBC::getReadPOSTDataCallback(const Names & /*column_names*/,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & /*processed_stage*/,
size_t /*max_block_size*/) const
{
String query = transformQueryForExternalDatabase(*query_info.query,
getColumns().ordinary,
bridge_helper->getIdentifierQuotingStyle(),
remote_database_name,
remote_table_name,
context);
return [query](std::ostream & os) { os << "query=" << query; };
}
BlockInputStreams StorageXDBC::read(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams)
{
check(column_names);
bridge_helper->startBridgeSync();
return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams);
}
Block StorageXDBC::getHeaderBlock(const Names & column_names) const
{
return getSampleBlockForColumns(column_names);
}
std::string StorageXDBC::getName() const
{
return bridge_helper->getName();
}
namespace
{
template <typename BridgeHelperMixin>
void registerXDBCStorage(StorageFactory & factory, const std::string & name)
{
factory.registerStorage(name, [&name](const StorageFactory::Arguments & args)
{
ASTs & engine_args = args.engine_args;
if (engine_args.size() != 3)
throw Exception("Storage " + name + " requires exactly 3 parameters: " + name + "('DSN', database or schema, table)",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (size_t i = 0; i < 3; ++i)
engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.local_context);
BridgeHelperPtr bridge_helper = std::make_shared<XDBCBridgeHelper<BridgeHelperMixin>>(args.context.getConfigRef(),
args.context.getSettingsRef().http_receive_timeout.value,
static_cast<const ASTLiteral &>(*engine_args[0]).value.safeGet<String>());
return std::make_shared<StorageXDBC>(args.table_name,
static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>(),
static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>(),
args.columns,
args.context,
bridge_helper);
});
}
}
void registerStorageJDBC(StorageFactory & factory)
{
registerXDBCStorage<JDBCBridgeMixin>(factory, "JDBC");
}
void registerStorageODBC(StorageFactory & factory)
{
registerXDBCStorage<ODBCBridgeMixin>(factory, "ODBC");
}
}

58
dbms/src/Storages/StorageXDBC.h Normal file
View File

@ -0,0 +1,58 @@
#pragma once
#include <Storages/StorageURL.h>
#include <ext/shared_ptr_helper.h>
#include <Common/XDBCBridgeHelper.h>
namespace DB
{
/** Implements storage in the XDBC database.
* Use ENGINE = xdbc(connection_string, table_name)
* Example ENGINE = odbc('dsn=test', table)
* Read only.
*/
class StorageXDBC : public IStorageURLBase
{
public:
BlockInputStreams read(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
unsigned num_streams) override;
StorageXDBC(const std::string & table_name_,
const std::string & remote_database_name,
const std::string & remote_table_name,
const ColumnsDescription & columns_,
const Context & context_, BridgeHelperPtr bridge_helper_);
private:
BridgeHelperPtr bridge_helper;
std::string remote_database_name;
std::string remote_table_name;
Poco::Logger * log;
std::string getReadMethod() const override;
std::vector<std::pair<std::string, std::string>> getReadURIParams(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
size_t max_block_size) const override;
std::function<void(std::ostream &)> getReadPOSTDataCallback(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
size_t max_block_size) const override;
Block getHeaderBlock(const Names & column_names) const override;
std::string getName() const override;
};
}

4
dbms/src/Storages/registerStorages.cpp
View File

@ -28,6 +28,8 @@ void registerStorageMaterializedView(StorageFactory & factory);
void registerStorageODBC(StorageFactory & factory);
#endif
void registerStorageJDBC(StorageFactory & factory);
#if USE_MYSQL
void registerStorageMySQL(StorageFactory & factory);
#endif
@ -61,6 +63,8 @@ void registerStorages()
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
registerStorageODBC(factory);
#endif
registerStorageJDBC(factory);
#if USE_MYSQL
registerStorageMySQL(factory);

34
dbms/src/Storages/transformQueryForExternalDatabase.cpp
View File

@ -1,5 +1,6 @@
#include <sstream>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypesNumber.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
@ -8,22 +9,46 @@
#include <Parsers/ASTExpressionList.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Storages/transformQueryForExternalDatabase.h>
#include <Storages/MergeTree/KeyCondition.h>
namespace DB
{
static void replaceConstFunction(IAST & node, const Context & context, const NamesAndTypesList & all_columns)
{
for (size_t i = 0; i < node.children.size(); ++i)
{
auto child = node.children[i];
if (ASTExpressionList * exp_list = typeid_cast<ASTExpressionList *>(&*child))
replaceConstFunction(*exp_list, context, all_columns);
if (ASTFunction * function = typeid_cast<ASTFunction *>(&*child))
{
auto result_block = KeyCondition::getBlockWithConstants(function->ptr(), context, all_columns);
if (!result_block.has(child->getColumnName()))
return;
auto result_column = result_block.getByName(child->getColumnName()).column;
node.children[i] = std::make_shared<ASTLiteral>((*result_column)[0]);
}
}
}
static bool isCompatible(const IAST & node)
{
if (const ASTFunction * function = typeid_cast<const ASTFunction *>(&node))
{
String name = function->name;
if (!(name == "and"
|| name == "or"
|| name == "not"
|| name == "equals"
|| name == "notEquals"
|| name == "like"
|| name == "notLike"
|| name == "in"
|| name == "greater"
|| name == "less"
|| name == "lessOrEquals"
@ -62,7 +87,8 @@ String transformQueryForExternalDatabase(
const String & table,
const Context & context)
{
ExpressionAnalyzer analyzer(query.clone(), context, {}, available_columns);
auto clone_query = query.clone();
ExpressionAnalyzer analyzer(clone_query, context, {}, available_columns);
const Names & used_columns = analyzer.getRequiredSourceColumns();
auto select = std::make_shared<ASTSelectQuery>();
@ -81,9 +107,10 @@ String transformQueryForExternalDatabase(
* copy only compatible parts of it.
*/
const ASTPtr & original_where = typeid_cast<const ASTSelectQuery &>(query).where_expression;
ASTPtr & original_where = typeid_cast<ASTSelectQuery &>(*clone_query).where_expression;
if (original_where)
{
replaceConstFunction(*original_where, context, available_columns);
if (isCompatible(*original_where))
{
select->where_expression = original_where;
@ -112,6 +139,7 @@ String transformQueryForExternalDatabase(
settings.identifier_quoting_style = identifier_quoting_style;
select->format(settings);
return out.str();
}

52
dbms/src/TableFunctions/TableFunctionODBC.cpp → dbms/src/TableFunctions/ITableFunctionXDBC.cpp
View File

@ -1,67 +1,71 @@
#include <TableFunctions/TableFunctionODBC.h>
#include <type_traits>
#include <ext/scope_guard.h>
#include <Core/Defines.h>
#include <DataTypes/DataTypeFactory.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/parseQuery.h>
#include <Poco/Net/HTTPRequest.h>
#include <Storages/StorageODBC.h>
#include <Storages/StorageXDBC.h>
#include <TableFunctions/ITableFunction.h>
#include <TableFunctions/ITableFunctionXDBC.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Poco/Net/HTTPRequest.h>
#include <Common/Exception.h>
#include <Common/typeid_cast.h>
#include <Common/ODBCBridgeHelper.h>
#include <Core/Defines.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int UNKNOWN_EXCEPTION;
extern const int LOGICAL_ERROR;
}
StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Context & context) const
StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Context & context) const
{
const ASTFunction & args_func = typeid_cast<const ASTFunction &>(*ast_function);
if (!args_func.arguments)
throw Exception("Table function 'odbc' must have arguments.", ErrorCodes::LOGICAL_ERROR);
throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR);
ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.arguments).children;
if (args.size() != 2 && args.size() != 3)
throw Exception("Table function 'odbc' requires 2 or 3 arguments: ODBC('DSN', table) or ODBC('DSN', schema, table)",
throw Exception("Table function '" + getName() + "' requires 2 or 3 arguments: " + getName() + "('DSN', table) or " + getName()
+ "('DSN', schema, table)",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (auto i = 0u; i < args.size(); ++i)
args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context);
std::string connection_string = "";
std::string schema_name = "";
std::string table_name = "";
std::string connection_string;
std::string schema_name;
std::string table_name;
if (args.size() == 3)
{
connection_string = static_cast<const ASTLiteral &>(*args[0]).value.safeGet<String>();
schema_name = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
table_name = static_cast<const ASTLiteral &>(*args[2]).value.safeGet<String>();
} else if (args.size() == 2)
}
else if (args.size() == 2)
{
connection_string = static_cast<const ASTLiteral &>(*args[0]).value.safeGet<String>();
table_name = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
}
const auto & config = context.getConfigRef();
ODBCBridgeHelper helper(config, context.getSettingsRef().http_receive_timeout.value, connection_string);
helper.startODBCBridgeSync();
Poco::URI columns_info_uri = helper.getColumnsInfoURI();
/* Infer external table structure */
BridgeHelperPtr helper = createBridgeHelper(config, context.getSettingsRef().http_receive_timeout.value, connection_string);
helper->startBridgeSync();
Poco::URI columns_info_uri = helper->getColumnsInfoURI();
columns_info_uri.addQueryParameter("connection_string", connection_string);
if (!schema_name.empty())
columns_info_uri.addQueryParameter("schema", schema_name);
@ -73,14 +77,22 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con
readStringBinary(columns_info, buf);
NamesAndTypesList columns = NamesAndTypesList::parse(columns_info);
auto result = StorageODBC::create(table_name, connection_string, schema_name, table_name, ColumnsDescription{columns}, context);
auto result = std::make_shared<StorageXDBC>(table_name, schema_name, table_name, ColumnsDescription{columns}, context, helper);
if (!result)
throw Exception("Failed to instantiate storage from table function " + getName(), ErrorCodes::UNKNOWN_EXCEPTION);
result->startup();
return result;
}
void registerTableFunctionJDBC(TableFunctionFactory & factory)
{
factory.registerFunction<TableFunctionJDBC>();
}
void registerTableFunctionODBC(TableFunctionFactory & factory)
{
factory.registerFunction<TableFunctionODBC>();
}
}
}

61
dbms/src/TableFunctions/ITableFunctionXDBC.h Normal file
View File

@ -0,0 +1,61 @@
#pragma once
#include <Storages/StorageXDBC.h>
#include <TableFunctions/ITableFunction.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/XDBCBridgeHelper.h>
#include <Common/config.h>
namespace DB
{
/**
* Base class for table functions, that works over external bridge
* Xdbc (Xdbc connect string, table) - creates a temporary StorageXDBC.
*/
class ITableFunctionXDBC : public ITableFunction
{
private:
StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override;
/* A factory method to create bridge helper, that will assist in remote interaction */
virtual BridgeHelperPtr createBridgeHelper(const Poco::Util::AbstractConfiguration & config_,
const Poco::Timespan & http_timeout_,
const std::string & connection_string_) const = 0;
};
class TableFunctionJDBC : public ITableFunctionXDBC
{
public:
static constexpr auto name = "jdbc";
std::string getName() const override
{
return name;
}
private:
BridgeHelperPtr createBridgeHelper(const Poco::Util::AbstractConfiguration & config_,
const Poco::Timespan & http_timeout_,
const std::string & connection_string_) const override
{
return std::make_shared<XDBCBridgeHelper<JDBCBridgeMixin>>(config_, http_timeout_, connection_string_);
}
};
class TableFunctionODBC : public ITableFunctionXDBC
{
public:
static constexpr auto name = "odbc";
std::string getName() const override
{
return name;
}
private:
BridgeHelperPtr createBridgeHelper(const Poco::Util::AbstractConfiguration & config_,
const Poco::Timespan & http_timeout_,
const std::string & connection_string_) const override
{
return std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(config_, http_timeout_, connection_string_);
}
};
}

22
dbms/src/TableFunctions/TableFunctionODBC.h
View File

@ -1,22 +0,0 @@
#pragma once
#include <Common/config.h>
#include <TableFunctions/ITableFunction.h>
namespace DB
{
/* odbc (odbc connect string, table) - creates a temporary StorageODBC.
*/
class TableFunctionODBC : public ITableFunction
{
public:
static constexpr auto name = "odbc";
std::string getName() const override
{
return name;
}
private:
StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override;
};
}

3
dbms/src/TableFunctions/registerTableFunctions.cpp
View File

@ -18,6 +18,8 @@ void registerTableFunctionURL(TableFunctionFactory & factory);
void registerTableFunctionODBC(TableFunctionFactory & factory);
#endif
void registerTableFunctionJDBC(TableFunctionFactory & factory);
#if USE_MYSQL
void registerTableFunctionMySQL(TableFunctionFactory & factory);
#endif
@ -38,6 +40,7 @@ void registerTableFunctions()
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
registerTableFunctionODBC(factory);
#endif
registerTableFunctionJDBC(factory);
#if USE_MYSQL
registerTableFunctionMySQL(factory);

1
dbms/tests/clickhouse-test
View File

@ -86,7 +86,6 @@ def main(args):
server_logs_level = "warning"
os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level)
# TODO ! use clickhouse-extract-from-config here:
if args.zookeeper is None:
code, out = commands.getstatusoutput(args.binary + "-extract-from-config --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
try:

4
dbms/tests/integration/test_dictionaries/generate_dictionaries.py
View File

@ -20,11 +20,11 @@ explicit_defaults = [
]
implicit_defaults = [
'1', '1', '1', '1',
'1', '1', '1', '',
'-1', '-1', '-1', '-1',
'2.71828', '2.71828',
'implicit-default',
'2015-11-25', '2015-11-25 00:00:00'
'2015-11-25', ''
]

11
dbms/tests/integration/test_dictionaries/test.py
View File

@ -121,3 +121,14 @@ def test_select_all_from_cached(cached_dictionary_structure):
diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True)
print test_table.process_diff(diff)
assert not diff
def test_null_value(started_cluster):
query = instance.query
assert TSV(query("select dictGetUInt8('clickhouse_cache', 'UInt8_', toUInt64(12121212))")) == TSV("1")
assert TSV(query("select dictGetString('clickhouse_cache', 'String_', toUInt64(12121212))")) == TSV("implicit-default")
assert TSV(query("select dictGetDate('clickhouse_cache', 'Date_', toUInt64(12121212))")) == TSV("2015-11-25")
# Check, that empty null_value interprets as default value
assert TSV(query("select dictGetUInt64('clickhouse_cache', 'UInt64_', toUInt64(12121212))")) == TSV("0")
assert TSV(query("select dictGetDateTime('clickhouse_cache', 'DateTime_', toUInt64(12121212))")) == TSV("0000-00-00 00:00:00")

15
dbms/tests/integration/test_storage_mysql/test.py
View File

@ -76,6 +76,21 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL
assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '60000'
conn.close()
def test_where(started_cluster):
table_name = 'test_where'
conn = get_mysql_conn()
create_mysql_table(conn, table_name)
node1.query('''
CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse');
'''.format(table_name, table_name))
node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name))
assert node1.query("SELECT count() FROM {} WHERE name LIKE '%name_%'".format(table_name)).rstrip() == '10000'
assert node1.query("SELECT count() FROM {} WHERE name NOT LIKE '%tmp_%'".format(table_name)).rstrip() == '10000'
assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 3)".format(table_name)).rstrip() == '10000'
assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '0'
assert node1.query("SELECT count() FROM {} WHERE money NOT IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '10000'
assert node1.query("SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format(table_name)).rstrip() == '1'
conn.close()
def get_mysql_conn():
conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308)

2
dbms/tests/queries/0_stateless/00500_point_in_polygon_bug_2.reference Normal file
View File

@ -0,0 +1,2 @@
0
1

2
dbms/tests/queries/0_stateless/00500_point_in_polygon_bug_2.sql Normal file
View File

@ -0,0 +1,2 @@
select pointInPolygon((35.45285, 58.72587), [(32.947998, 59.506455), (34.222412, 59.215312), (33.343506, 58.551061), (34.859619, 58.938673), (36.463623, 58.528125), (35.397949, 59.215312), (36.804199, 59.495303), (35.2771, 59.50088), (34.892578, 60.267066), (34.343262, 59.517603), (32.947998, 59.506455)]);
select pointInPolygon((35.72308, 58.8294), [(32.947998, 59.506455), (34.222412, 59.215312), (33.343506, 58.551061), (34.859619, 58.938673), (36.463623, 58.528125), (35.397949, 59.215312), (36.804199, 59.495303), (35.2771, 59.50088), (34.892578, 60.267066), (34.343262, 59.517603), (32.947998, 59.506455)]);

1
dbms/tests/queries/0_stateless/00700_decimal_aggregates.sql
View File

@ -1,4 +1,3 @@
SET allow_experimental_decimal_type = 1;
SET send_logs_level = 'none';
CREATE DATABASE IF NOT EXISTS test;

1
dbms/tests/queries/0_stateless/00700_decimal_arithm.sql
View File

@ -1,4 +1,3 @@
SET allow_experimental_decimal_type = 1;
SET send_logs_level = 'none';
CREATE DATABASE IF NOT EXISTS test;

Some files were not shown because too many files have changed in this diff Show More